From 7e509a0b8fa5a1a64291255ad486808260f716e7 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Mon, 1 Jun 2026 21:34:19 +0200 Subject: [PATCH 01/45] feat(chat): Add durable agent continuation queue Move Slack turn execution onto a durable mailbox and Vercel Queue wake-up path so serverless timeouts and vanished workers can recover without another user message. Add conversation leases, worker check-ins, heartbeat repair, timeout continuation scheduling, and the /api/internal/agent/continue callback. Remove routine visible continuation notices and document the new task execution contract. Co-Authored-By: GPT-5 Codex --- .agents/skills/pi-agent-integration/SKILL.md | 69 +- .../skills/pi-agent-integration/SOURCES.md | 108 +-- .agents/skills/pi-agent-integration/SPEC.md | 84 ++ .../references/api-surface.md | 151 ++-- .../references/common-use-cases.md | 86 +- .../references/harness.md | 96 +++ .../references/troubleshooting-workarounds.md | 86 +- AGENTS.md | 1 + packages/junior/package.json | 1 + packages/junior/src/app.ts | 17 + .../src/chat/agent-dispatch/heartbeat.ts | 85 +- .../junior/src/chat/agent-dispatch/runner.ts | 7 +- packages/junior/src/chat/app/production.ts | 225 ++---- packages/junior/src/chat/config.ts | 10 +- .../junior/src/chat/ingress/junior-chat.ts | 9 +- .../junior/src/chat/ingress/slack-webhook.ts | 580 +++++++++++++ packages/junior/src/chat/respond.ts | 26 +- .../junior/src/chat/runtime/reply-executor.ts | 63 +- .../src/chat/runtime/request-deadline.ts | 42 + .../junior/src/chat/runtime/slack-resume.ts | 34 - .../src/chat/runtime/timeout-resume-runner.ts | 313 ++++++++ .../src/chat/services/subscribed-decision.ts | 5 +- .../src/chat/services/timeout-resume.ts | 89 +- .../services/turn-continuation-response.ts | 7 - .../junior/src/chat/slack/adapter-context.ts | 124 +++ .../chat/slack/turn-continuation-notice.ts | 24 - .../src/chat/task-execution/heartbeat.ts | 133 +++ .../junior/src/chat/task-execution/queue.ts | 19 + .../src/chat/task-execution/slack-work.ts | 306 +++++++ .../junior/src/chat/task-execution/store.ts | 759 ++++++++++++++++++ .../chat/task-execution/vercel-callback.ts | 77 ++ .../src/chat/task-execution/vercel-queue.ts | 71 ++ .../junior/src/chat/task-execution/worker.ts | 299 +++++++ packages/junior/src/handlers/heartbeat.ts | 11 +- .../junior/src/handlers/mcp-oauth-callback.ts | 22 +- .../junior/src/handlers/oauth-callback.ts | 11 +- packages/junior/src/handlers/turn-resume.ts | 349 +------- packages/junior/src/handlers/webhooks.ts | 163 ++-- packages/junior/src/nitro.ts | 2 +- packages/junior/src/vercel.ts | 13 + .../integration/conversation-work.test.ts | 702 ++++++++++++++++ .../tests/integration/heartbeat.test.ts | 68 ++ .../integration/slack/bot-handlers.test.ts | 53 +- .../integration/turn-resume-slack.test.ts | 128 ++- .../junior/tests/unit/cli/init-cli.test.ts | 12 +- .../tests/unit/config/chat-config.test.ts | 10 +- .../unit/config/turn-timeout-matrix.test.ts | 12 +- .../tests/unit/handlers/oauth-resume.test.ts | 9 +- .../tests/unit/handlers/turn-resume.test.ts | 11 +- .../runtime/respond-timeout-resume.test.ts | 61 +- .../tests/unit/runtime/timeout-resume.test.ts | 124 +-- packages/junior/tests/unit/vercel.test.ts | 12 + packages/junior/tsup.config.ts | 1 + pnpm-lock.yaml | 29 +- specs/agent-session-resumability.md | 222 +++-- specs/agent-turn-handling.md | 2 +- specs/chat-architecture.md | 109 +-- specs/index.md | 4 +- specs/scheduler.md | 3 +- specs/slack-agent-delivery.md | 10 +- specs/task-execution.md | 469 +++++++++++ specs/trusted-plugin-dispatch.md | 1 + 62 files changed, 5231 insertions(+), 1398 deletions(-) create mode 100644 .agents/skills/pi-agent-integration/SPEC.md create mode 100644 .agents/skills/pi-agent-integration/references/harness.md create mode 100644 packages/junior/src/chat/ingress/slack-webhook.ts create mode 100644 packages/junior/src/chat/runtime/request-deadline.ts create mode 100644 packages/junior/src/chat/runtime/timeout-resume-runner.ts delete mode 100644 packages/junior/src/chat/services/turn-continuation-response.ts create mode 100644 packages/junior/src/chat/slack/adapter-context.ts delete mode 100644 packages/junior/src/chat/slack/turn-continuation-notice.ts create mode 100644 packages/junior/src/chat/task-execution/heartbeat.ts create mode 100644 packages/junior/src/chat/task-execution/queue.ts create mode 100644 packages/junior/src/chat/task-execution/slack-work.ts create mode 100644 packages/junior/src/chat/task-execution/store.ts create mode 100644 packages/junior/src/chat/task-execution/vercel-callback.ts create mode 100644 packages/junior/src/chat/task-execution/vercel-queue.ts create mode 100644 packages/junior/src/chat/task-execution/worker.ts create mode 100644 packages/junior/tests/integration/conversation-work.test.ts create mode 100644 specs/task-execution.md diff --git a/.agents/skills/pi-agent-integration/SKILL.md b/.agents/skills/pi-agent-integration/SKILL.md index 3241ef8c7..9d0e5cb30 100644 --- a/.agents/skills/pi-agent-integration/SKILL.md +++ b/.agents/skills/pi-agent-integration/SKILL.md @@ -1,53 +1,58 @@ --- name: pi-agent-integration -description: Integrate `@mariozechner/pi-agent-core` as the agent abstraction inside another library or runtime. Use when implementing or refactoring Pi Agent wrappers, streaming bridges, `convertToLlm`/`transformContext`, queueing via `steer`/`followUp`, `continue()` semantics, or timeout/abort/session behavior. +description: Integrate the latest `@earendil-works/pi-agent-core` APIs into an app, library, runtime, or agent harness. Use for Pi `Agent`, `AgentHarness`, streaming bridges, tool execution hooks, `convertToLlm`/`transformContext`, queueing via `steer`/`followUp`, `continue()` semantics, `streamFn`/`streamProxy`, timeout/abort, session, skill, or compaction behavior. --- -Implement Pi-agent consumers with stable streaming, correct queue semantics, and minimal wrapper surface area. +Implement Pi-agent consumers against the latest published Pi API with stable streaming, correct queue semantics, and minimal wrapper surface area. ## Step 1: Classify the request Pick the path before editing: -| Request type | Read first | -| --- | --- | -| Wiring or updating agent wrapper APIs/options | `references/api-surface.md` | -| Adding behavior in a consumer library (chat, orchestration, tools) | `references/common-use-cases.md` | -| Debugging broken streaming/tool/continue behavior | `references/troubleshooting-workarounds.md` | +| Request type | Read first | +| --------------------------------------------------------------------------------- | ------------------------------------------- | +| Wiring or updating `Agent`, loop, provider, stream, or tool APIs | `references/api-surface.md` | +| Adding Pi behavior in a consuming app, library, or runtime | `references/common-use-cases.md` | +| Using Pi's built-in harness, sessions, skills, resources, or compaction | `references/harness.md` | +| Debugging broken streaming, tools, queues, continuation, proxy, or abort behavior | `references/troubleshooting-workarounds.md` | -If the task spans multiple categories, load only the relevant files above. +If a task spans multiple categories, load only the relevant references above. Keep guidance Pi-specific unless the user explicitly asks about a consuming product. ## Step 2: Apply integration guardrails -1. Treat `Agent` as the execution engine and keep wrapper abstractions thin. -2. Stream user-visible text only from `message_update` + `assistantMessageEvent.type === "text_delta"`. -3. Bridge deltas into `AsyncIterable` and pass that iterable to downstream streaming surfaces. -4. Preserve message boundaries when streaming multi-message assistant output (insert separators intentionally, then normalize). -5. Never call `prompt()` or `continue()` while the agent is running; use `steer()`/`followUp()` for mid-run input. -6. Keep `convertToLlm` and `transformContext` explicit, deterministic, and easy to test. -7. Keep tool calls/results as internal execution artifacts unless product UX explicitly requires otherwise. +1. Treat npm `latest` for `@earendil-works/pi-agent-core` as the source of truth before relying on a contract. +2. Use `Agent` when event handling must be awaited as part of run settlement; use low-level `agentLoop` only when an observational event stream is enough. +3. Stream user-visible text only from `message_update` where `assistantMessageEvent.type === "text_delta"`. +4. Preserve assistant message boundaries deliberately when forwarding multi-message output. +5. Do not call `prompt()` or `continue()` while an agent is active; queue mid-run input with `steer()` or `followUp()`. +6. Treat normal `continue()` as a resume from a non-empty `user` or `toolResult` tail. An `assistant` tail can only drain queued steering/follow-up messages, otherwise it throws. +7. Keep `streamFn`, `convertToLlm`, `transformContext`, `getApiKey`, queue providers, and loop hooks no-throw for expected request/runtime failures; return safe values or encode failures in protocol events. +8. Keep tool calls, tool progress, tool results, thinking deltas, and provider payloads internal unless the product UX explicitly exposes them. +9. Prefer Pi's built-in harness when sessions, skills, prompt templates, resources, filesystem/shell environment, compaction, or tree navigation are required. ## Step 3: Implement with minimal surface -1. Prefer constructor options over custom wrapper state machines (`streamFn`, `getApiKey`, `sessionId`, `thinkingBudgets`, `maxRetryDelayMs`). -2. Use `transformContext` for pruning/injection and `convertToLlm` for message-role conversion/filtering. -3. Keep queue mode explicit (`steeringMode`, `followUpMode`) when concurrency/order matters. -4. For server-proxied model access, use `streamFn` with `streamProxy`-style behavior instead of bespoke provider logic in consumers. -5. Keep failure behavior explicit: timeout/abort paths should set observable diagnostics and terminate streaming cleanly. +1. Prefer Pi options over custom wrapper state machines: `streamFn`, `getApiKey`, `sessionId`, `thinkingBudgets`, `transport`, `maxRetryDelayMs`, `onPayload`, `onResponse`, `beforeToolCall`, `afterToolCall`, `prepareNextTurn`, `toolExecution`, `steeringMode`, and `followUpMode`. +2. Mutate `Agent` state through `agent.state` properties and `reset()`; do not invent setter wrappers unless the consumer API needs them. +3. Use `transformContext` for message-level pruning/injection and `convertToLlm` for provider-compatible role conversion/filtering. +4. Keep queue modes explicit (`"one-at-a-time"` or `"all"`) when ordering or batching matters. +5. For server-proxied model access, use `streamFn` with `streamProxy`-style behavior instead of provider logic scattered through consumers. +6. For tool policy, use `toolExecution`, per-tool `executionMode`, `beforeToolCall`, `afterToolCall`, thrown tool errors, and `terminate` before adding a custom tool runner. +7. Keep timeout/abort paths observable and make sure streams/iterables settle cleanly. ## Step 4: Verify behavior -1. Verify event-to-stream bridge emits only text deltas and always closes the iterable. -2. Verify `prompt()`/`continue()` race handling (throws while streaming; queue path works via `steer`/`followUp`). -3. Verify `continue()` preconditions: non-empty context and valid last-message role semantics. -4. Verify custom message types survive agent state while `convertToLlm` emits only LLM-compatible roles. -5. Verify tool execution and turn lifecycle events remain internal unless explicitly exposed. -6. Verify newline joining/normalization parity between streamed and finalized outputs. +1. Verify the event-to-stream bridge emits only text deltas, preserves intended boundaries, and closes on success, error, and abort. +2. Verify `prompt()`/`continue()` race handling and queued `steer()`/`followUp()` behavior. +3. Verify `continue()` preconditions for empty history, `user` tail, `toolResult` tail, and `assistant` tail with and without queued messages. +4. Verify custom message types remain in agent state while `convertToLlm` emits only provider-compatible messages. +5. Verify `streamFn` encodes expected provider failures instead of throwing/rejecting. +6. Verify tool execution ordering under default parallel mode, sequential overrides, hook blocking/patching, progress updates, and `terminate` behavior. +7. Verify `Agent.subscribe()` listener settlement and `waitForIdle()` behavior when listeners perform async work. +8. Verify `AgentHarness` session, resource, hook, compaction, and abort behavior when the harness path is used. -## Step 5: Migration and version checks - -1. Check for queue API migrations (`queueMessage` -> `steer`/`followUp`) before editing old wrappers. -2. Check renamed hooks/options (`messageTransformer` -> `convertToLlm`, `preprocessor` -> `transformContext`). -3. Check default/available options in current package version before adding compatibility shims. -4. Favor hard cutovers unless backward compatibility is explicitly requested. +## Step 5: Version discipline +1. Target the latest published Pi package only. +2. Re-check the latest package metadata and declarations before material API updates. +3. Do not add backward-compatibility shims or old package-name guidance unless the user explicitly asks for a migration. diff --git a/.agents/skills/pi-agent-integration/SOURCES.md b/.agents/skills/pi-agent-integration/SOURCES.md index 8cedfa245..afb58d9eb 100644 --- a/.agents/skills/pi-agent-integration/SOURCES.md +++ b/.agents/skills/pi-agent-integration/SOURCES.md @@ -1,63 +1,81 @@ # Sources -Retrieved: 2026-03-05 +Retrieved: 2026-06-01 Skill class: `integration-documentation` -Selected profile: `references/examples/documentation-skill.md` +Primary execution shape: `reference-backed-expert` +Scope: Pi package documentation only; no consuming-product-specific contracts. ## Source inventory -| Source | Trust tier | Confidence | Contribution | Usage constraints | -| --- | --- | --- | --- | --- | -| `AGENTS.md` (junior) | canonical | high | Repository conventions and Pi streaming standard (`message_update`/`text_delta` -> `AsyncIterable`) | Repo-local guidance | -| `.agents/skills/skill-writer/SKILL.md` | canonical | high | Required workflow for synthesis/authoring/validation outputs | Skill-authoring process source | -| `.agents/skills/skill-writer/references/mode-selection.md` | canonical | high | Class selection and required outputs | Process guidance | -| `.agents/skills/skill-writer/references/synthesis-path.md` | canonical | high | Provenance, coverage matrix, depth gates | Process guidance | -| `.agents/skills/skill-writer/references/authoring-path.md` | canonical | high | Required artifact set for integration-documentation skills | Process guidance | -| `.agents/skills/skill-writer/references/description-optimization.md` | canonical | high | Trigger quality constraints | Process guidance | -| `.agents/skills/skill-writer/references/evaluation-path.md` | canonical | high | Lightweight evaluation rubric | Process guidance | -| `/packages/agent/README.md` | canonical | high | Public API intent, event flow, message pipeline semantics | External repo snapshot at retrieval date | -| `/packages/agent/src/types.ts` | canonical | high | Type-level contracts for `AgentLoopConfig`, events, and tools | Source of truth for interfaces | -| `/packages/agent/src/agent.ts` | canonical | high | Runtime semantics for `prompt`, `continue`, queueing, state transitions | Source of truth for behavior | -| `/packages/agent/src/agent-loop.ts` | canonical | high | Loop/event ordering and transform/convert call boundary | Source of truth for loop behavior | -| `/packages/agent/src/proxy.ts` | canonical | medium | Proxy streaming model and error path behavior | Focused on proxy mode only | -| `/packages/agent/CHANGELOG.md` | secondary | medium | Migration/renaming guidance and breaking changes | Historical summaries, validate against source | -| `/packages/agent/test/agent.test.ts` | canonical | high | Concurrency/queue/continue edge-case behavior | Test-backed behavioral assertions | -| `/packages/agent/test/agent-loop.test.ts` | canonical | high | transform/convert ordering, event semantics | Test-backed behavioral assertions | -| `specs/harness-agent-spec.md` | canonical | high | Consumer-side integration contract in junior runtime | Repo-local runtime spec | -| `packages/junior/src/chat/respond.ts` | canonical | high | Real-world Pi streaming bridge and timeout handling pattern | Consumer implementation snapshot | -| `packages/junior/src/chat/runtime/streaming.ts` | canonical | high | `AsyncIterable` bridge behavior | Consumer implementation snapshot | +| Source | Trust tier | Confidence | Contribution | Usage constraints | +| -------------------------------------------------------------------- | ---------- | ---------- | ------------------------------------------------------------------------------------------- | ----------------------------------------- | +| npm metadata for `@earendil-works/pi-agent-core` | canonical | high | Confirmed latest package name, latest version, repository, dist-tags | Re-check before future material API edits | +| `@earendil-works/pi-agent-core@0.78.0/package.json` | canonical | high | Runtime engine, exports, repository, dependency baseline | Published package snapshot | +| `@earendil-works/pi-agent-core@0.78.0/README.md` | canonical | high | Public API intent, event flow, tool execution, continuation, proxy, low-level loop guidance | Published package snapshot | +| `@earendil-works/pi-agent-core@0.78.0/dist/agent.d.ts` | canonical | high | `AgentOptions`, `Agent` methods, state, queue, lifecycle surface | Declaration source of truth | +| `@earendil-works/pi-agent-core@0.78.0/dist/agent.js` | canonical | high | Runtime semantics for `continue()`, queue draining, listener settlement, state updates | Used where README/types were ambiguous | +| `@earendil-works/pi-agent-core@0.78.0/dist/types.d.ts` | canonical | high | `StreamFn`, message pipeline, tool hooks, queue mode, tool execution, event types | Declaration source of truth | +| `@earendil-works/pi-agent-core@0.78.0/dist/agent-loop.d.ts` | canonical | high | Low-level loop signatures and continuation caveat | Declaration source of truth | +| `@earendil-works/pi-agent-core@0.78.0/dist/agent-loop.js` | canonical | high | Low-level loop ordering, `shouldStopAfterTurn`, `prepareNextTurn`, tool execution internals | Used where README/types were ambiguous | +| `@earendil-works/pi-agent-core@0.78.0/dist/proxy.d.ts` | canonical | high | `streamProxy` events and serializable proxy options | Declaration source of truth | +| `@earendil-works/pi-agent-core@0.78.0/dist/harness/*.d.ts` | canonical | high | `AgentHarness`, session, skill, prompt-template, compaction, environment contracts | Declaration source of truth | +| `.agents/skills/skill-writer/SKILL.md` | canonical | high | Required workflow for skill synthesis, authoring, and validation | Skill-authoring process source | +| `.agents/skills/skill-writer/references/mode-selection.md` | canonical | high | Classified this as `integration-documentation` | Process guidance | +| `.agents/skills/skill-writer/references/execution-shapes.md` | canonical | high | Selected `reference-backed-expert` shape | Process guidance | +| `.agents/skills/skill-writer/references/synthesis-path.md` | canonical | high | Required source inventory, decisions, coverage, gaps | Process guidance | +| `.agents/skills/skill-writer/references/authoring-path.md` | canonical | high | Runtime authoring and precision-pass rules | Process guidance | +| `.agents/skills/skill-writer/references/reference-architecture.md` | canonical | high | Added focused `references/harness.md` as a routed lookup leaf | Process guidance | +| `.agents/skills/skill-writer/references/spec-template.md` | canonical | high | Added `SPEC.md` for material scope/reference changes | Process guidance | +| `.agents/skills/skill-writer/references/description-optimization.md` | canonical | high | Trigger quality pass | Process guidance | +| `.agents/skills/skill-writer/references/registration-validation.md` | canonical | high | Validation expectations | Process guidance | ## Decisions -| Decision | Status | Evidence | -| --- | --- | --- | -| Classify skill as `integration-documentation` | adopted | `mode-selection.md` + user goal ("using Pi in another library") | -| Keep skill focused on consumer integration (not authoring internals) | adopted | User request + `harness-agent-spec.md` + `respond.ts` | -| Make event-stream bridge (`message_update`/`text_delta`) a primary guardrail | adopted | `AGENTS.md`, `README.md`, `respond.ts` | -| Require explicit queue/concurrency guidance (`steer`/`followUp`, `continue`) | adopted | `agent.ts`, tests, changelog | -| Include migration checks for renamed APIs | adopted | `CHANGELOG.md`, `agent.ts`, `types.ts` | -| Add proxy transport guidance as optional path | adopted | `proxy.ts` + constructor `streamFn` option | -| Add provider-specific model recommendations | rejected | Out of scope for abstraction-level integration skill | +| Decision | Status | Evidence | +| ---------------------------------------------------------------- | -------- | ---------------------------------------------------------------------------------- | +| Keep the skill Pi-only | adopted | User direction on 2026-06-01 | +| Target npm `latest` only | adopted | User direction + npm metadata | +| Use `@earendil-works/pi-agent-core` as the only package identity | adopted | Latest package metadata | +| Remove consuming-product-specific source references | adopted | User direction + portability goal | +| Add a routed harness reference | adopted | Latest package exports substantial `AgentHarness`, session, skill, compaction APIs | +| Keep `SKILL.md` as router/guardrail layer | adopted | `skill-writer` reference architecture | +| Add `SPEC.md` | adopted | Material scope and reference architecture change | +| Add backward compatibility or old package migration guidance | rejected | Latest-only user direction | ## Coverage matrix -| Dimension | Coverage status | Evidence | -| --- | --- | --- | -| API surface and behavior contracts | complete | `types.ts`, `agent.ts`, `agent-loop.ts`, `README.md` | -| Config/runtime options | complete | `agent.ts` options + `README.md` options sections | -| Common downstream use cases | complete | `respond.ts`, `streaming.ts`, `harness-agent-spec.md`, tests | -| Known issues/failure modes with workarounds | complete | `agent.test.ts`, `agent-loop.test.ts`, changelog fixes | -| Version/migration variance | complete | `CHANGELOG.md` breaking/renamed APIs | +| Dimension | Coverage status | Evidence | +| ------------------------------------------- | --------------- | --------------------------------------------------------------------------------------- | +| API surface and behavior contracts | covered | `agent.d.ts`, `agent.js`, `types.d.ts`, `agent-loop.d.ts`, `agent-loop.js`, `README.md` | +| Config/runtime options | covered | `AgentOptions`, `AgentLoopConfig`, `AgentHarnessOptions`, package metadata | +| Common downstream use cases | covered | `README.md`, declarations, runtime implementation | +| Known issues/failure modes with workarounds | covered | `agent.js`, `agent-loop.js`, type contracts | +| Version/migration variance | constrained | Latest-only package targeting; migration intentionally omitted | +| Harness/session/skill/compaction surface | covered | `dist/harness/*.d.ts` | + +## Trigger quality notes + +Should trigger: + +- "integrate pi-agent-core Agent into my app" +- "stream Pi Agent text deltas into our SDK" +- "how should I use AgentHarness sessions and skills" +- "fix continue() throwing in pi-agent-core" +- "wire streamProxy for Pi" + +Should not trigger: + +- "write a generic OpenAI API streaming adapter" +- "document a consuming app's chat runtime behavior" +- "create a new Codex skill unrelated to Pi" +- "debug a React component" +- "explain TypeBox generally" ## Open gaps -- Add integration examples for browser-only consumers that use `streamProxy` with non-fetch runtimes. -- Expand troubleshooting with provider-specific retry/backoff examples after confirming stable patterns in upstream docs. +- Re-run npm package retrieval before the next material update; the skill intentionally follows `latest`. +- Add concrete code examples only after collecting stable upstream examples or tests from the Pi repository. The current runtime guidance is source-backed but example-light. ## Stopping rationale -Additional retrieval is currently low-yield because: - -1. API contracts are already covered by source code and tests in `packages/agent`. -2. Consumer integration patterns are already represented by concrete junior runtime code (`respond.ts`, streaming bridge). -3. Remaining gaps are variant-specific extensions, not blockers for the core integration skill. +Further retrieval is low-yield for this pass because published package metadata, README, declarations, and implementation files cover the latest API contracts needed by this Pi-only integration skill. diff --git a/.agents/skills/pi-agent-integration/SPEC.md b/.agents/skills/pi-agent-integration/SPEC.md new file mode 100644 index 000000000..00a7affd4 --- /dev/null +++ b/.agents/skills/pi-agent-integration/SPEC.md @@ -0,0 +1,84 @@ +# Pi Agent Integration Specification + +## Intent + +This skill helps agents integrate the latest `@earendil-works/pi-agent-core` APIs into apps, libraries, runtimes, and harnesses without inventing avoidable wrapper behavior. + +It is an integration-documentation skill, not product documentation for any consuming app. + +## Scope + +In scope: + +- Pi `Agent` setup, streaming, queueing, continuation, abort, and state behavior. +- Pi low-level loop APIs. +- Pi `streamFn` and `streamProxy` integration. +- Pi tool execution, hooks, queue modes, and termination behavior. +- Pi `AgentHarness`, sessions, resources, skills, prompt templates, compaction, and environment interfaces. +- Troubleshooting based on published Pi package contracts. + +Out of scope: + +- Consuming-product-specific runtime policies, chat behavior, telemetry, or storage contracts. +- Legacy package migrations unless explicitly requested by the user. +- Provider-specific model recommendations outside the Pi API surface. + +## Users And Trigger Context + +- Primary users: agents implementing or reviewing Pi integrations. +- Common user requests: "integrate pi-agent-core", "wire Agent streaming", "debug continue()", "use AgentHarness", "fix Pi tool execution", "proxy Pi model calls". +- Should not trigger for generic LLM SDK usage, unrelated skill authoring, or product-specific behavior that does not mention or clearly depend on Pi. + +## Runtime Contract + +- Required first actions: classify the request and load only the routed reference files needed. +- Required outputs: implementation guidance, code edits, or review findings grounded in latest Pi package contracts. +- Non-negotiable constraints: keep guidance Pi-only, target npm `latest`, and do not add compatibility shims unless requested. +- Expected bundled files loaded at runtime: `SKILL.md` plus one or more direct `references/*.md` files. + +## Source And Evidence Model + +Authoritative sources: + +- npm metadata for `@earendil-works/pi-agent-core` +- latest published package README +- latest published package declarations +- latest published package implementation files when declarations are ambiguous + +Useful improvement sources: + +- upstream Pi repository tests and changelog, when available +- concrete failure reports from Pi integrations +- validation results from skill updates + +Data that must not be stored: + +- secrets +- customer data +- private application URLs or identifiers +- consuming-product-specific internal contracts unless the user explicitly asks for that scope + +## Reference Architecture + +- `SKILL.md` contains routing, guardrails, minimal implementation rules, verification, and version discipline. +- `references/` contains focused lookup leaves for API surface, common use cases, harness use, and troubleshooting. +- `SOURCES.md` contains source inventory, decisions, coverage, trigger quality notes, and gaps. +- `scripts/` and `assets/` are unused. + +## Validation + +- Lightweight validation: run the skill structural validator after artifact changes. +- Deeper validation: manually confirm every runtime reference is directly routed from `SKILL.md`, avoids host-specific paths, and changes an agent decision or verification step. +- Acceptance gates: package identity is current, no consuming-product-specific guidance remains, latest-only stance is explicit, and `continue()`/stream/tool/harness contracts match published Pi sources. + +## Known Limitations + +- The skill intentionally follows npm `latest`; it may need refresh when Pi publishes a new latest version. +- The skill is intentionally example-light until stable upstream examples or tests are captured as evidence. + +## Maintenance Notes + +- Update `SKILL.md` when trigger scope, routing, guardrails, or verification gates change. +- Update `references/*.md` when the Pi API behavior changes. +- Update `SOURCES.md` when source baselines, decisions, coverage, or gaps change. +- Update `SPEC.md` when scope, evidence policy, reference architecture, or validation expectations change. diff --git a/.agents/skills/pi-agent-integration/references/api-surface.md b/.agents/skills/pi-agent-integration/references/api-surface.md index 0c0cee299..7e9e2bc09 100644 --- a/.agents/skills/pi-agent-integration/references/api-surface.md +++ b/.agents/skills/pi-agent-integration/references/api-surface.md @@ -1,53 +1,102 @@ # API Surface -Primary package: `@mariozechner/pi-agent-core` - -## Core exports - -- `Agent` class (`src/agent.ts`) -- `agentLoop`, `agentLoopContinue` (`src/agent-loop.ts`) -- `streamProxy` (`src/proxy.ts`) -- Types from `src/types.ts`: `AgentMessage`, `AgentTool`, `AgentEvent`, `AgentState`, `AgentLoopConfig`, `StreamFn` - -## `Agent` constructor options - -- `initialState` (`systemPrompt`, `model`, `thinkingLevel`, `tools`, `messages`) -- `convertToLlm(messages)` for message conversion/filtering -- `transformContext(messages, signal)` for pruning/injection before conversion -- `steeringMode`, `followUpMode` (`"one-at-a-time"` or `"all"`) -- `streamFn` for custom/proxied streaming -- `sessionId`, `getApiKey`, `thinkingBudgets`, `transport`, `maxRetryDelayMs` - -## Core runtime methods - -- Prompting: `prompt(string | AgentMessage | AgentMessage[])`, `continue()` -- Queueing: `steer(message)`, `followUp(message)`, plus clear/dequeue helpers -- State mutation: `setSystemPrompt`, `setModel`, `setThinkingLevel`, `setTools`, `replaceMessages`, `appendMessage`, `clearMessages`, `reset` -- Lifecycle: `abort()`, `waitForIdle()`, `subscribe(listener)` - -## Event contract - -- Lifecycle events: `agent_start`, `turn_start`, `turn_end`, `agent_end` -- Message events: `message_start`, `message_update`, `message_end` -- Tool events: `tool_execution_start`, `tool_execution_update`, `tool_execution_end` -- Streaming text should be read from `message_update` where `assistantMessageEvent.type === "text_delta"` - -## Message pipeline contract - -`AgentMessage[]` -> `transformContext()` -> `convertToLlm()` -> LLM `Message[]` - -- `transformContext`: keep message-level behavior (pruning, external context injection) -- `convertToLlm`: convert/filter to provider-compatible `user`/`assistant`/`toolResult` messages - -## Continue/queue semantics - -- `prompt()` and `continue()` throw if `isStreaming` is true. -- `continue()` requires message history and valid tail state. -- If tail is `assistant`, `continue()` can resume queued `steer`/`followUp`; otherwise it throws. -- Mid-run user input should be queued with `steer` or `followUp`, not re-entered with `prompt`. - -## Version and migration points to check - -- Queue API migration: `queueMessage` replaced by `steer`/`followUp` -- Option migration: `messageTransformer` -> `convertToLlm`, `preprocessor` -> `transformContext` -- Transport abstraction changes: prefer `streamFn` customization for proxy/server routing +Open this when wiring or updating Pi `Agent`, low-level loop, provider, stream, or tool APIs. + +Primary package: `@earendil-works/pi-agent-core` +Current source baseline: npm `latest` at last synthesis was `0.78.0`. + +## Package facts + +| Area | Latest contract | +| ------------ | --------------------------------------------------------------------- | +| Package | `@earendil-works/pi-agent-core` | +| Runtime | Node `>=22.19.0` | +| Repository | `github.com/earendil-works/pi`, `packages/agent` | +| Main imports | `@earendil-works/pi-agent-core`, `@earendil-works/pi-agent-core/node` | +| Model layer | Built on `@earendil-works/pi-ai` | + +## Top-level exports + +- Core execution: `Agent`, `agentLoop`, `agentLoopContinue`, `runAgentLoop`, `runAgentLoopContinue`. +- Provider proxy: `streamProxy`, `ProxyAssistantMessageEvent`, `ProxyStreamOptions`. +- Core types: `AgentMessage`, `AgentTool`, `AgentToolResult`, `AgentEvent`, `AgentState`, `AgentContext`, `AgentLoopConfig`, `StreamFn`, `QueueMode`, `ToolExecutionMode`, `ThinkingLevel`. +- Harness layer: `AgentHarness`, session repositories, skill loading helpers, prompt-template helpers, compaction helpers, message helpers, and harness types. +- Node entry: `NodeExecutionEnv` from `@earendil-works/pi-agent-core/node`. + +## `Agent` options + +| Option | Use | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `initialState` | Seed `systemPrompt`, `model`, `thinkingLevel`, `tools`, and `messages`. | +| `convertToLlm(messages)` | Convert/filter `AgentMessage[]` to provider-compatible LLM messages. Must not throw/reject for expected cases. | +| `transformContext(messages, signal)` | Prune or inject context before conversion. Must not throw/reject for expected cases. | +| `streamFn` | Replace provider streaming, usually for proxying or tracing. Must return a stream and encode expected failures in stream events/results. | +| `getApiKey(provider)` | Resolve short-lived provider credentials per LLM call. Return `undefined` rather than throwing for missing expected auth. | +| `onPayload`, `onResponse` | Observe or patch provider payload/response through `pi-ai` stream options. | +| `beforeToolCall`, `afterToolCall` | Block, inspect, patch, or terminate tool results at Pi's tool boundary. Honor `AbortSignal`. | +| `prepareNextTurn` | Replace context/model/thinking state before another provider request in the current run. | +| `steeringMode`, `followUpMode` | Drain queued messages as `"one-at-a-time"` or `"all"`. Defaults are one-at-a-time. | +| `sessionId` | Forward provider cache/session identity. | +| `thinkingBudgets` | Override per-thinking-level token budgets. | +| `transport` | Select preferred provider transport. | +| `maxRetryDelayMs` | Bound provider-requested retry delays. | +| `toolExecution` | Execute tool batches as `"parallel"` by default or `"sequential"`. | + +## `Agent` runtime surface + +- Prompting: `prompt(string, images?)`, `prompt(AgentMessage)`, `prompt(AgentMessage[])`. +- Continuation: `continue()`. +- Queueing: `steer(message)`, `followUp(message)`, `clearSteeringQueue()`, `clearFollowUpQueue()`, `clearAllQueues()`, `hasQueuedMessages()`. +- Lifecycle: `abort()`, `waitForIdle()`, `subscribe(listener)`, `signal`. +- State: mutate `agent.state.systemPrompt`, `agent.state.model`, `agent.state.thinkingLevel`, `agent.state.tools`, and `agent.state.messages`; use `reset()` to clear transcript/runtime/queues. +- Runtime state: `agent.state.isStreaming`, `agent.state.streamingMessage`, `agent.state.pendingToolCalls`, `agent.state.errorMessage`. + +## Events and streaming + +- Lifecycle events: `agent_start`, `turn_start`, `turn_end`, `agent_end`. +- Message events: `message_start`, `message_update`, `message_end`. +- Tool events: `tool_execution_start`, `tool_execution_update`, `tool_execution_end`. +- `message_update` is assistant-only but includes text, thinking, and tool-call deltas. Forward user-visible text only when `assistantMessageEvent.type === "text_delta"`. +- `Agent.subscribe()` awaits listener promises in registration order. `agent_end` is the final event, but `prompt()`, `continue()`, and `waitForIdle()` settle only after awaited `agent_end` listeners finish. + +## Message pipeline + +`AgentMessage[]` -> `transformContext()` -> `AgentMessage[]` -> `convertToLlm()` -> LLM `Message[]`. + +- Keep app-specific/custom messages in agent state when useful. +- Filter or map custom messages in `convertToLlm`. +- For low-level continuation, the last message must convert to `user` or `toolResult`; Pi can only check raw assistant tails before conversion. + +## Continue and queue semantics + +- `prompt()` throws while a run is active. +- `continue()` throws while a run is active. +- `continue()` throws on empty history. +- `continue()` normally resumes from a `user` or `toolResult` tail. +- If the tail is `assistant`, `continue()` drains queued steering first, then queued follow-ups; if neither exists, it throws. +- `steer()` injects queued messages after the current assistant turn and tool batch finish. +- `followUp()` runs only after the agent would otherwise stop. + +## Tool execution + +- Default batch mode is `parallel`: tool preflight is sequential, allowed tools execute concurrently, `tool_execution_end` follows completion order, and tool-result messages/`turn_end.toolResults` follow assistant source order. +- Global `toolExecution: "sequential"` executes one call at a time. +- A per-tool `executionMode: "sequential"` forces the whole batch sequential. +- `beforeToolCall` runs after tool-start emission and argument validation; returning `{ block: true }` produces an error tool result. +- `afterToolCall` can replace `content`, `details`, `isError`, or `terminate`; `content` and `details` are full replacements, not deep merges. +- Tool `execute()` should throw on failure rather than returning failure text as successful content. +- `terminate: true` skips the automatic follow-up LLM call only when every finalized result in the batch terminates. + +## Low-level loop API + +- Use `agentLoop(prompts, context, config, signal?, streamFn?)` to start with prompt messages. +- Use `agentLoopContinue(context, config, signal?, streamFn?)` to continue existing context. +- Raw loop streams preserve event order but do not wait for async event handling to settle before producer phases continue. +- Use `Agent` instead when event processing must be a barrier before tool preflight or run settlement. +- `AgentLoopConfig` adds low-level-only hooks such as `shouldStopAfterTurn`, `getSteeringMessages`, and `getFollowUpMessages`. + +## Proxy and stream functions + +- `streamFn` has the same shape as `pi-ai` `streamSimple`. +- Expected provider/request/runtime failures must be encoded in the returned stream with protocol events and a final assistant message with `stopReason: "error"` or `"aborted"`. +- `streamProxy(model, context, options)` proxies through a server with `authToken`, `proxyUrl`, local `signal`, and serializable stream options. diff --git a/.agents/skills/pi-agent-integration/references/common-use-cases.md b/.agents/skills/pi-agent-integration/references/common-use-cases.md index a883cf8e4..59f3f8b4a 100644 --- a/.agents/skills/pi-agent-integration/references/common-use-cases.md +++ b/.agents/skills/pi-agent-integration/references/common-use-cases.md @@ -1,33 +1,77 @@ # Common Use Cases -Use these patterns when Pi `Agent` is consumed by another library/runtime. +Open this when adding Pi behavior in a consuming app, library, runtime, or adapter. -1. Stream assistant text into another SDK surface: -Use `agent.subscribe` and forward only `message_update` + `text_delta` into an `AsyncIterable` bridge. +## Stream assistant text into another surface -2. Preserve streamed-vs-final output parity: -Insert separators between assistant message boundaries during delta streaming so final joined text matches non-streamed output semantics. +Use `agent.subscribe()` and forward only: -3. Add custom app messages without leaking them to LLM calls: -Keep custom message types in agent state; filter/convert them in `convertToLlm`. +```ts +event.type === "message_update" && + event.assistantMessageEvent.type === "text_delta"; +``` -4. Prune or augment context safely: -Use `transformContext` for context window control and deterministic context injection before `convertToLlm`. +Bridge those deltas into the consumer's streaming abstraction. Insert separators only at intentional assistant message boundaries and apply the same normalization to streamed and finalized output. -5. Support user steering while tools are running: -Use `steer` for interruptions and `followUp` for deferred prompts instead of issuing parallel `prompt()` calls. +## Proxy provider access -6. Implement timeout-controlled turns: -Race prompt execution against a timeout, call `agent.abort()` on timeout, and surface explicit timeout diagnostics. +Use `streamFn` when model calls must route through a backend, tracing layer, gateway, or policy boundary. -7. Resume across session slices/checkpoints: -Restore message history (for example via `replaceMessages`) and call `continue()` with valid tail-state semantics. +- Preserve the `StreamFn` contract: return a stream; do not throw/reject for expected provider failures. +- Use `streamProxy` when a browser or untrusted client needs server-owned auth. +- Use `onPayload` and `onResponse` when the consumer needs provider payload/response observation without replacing the stream function. -8. Route through backend-proxied model access: -Provide a custom `streamFn` (or `streamProxy`) so auth/provider calls stay server-side while preserving local `Agent` event semantics. +## Resolve short-lived credentials -9. Handle expiring provider tokens: -Use `getApiKey` dynamic resolution for each LLM call instead of static long-lived API keys. +Use `getApiKey(provider)` for per-call provider credentials. Return `undefined` for expected unauthenticated states and let the consumer own visible auth recovery. -10. Tune transport/retry constraints: -Set `transport` and `maxRetryDelayMs` intentionally for consumer runtime behavior and bounded latency. +## Add custom app messages + +Extend `CustomAgentMessages` and keep custom entries in `agent.state.messages` when they matter to UI/session state. Use `convertToLlm` to filter UI-only messages or map custom messages to provider-compatible `user`, `assistant`, or `toolResult` messages. + +## Prune or augment context + +Use `transformContext(messages, signal)` for message-level pruning, compaction insertion, external context injection, and other operations that should happen before provider conversion. + +Keep `transformContext` deterministic and no-throw for expected cases. Return the original messages or a conservative safe subset when pruning cannot run. + +## Support steering and follow-ups + +Use `steer()` for user input that should influence the next model call after the current assistant turn and tool batch finish. Use `followUp()` for input that should wait until the agent would otherwise stop. + +Set `steeringMode` and `followUpMode` explicitly when queued-message batching affects UX or correctness. + +## Retry or resume generation + +Use `continue()` only when the agent is idle and has a valid transcript. + +- `user` or `toolResult` tail: normal continuation. +- `assistant` tail with queued steering/follow-up: drains queued messages as a new prompt path. +- `assistant` tail without queued messages: throws. + +For provider retry, trim only retryable trailing assistant error messages and continue from a safe `user` or `toolResult` boundary. + +## Bound and abort runs + +Race the prompt/continue promise against the consumer timeout. On timeout, call `agent.abort()`, wait for run settlement when possible, and close downstream streams/iterables in `finally`. + +## Execute tools through Pi + +Prefer Pi's tool execution surface over a custom runner. + +- Use `toolExecution` for global parallel/sequential policy. +- Use per-tool `executionMode` for tools that cannot run in a concurrent batch. +- Use `beforeToolCall` to block or authorize a call after validation. +- Use `afterToolCall` to patch content/details/error/termination at the final tool boundary. +- Throw from `execute()` on tool failure; Pi will create an error tool result for the model. +- Use `onUpdate` for progress, not user-visible final replies. + +## Stop gracefully between turns + +Use low-level `shouldStopAfterTurn` when the consumer owns the loop and needs to stop after a completed assistant turn before queues are polled. + +Use `prepareNextTurn` when the next provider request needs a replacement context, model, or thinking level. + +## Choose `AgentHarness` + +Use Pi's `AgentHarness` instead of a custom wrapper when the consumer needs a session tree, skill loading/invocation, prompt templates, resources, filesystem/shell environment, compaction, branch navigation, provider request hooks, or high-level queue UX. Read `references/harness.md`. diff --git a/.agents/skills/pi-agent-integration/references/harness.md b/.agents/skills/pi-agent-integration/references/harness.md new file mode 100644 index 000000000..58caa4036 --- /dev/null +++ b/.agents/skills/pi-agent-integration/references/harness.md @@ -0,0 +1,96 @@ +# AgentHarness + +Open this when using Pi's built-in harness, sessions, skills, prompt templates, resources, environment, compaction, or tree navigation. + +## When to use it + +Use `AgentHarness` when the consumer needs more than a single `Agent` transcript: + +- durable session tree and leaf navigation +- skills or prompt-template invocation +- app-owned resources included in system prompts +- active tool subsets +- filesystem and shell environment abstraction +- compaction or branch summary operations +- provider auth/header hooks +- high-level queued UX with `steer`, `followUp`, and `nextTurn` + +Use bare `Agent` when the consumer already owns these concerns and only needs Pi execution/events/tools. + +## Constructor inputs + +| Option | Purpose | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------- | +| `env` | `ExecutionEnv` filesystem/shell capability. Operations return `Result` values rather than throwing for expected failures. | +| `session` | Session tree storage for messages, settings, compactions, labels, and leaf state. | +| `tools` | Available `AgentTool` definitions. | +| `resources` | App-owned skills and prompt templates. | +| `systemPrompt` | Static prompt or async function using env/session/model/thinking/tools/resources. | +| `getApiKeyAndHeaders(model)` | Per-request provider auth and headers. | +| `streamOptions` | Curated provider options: transport, timeout, retries, headers, metadata, cache retention. | +| `model`, `thinkingLevel` | Active model and reasoning level. | +| `activeToolNames` | Optional initial active tool subset. | +| `steeringMode`, `followUpMode` | Queue draining policy. | + +## Main methods + +- Run work: `prompt(text, { images })`, `skill(name, additionalInstructions)`, `promptFromTemplate(name, args)`. +- Queue work: `steer(text)`, `followUp(text)`, `nextTurn(text)`. +- Mutate session: `appendMessage(message)`, `compact(customInstructions)`, `navigateTree(targetId, options)`. +- Update runtime settings: `setModel`, `setThinkingLevel`, `setTools`, `setActiveTools`, `setResources`, `setStreamOptions`, `setSteeringMode`, `setFollowUpMode`. +- Inspect runtime settings: `getModel`, `getThinkingLevel`, `getTools`, `getActiveTools`, `getResources`, `getStreamOptions`, `getSteeringMode`, `getFollowUpMode`. +- Lifecycle: `abort()`, `waitForIdle()`, `subscribe(listener)`, `on(type, handler)`. + +## Harness events and hooks + +Harness subscribers receive both core `AgentEvent` values and harness-owned events. + +Use `on(type, handler)` for hook-style events that can return patches: + +- `before_agent_start`: replace initial messages or system prompt. +- `context`: replace context messages before provider conversion. +- `before_provider_request`: patch stream options. +- `before_provider_payload`: replace provider payload. +- `tool_call`: block tool execution with a reason. +- `tool_result`: patch content, details, error state, or termination. +- `session_before_compact`: cancel or provide compaction output. +- `session_before_tree`: cancel or provide branch summary/tree options. + +Observe these events for state and diagnostics: + +- `after_provider_response` +- `session_compact` +- `session_tree` +- `model_update` +- `thinking_level_update` +- `resources_update` +- `tools_update` +- `queue_update` +- `save_point` +- `abort` +- `settled` + +## Queue guidance + +- `steer()` targets the active run's next opportunity after the current assistant turn/tool batch. +- `followUp()` runs after the agent would otherwise stop. +- `nextTurn()` queues text for the next explicit turn and should be kept distinct from mid-run steering. +- `queue_update` exposes current queued `steer`, `followUp`, and `nextTurn` messages. + +## Session and compaction guidance + +- Keep app-specific data in session entries or resources when it should survive turns. +- Use `compact()` for harness-managed history reduction. +- Use `navigateTree()` when moving the active leaf or summarizing a branch. +- Treat compaction/tree hooks as policy boundaries; return structured results instead of mutating storage behind the harness. + +## Verification + +Verify: + +1. Hook return patches are applied at the documented boundary. +2. Session writes flush before relying on persisted state. +3. Active tool names match the available tool set. +4. `abort()` clears queued steer/follow-up messages and emits `abort`. +5. `waitForIdle()` waits for the active run and awaited listeners. +6. Compaction and tree navigation preserve expected leaf/session state. diff --git a/.agents/skills/pi-agent-integration/references/troubleshooting-workarounds.md b/.agents/skills/pi-agent-integration/references/troubleshooting-workarounds.md index aad48a693..04164dff7 100644 --- a/.agents/skills/pi-agent-integration/references/troubleshooting-workarounds.md +++ b/.agents/skills/pi-agent-integration/references/troubleshooting-workarounds.md @@ -1,50 +1,40 @@ # Troubleshooting and Workarounds -Use this table when Pi-agent integration behavior is wrong in a consumer library. - -| Symptom | Likely cause | Fix | -| --- | --- | --- | -| `prompt()` throws "Agent is already processing a prompt..." | Concurrent prompt while `isStreaming` is true | Queue input with `steer`/`followUp` or await existing run completion | -| `continue()` throws while agent is active | `continue()` called during streaming | Wait for idle, then call `continue()` | -| `continue()` throws "No messages to continue from" | Empty message history | Seed context with user/toolResult history before `continue()` | -| `continue()` throws from assistant-tail state | No queued steering/follow-up messages when tail is assistant | Queue `steer`/`followUp` first, or call `prompt()` with new user message | -| Stream shows no text even though turn finishes | Listener filtering wrong event type | Consume `message_update` events with `assistantMessageEvent.type === "text_delta"` | -| Streamed text and final text differ in formatting | Missing boundaries between assistant message segments | Insert explicit separators between message boundaries and normalize downstream | -| Tool call artifacts leak into user-visible output | Consumer is rendering tool calls/tool results directly | Keep tool lifecycle artifacts internal and render only resolved assistant text | -| Custom message roles break provider calls | `convertToLlm` passes non-LLM-compatible roles | Filter/transform to provider-compatible message roles in `convertToLlm` | -| Context pruning removes critical state unexpectedly | `transformContext` is non-deterministic or too aggressive | Make pruning deterministic and test with before/after context assertions | -| Queue order surprises in multi-message steering | Queue mode defaults not explicit | Set `steeringMode`/`followUpMode` intentionally (`one-at-a-time` vs `all`) | -| Timeouts do not cleanly stop UI stream | Timeout path does not call `abort()` and close stream bridge | Abort agent on timeout, always end iterable in `finally` | -| Proxy streaming errors are opaque | Proxy response/event parsing not surfaced | Validate proxy response status/body and emit explicit error diagnostics | - -## Issue/fix checklist - -1. Concurrent prompt failure: -Use `steer`/`followUp` during active runs; do not call `prompt` again until idle. - -2. Continue during stream: -Gate `continue()` with `isStreaming`/`waitForIdle()` checks. - -3. Empty continue context: -Load prior `AgentMessage[]` before `continue()` calls. - -4. Assistant-tail continue rejection: -Queue a steering or follow-up message first, or start a fresh prompt. - -5. Missing text deltas: -Filter to `message_update` + `text_delta`; ignore other delta types for user text stream. - -6. Stream/final mismatch: -Insert message-boundary separators and apply identical normalization in streamed and final output paths. - -7. Invalid custom roles at provider boundary: -Map custom messages in `convertToLlm`; keep only provider-compatible roles. - -8. Over-pruned context: -Make `transformContext` deterministic and verify retained messages in tests. - -9. Queue-order surprises: -Set `steeringMode` and `followUpMode` explicitly in wrappers. - -10. Timeout leak: -Always abort and close iterable in `finally` blocks. +Open this when Pi-agent integration behavior is wrong in a consumer. + +| Symptom | Likely cause | Fix | +| ------------------------------------------------------------------ | ----------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | +| `prompt()` throws "Agent is already processing a prompt..." | A run is active | Queue input with `steer()`/`followUp()` or await `waitForIdle()` | +| `continue()` throws "Agent is already processing..." | Continuation called during an active run | Await the current prompt/continue or queue input | +| `continue()` throws "No messages to continue from" | Empty transcript | Restore or append prior `AgentMessage[]` first | +| `continue()` throws "Cannot continue from message role: assistant" | Assistant tail with no queued steering/follow-up | Trim to a safe `user`/`toolResult` boundary, queue a message, or start a new prompt | +| Low-level continuation reaches a provider role error | Last custom message converts to `assistant` or an invalid provider role | Ensure `convertToLlm` leaves a final `user` or `toolResult` message | +| Stream shows no user-visible text | Listener is forwarding the wrong event/delta | Forward only `message_update` + `assistantMessageEvent.type === "text_delta"` | +| Thinking or tool-call text leaks to users | Consumer renders all assistant deltas | Ignore `thinking_*` and `toolcall_*` deltas unless a deliberate UX exposes them | +| Streamed and final text differ | Missing or inconsistent assistant-message boundaries | Insert separators intentionally and normalize streamed/final output the same way | +| Run settles late after `agent_end` | Async subscribers are still running | Treat `agent_end` as final event emission, not full idle; await `waitForIdle()` | +| Tool preflight sees stale state with low-level loop | Raw `agentLoop` event handling is observational | Use `Agent` when message event handling must be a barrier before tool preflight | +| Provider failures bypass normal event flow | `streamFn` throws/rejects for expected failures | Return a stream that encodes `error`/`aborted` and a final assistant message | +| Transform/conversion breaks lifecycle | `transformContext` or `convertToLlm` throws/rejects | Return original messages, filtered messages, or another safe fallback for expected cases | +| Missing auth crashes the loop | `getApiKey` throws for expected unauthenticated state | Return `undefined` and let the consumer own visible auth recovery | +| Tool results appear out of expected order | Default tool mode is parallel | Account for completion-order `tool_execution_end`; use `sequential` when required | +| A sequential-only tool still changes whole batch behavior | Per-tool `executionMode: "sequential"` forces the whole batch sequential | Isolate the tool call or accept sequential batch execution | +| Tool failure is treated as success | Tool returned failure text as normal content | Throw from `execute()` so Pi emits an error tool result | +| `terminate: true` does not stop the next LLM call | Mixed batch where not every finalized tool result terminates | Ensure every result in the batch sets `terminate: true`, or split the batch | +| Queue order surprises | Default queue mode drains one message at a time | Set `steeringMode`/`followUpMode` explicitly | +| Proxy errors are opaque | Proxy response/event handling hides status/body | Validate proxy status/body in the proxy server and encode visible stream errors | +| Harness hook changes are ignored | Handler is attached to the wrong event type or uses `subscribe()` instead of `on()` | Use `harness.on(type, handler)` for patch-returning hooks | +| Harness session state is missing | Work relies on pending writes before they flush | Wait for the harness method/idle boundary before reading persisted session state | + +## Debugging checklist + +1. Confirm the package name is `@earendil-works/pi-agent-core` and the API was checked against npm `latest`. +2. Identify whether the consumer uses `Agent`, low-level loop APIs, or `AgentHarness`. +3. Check active-run state before any `prompt()` or `continue()` call. +4. Inspect the transcript tail before continuation. +5. Check whether queued steering/follow-up messages exist when continuing from an assistant tail. +6. Confirm stream forwarding filters to text deltas only. +7. Confirm expected provider failures are encoded in the stream rather than thrown. +8. Confirm transform/conversion/auth hooks return safe values for expected failures. +9. Confirm tool execution mode and per-tool execution overrides. +10. Confirm async listeners or harness hooks are not delaying settlement unexpectedly. diff --git a/AGENTS.md b/AGENTS.md index 6404b2af7..da6865086 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,6 +110,7 @@ Co-Authored-By: (agent model name) - `specs/security-policy.md` (global runtime/container/token security policy) - `specs/data-redaction-policy.md` (conversation privacy classification and raw payload redaction policy) - `specs/chat-architecture.md` (chat composition, service, and test-seam architecture contract) +- `specs/task-execution.md` (durable conversation mailbox, queue wake-up, lease, and heartbeat execution contract) - `specs/agent-turn-handling.md` (agent user-message response policy: reply/silence, tool use, Slack side effects, resumed turns, and completion) - `specs/slack-agent-delivery.md` (Slack entry surfaces, reply delivery, continuation, files, images, and resume behavior contract) - `specs/slack-outbound-contract.md` (Slack outbound boundary, message/file/reaction safety rules, and markdown-to-`mrkdwn` ownership) diff --git a/packages/junior/package.json b/packages/junior/package.json index f51684b50..2fa627afe 100644 --- a/packages/junior/package.json +++ b/packages/junior/package.json @@ -69,6 +69,7 @@ "@slack/web-api": "^7.16.0", "@sentry/node": "catalog:", "@vercel/functions": "^3.6.0", + "@vercel/queue": "^0.2.0", "@vercel/sandbox": "2.0.0", "ai": "^6.0.190", "bash-tool": "^1.3.16", diff --git a/packages/junior/src/app.ts b/packages/junior/src/app.ts index a626b2786..562b9a9ac 100644 --- a/packages/junior/src/app.ts +++ b/packages/junior/src/app.ts @@ -36,6 +36,11 @@ import { } from "@/handlers/sandbox-egress-proxy"; import { POST as turnResumePOST } from "@/handlers/turn-resume"; import { POST as webhooksPOST } from "@/handlers/webhooks"; +import { + createVercelConversationWorkCallback, + type VercelConversationWorkCallbackOptions, +} from "@/chat/task-execution/vercel-callback"; +import { getProductionConversationWorkOptions } from "@/chat/app/production"; import type { WaitUntilFn } from "@/handlers/types"; export { defineJuniorPlugins } from "@/plugins"; @@ -48,6 +53,8 @@ export type { export interface JuniorAppOptions { /** Install-wide provider defaults (`provider.key` format). Channel overrides take precedence. */ configDefaults?: Record; + /** Queue consumer wiring for the durable conversation worker. */ + conversationWork?: VercelConversationWorkCallbackOptions; /** Direct plugin set override. Usually omitted when `juniorNitro()` uses a plugin module. */ plugins?: JuniorPluginSet; waitUntil?: WaitUntilFn; @@ -325,6 +332,16 @@ export async function createApp(options?: JuniorAppOptions): Promise { return agentDispatchPOST(c.req.raw, waitUntil); }); + let agentContinuePOST: + | ReturnType + | undefined; + app.post("/api/internal/agent/continue", (c) => { + agentContinuePOST ??= createVercelConversationWorkCallback( + options?.conversationWork ?? getProductionConversationWorkOptions(), + ); + return agentContinuePOST(c.req.raw); + }); + app.get("/api/internal/heartbeat", (c) => { return heartbeatGET(c.req.raw, waitUntil); }); diff --git a/packages/junior/src/chat/agent-dispatch/heartbeat.ts b/packages/junior/src/chat/agent-dispatch/heartbeat.ts index 303e5895e..bbd919ecc 100644 --- a/packages/junior/src/chat/agent-dispatch/heartbeat.ts +++ b/packages/junior/src/chat/agent-dispatch/heartbeat.ts @@ -1,5 +1,13 @@ import { getAgentPlugins } from "@/chat/plugins/agent-hooks"; import { logException, logInfo } from "@/chat/logging"; +import { + getAwaitingTurnContinuationRequest, + scheduleTurnTimeoutResume, +} from "@/chat/services/timeout-resume"; +import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; +import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; +import { getVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; +import { listAgentTurnSessionSummaries } from "@/chat/state/turn-session"; import { createHeartbeatContext } from "./context"; import { scheduleDispatchCallback } from "./signing"; import { @@ -16,6 +24,8 @@ const DEFAULT_RECOVERY_LIMIT = 25; const DEFAULT_PLUGIN_LIMIT = 25; const DISPATCH_MAX_AGE_MS = 24 * 60 * 60 * 1000; const PLUGIN_HEARTBEAT_TIMEOUT_MS = 25_000; +const TIMEOUT_RESUME_STALE_MS = 2 * 60 * 1000; +const TIMEOUT_RESUME_RECOVERY_SCAN_LIMIT = 500; function isStaleDispatch(args: { nowMs: number; @@ -87,6 +97,68 @@ async function runWithTimeout( } } +/** Re-drive stale turn timeout continuations whose internal callback vanished. */ +export async function recoverStaleTimeoutResumes(args: { + conversationWorkQueue?: ConversationWorkQueue; + limit?: number; + nowMs: number; +}): Promise { + const summaries = await listAgentTurnSessionSummaries( + TIMEOUT_RESUME_RECOVERY_SCAN_LIMIT, + ); + let recovered = 0; + for (const summary of summaries) { + if (recovered >= (args.limit ?? DEFAULT_RECOVERY_LIMIT)) { + break; + } + if ( + summary.state !== "awaiting_resume" || + summary.resumeReason !== "timeout" || + summary.updatedAtMs + TIMEOUT_RESUME_STALE_MS > args.nowMs + ) { + continue; + } + + try { + const request = await getAwaitingTurnContinuationRequest({ + conversationId: summary.conversationId, + sessionId: summary.sessionId, + }); + if (!request) { + continue; + } + await scheduleTurnTimeoutResume(request, { + queue: args.conversationWorkQueue, + }); + recovered += 1; + logInfo( + "agent_turn_timeout_resume_recovery_scheduled", + {}, + { + "app.ai.conversation_id": summary.conversationId, + "app.ai.session_id": summary.sessionId, + "app.ai.resume_session_version": request.expectedVersion, + "app.ai.resume_slice_id": summary.sliceId, + }, + "Heartbeat rescheduled stale timeout resume", + ); + } catch (error) { + logException( + error, + "agent_turn_timeout_resume_recovery_failed", + {}, + { + "app.ai.conversation_id": summary.conversationId, + "app.ai.session_id": summary.sessionId, + }, + "Heartbeat timeout resume recovery failed", + ); + } + } + + return recovered; +} + /** Re-drive stale core dispatches before invoking plugin heartbeat hooks. */ export async function recoverStaleDispatches(args: { limit?: number; @@ -193,7 +265,18 @@ export async function runTrustedPluginHeartbeats(args: { } /** Run the core heartbeat phases. */ -export async function runHeartbeat(args: { nowMs: number }): Promise { +export async function runHeartbeat(args: { + conversationWorkQueue?: ConversationWorkQueue; + nowMs: number; +}): Promise { + await recoverConversationWork({ + nowMs: args.nowMs, + queue: args.conversationWorkQueue ?? getVercelConversationWorkQueue(), + }); + await recoverStaleTimeoutResumes({ + conversationWorkQueue: args.conversationWorkQueue, + nowMs: args.nowMs, + }); await recoverStaleDispatches({ nowMs: args.nowMs }); await runTrustedPluginHeartbeats({ nowMs: args.nowMs }); } diff --git a/packages/junior/src/chat/agent-dispatch/runner.ts b/packages/junior/src/chat/agent-dispatch/runner.ts index 1d9770604..4ad1d44b0 100644 --- a/packages/junior/src/chat/agent-dispatch/runner.ts +++ b/packages/junior/src/chat/agent-dispatch/runner.ts @@ -42,7 +42,6 @@ import { buildSlackReplyFooter } from "@/chat/slack/footer"; import { finalizeFailedTurnReply } from "@/chat/services/turn-failure-response"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; import { PluginCredentialFailureError } from "@/chat/services/plugin-auth-orchestration"; -import { canScheduleTurnTimeoutResume } from "@/chat/services/timeout-resume"; import { isRetryableTurnError } from "@/chat/runtime/turn"; import { scheduleDispatchCallback } from "./signing"; import { @@ -426,11 +425,7 @@ export async function runAgentDispatchSlice( } if (isRetryableTurnError(error, "turn_timeout_resume")) { const version = error.metadata?.version; - const nextSliceId = error.metadata?.sliceId; - if ( - typeof version === "number" && - canScheduleTurnTimeoutResume(nextSliceId) - ) { + if (typeof version === "number") { const awaiting = await markDispatch({ dispatch, status: "awaiting_resume", diff --git a/packages/junior/src/chat/app/production.ts b/packages/junior/src/chat/app/production.ts index 02e0e85f4..4d5317541 100644 --- a/packages/junior/src/chat/app/production.ts +++ b/packages/junior/src/chat/app/production.ts @@ -1,200 +1,73 @@ import type { SlackAdapter } from "@chat-adapter/slack"; import { createSlackRuntime } from "@/chat/app/factory"; -import { createUserTokenStore } from "@/chat/capabilities/factory"; import { - botConfig, getSlackBotToken, getSlackClientId, getSlackClientSecret, getSlackSigningSecret, } from "@/chat/config"; -import { unlinkProvider } from "@/chat/credentials/unlink-provider"; -import { JuniorChat } from "@/chat/ingress/junior-chat"; -import { createChatSdkLogger, logException, withSpan } from "@/chat/logging"; -import { publishAppHomeView } from "@/chat/slack/app-home"; +import { createChatSdkLogger } from "@/chat/logging"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; -import { getSlackClient } from "@/chat/slack/client"; -import { rehydrateAttachmentFetchers } from "@/chat/queue/thread-message-dispatcher"; -import { handleSlashCommand } from "@/chat/ingress/slash-command"; -import { getStateAdapter } from "@/chat/state/adapter"; +import type { SlackWebhookServices } from "@/chat/ingress/slack-webhook"; +import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; +import { getVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; +import type { VercelConversationWorkCallbackOptions } from "@/chat/task-execution/vercel-callback"; -let productionBot: JuniorChat<{ slack: SlackAdapter }> | undefined; +let productionSlackAdapter: SlackAdapter | undefined; let productionSlackRuntime: ReturnType | undefined; -function createProductionBot(): JuniorChat<{ slack: SlackAdapter }> { - const logger = createChatSdkLogger(); - return new JuniorChat<{ slack: SlackAdapter }>({ - userName: botConfig.userName, - logger, - concurrency: { - strategy: "queue", - // The SDK's default queueEntryTtlMs is 90s, but Junior turns can - // run up to botConfig.turnTimeoutMs (default 12min). A follow-up - // message that arrives during a long turn would expire in the - // queue before the lock is released. Set the TTL to exceed the - // maximum turn duration so queued messages survive. - queueEntryTtlMs: botConfig.turnTimeoutMs + 60_000, - }, - adapters: { - slack: (() => { - const signingSecret = getSlackSigningSecret(); - const botToken = getSlackBotToken(); - const clientId = getSlackClientId(); - const clientSecret = getSlackClientSecret(); +function createProductionSlackAdapter(): SlackAdapter { + const signingSecret = getSlackSigningSecret(); + const botToken = getSlackBotToken(); + const clientId = getSlackClientId(); + const clientSecret = getSlackClientSecret(); - if (!signingSecret) { - throw new Error("SLACK_SIGNING_SECRET is required"); - } + if (!signingSecret) { + throw new Error("SLACK_SIGNING_SECRET is required"); + } - return createJuniorSlackAdapter({ - logger: logger.child("slack"), - signingSecret, - ...(botToken ? { botToken } : {}), - ...(clientId ? { clientId } : {}), - ...(clientSecret ? { clientSecret } : {}), - }); - })(), - }, - state: getStateAdapter(), + return createJuniorSlackAdapter({ + logger: createChatSdkLogger().child("slack"), + signingSecret, + ...(botToken ? { botToken } : {}), + ...(clientId ? { clientId } : {}), + ...(clientSecret ? { clientSecret } : {}), }); } -// Timed-out turns commit a safe session boundary and schedule continuation when -// they hit a safe boundary. MCP auth pauses remain retryable too, -// resumed via the OAuth callback path. -function registerProductionHandlers( - bot: JuniorChat<{ slack: SlackAdapter }>, - slackRuntime: ReturnType, -): void { - bot.onNewMention((thread, message, context) => { - rehydrateAttachmentFetchers(message); - context?.skipped.forEach((skipped) => rehydrateAttachmentFetchers(skipped)); - return slackRuntime.handleNewMention(thread, message, { - messageContext: context, - }); - }); - // Route DMs through the mention handler so every DM gets a reply. - // Without this, the SDK routes DMs in subscribed threads to - // onSubscribedMessage (Chat.dispatchToHandlers checks isSubscribed - // before isDM), where the reply-policy classifier can decide to - // stay silent — wrong for 1:1 conversations. onDirectMessage is - // checked first (Chat.dispatchToHandlers:3128), bypassing the - // subscription branch entirely. - bot.onDirectMessage((thread, message, _channel, context) => { - rehydrateAttachmentFetchers(message); - context?.skipped.forEach((skipped) => rehydrateAttachmentFetchers(skipped)); - return slackRuntime.handleNewMention(thread, message, { - messageContext: context, - }); - }); - bot.onSubscribedMessage((thread, message, context) => { - rehydrateAttachmentFetchers(message); - context?.skipped.forEach((skipped) => rehydrateAttachmentFetchers(skipped)); - return slackRuntime.handleSubscribedMessage(thread, message, { - messageContext: context, - }); - }); - bot.onAssistantThreadStarted((event) => - slackRuntime.handleAssistantThreadStarted(event), - ); - bot.onAssistantContextChanged((event) => - slackRuntime.handleAssistantContextChanged(event), - ); - - bot.onSlashCommand("/jr", (event) => - withSpan( - "chat.slash_command", - "chat.slash_command", - { slackUserId: event.user.userId }, - async () => { - try { - await handleSlashCommand(event); - } catch (error) { - logException(error, "slash_command_failed", { - slackUserId: event.user.userId, - }); - throw error; - } - }, - ), - ); - - bot.onAppHomeOpened((event) => - withSpan( - "chat.app_home_opened", - "chat.app_home_opened", - { slackUserId: event.userId }, - async () => { - try { - await publishAppHomeView( - getSlackClient(), - event.userId, - createUserTokenStore(), - ); - } catch (error) { - logException(error, "app_home_opened_failed", { - slackUserId: event.userId, - }); - } - }, - ), - ); - - bot.onAction("app_home_disconnect", async (event) => { - const provider = event.value; - if (!provider) return; - const userId = event.user.userId; - await withSpan( - "chat.app_home_disconnect", - "chat.app_home_disconnect", - { slackUserId: userId }, - async () => { - try { - await unlinkProvider(userId, provider, createUserTokenStore()); - await publishAppHomeView( - getSlackClient(), - userId, - createUserTokenStore(), - ); - } catch (error) { - logException( - error, - "app_home_disconnect_failed", - { slackUserId: userId }, - { - "app.credential.provider": provider, - }, - ); - } - }, - ); - }); +/** Return the lazily initialized production Slack adapter. */ +export function getProductionSlackAdapter(): SlackAdapter { + productionSlackAdapter ??= createProductionSlackAdapter(); + return productionSlackAdapter; } -function initializeProductionApp(): void { - if (productionBot && productionSlackRuntime) { - return; - } - - const bot = createProductionBot(); - const registerSingleton = ( - bot as unknown as { registerSingleton?: () => unknown } - ).registerSingleton; - if (typeof registerSingleton === "function") { - registerSingleton.call(bot); - } - - const slackRuntime = createSlackRuntime({ - getSlackAdapter: () => bot.getAdapter("slack"), +/** Return the lazily initialized production Slack runtime. */ +export function getProductionSlackRuntime(): ReturnType< + typeof createSlackRuntime +> { + productionSlackRuntime ??= createSlackRuntime({ + getSlackAdapter: getProductionSlackAdapter, }); + return productionSlackRuntime; +} - registerProductionHandlers(bot, slackRuntime); - productionBot = bot; - productionSlackRuntime = slackRuntime; +/** Return production services for Slack webhook ingress. */ +export function getProductionSlackWebhookServices(): SlackWebhookServices { + return { + getSlackAdapter: getProductionSlackAdapter, + queue: getVercelConversationWorkQueue(), + runtime: getProductionSlackRuntime(), + }; } -/** Return the lazily initialized production chat app. */ -export function getProductionBot(): JuniorChat<{ slack: SlackAdapter }> { - initializeProductionApp(); - return productionBot as JuniorChat<{ slack: SlackAdapter }>; +/** Return the production queue callback options for conversation work. */ +export function getProductionConversationWorkOptions(): VercelConversationWorkCallbackOptions { + const runtime = getProductionSlackRuntime(); + return { + queue: getVercelConversationWorkQueue(), + run: createSlackConversationWorker({ + getSlackAdapter: getProductionSlackAdapter, + runtime, + }), + }; } diff --git a/packages/junior/src/chat/config.ts b/packages/junior/src/chat/config.ts index 95d6a77af..8f197f3af 100644 --- a/packages/junior/src/chat/config.ts +++ b/packages/junior/src/chat/config.ts @@ -4,7 +4,7 @@ import { resolveGatewayModel } from "@/chat/pi/client"; const MIN_AGENT_TURN_TIMEOUT_MS = 10 * 1000; const DEFAULT_AGENT_TURN_TIMEOUT_MS = 12 * 60 * 1000; -const DEFAULT_FUNCTION_MAX_DURATION_SECONDS = 800; +const DEFAULT_FUNCTION_MAX_DURATION_SECONDS = 300; const ADVISOR_THINKING_LEVELS = [ "minimal", "low", @@ -16,9 +16,11 @@ const ADVISOR_THINKING_LEVELS = [ export type AdvisorThinkingLevel = (typeof ADVISOR_THINKING_LEVELS)[number]; const DEFAULT_ADVISOR_THINKING_LEVEL: AdvisorThinkingLevel = "xhigh"; -/** Buffer between the Vercel function timeout and the agent turn timeout, - * so the agent can abort and post a failure reply before Vercel kills it. */ -const FUNCTION_TIMEOUT_BUFFER_SECONDS = 20; +/** + * Buffer between the Vercel function timeout and the agent turn timeout so + * Junior can abort, persist, and schedule continuation before host teardown. + */ +export const FUNCTION_TIMEOUT_BUFFER_SECONDS = 20; const DEFAULT_ASSISTANT_LOADING_MESSAGES = [ "Consulting the orb", "Bribing the gremlins", diff --git a/packages/junior/src/chat/ingress/junior-chat.ts b/packages/junior/src/chat/ingress/junior-chat.ts index 77cd89436..2991b60e8 100644 --- a/packages/junior/src/chat/ingress/junior-chat.ts +++ b/packages/junior/src/chat/ingress/junior-chat.ts @@ -13,6 +13,7 @@ import { } from "chat"; import { normalizeIncomingSlackThreadId } from "@/chat/ingress/message-router"; import { isExternalSlackUser } from "@/chat/ingress/workspace-membership"; +import { runWithTurnRequestDeadline } from "@/chat/runtime/request-deadline"; type ChatInternals = { logger?: { @@ -89,7 +90,7 @@ export class JuniorChat< const runtime = this as unknown as ChatInternals; return enqueueBackgroundTask( options, - (async (): Promise => { + runWithTurnRequestDeadline(async (): Promise => { let message: Message; try { message = await messageOrFactory(); @@ -109,7 +110,7 @@ export class JuniorChat< normalized; } await super.processMessage(adapter, normalized, message, options); - })(), + }), ); } @@ -122,7 +123,9 @@ export class JuniorChat< if (normalized !== threadId && "threadId" in message) { (message as unknown as Record).threadId = normalized; } - return super.processMessage(adapter, normalized, message, options); + return runWithTurnRequestDeadline(() => + super.processMessage(adapter, normalized, message, options), + ); } override processReaction( diff --git a/packages/junior/src/chat/ingress/slack-webhook.ts b/packages/junior/src/chat/ingress/slack-webhook.ts new file mode 100644 index 000000000..1f076fd6d --- /dev/null +++ b/packages/junior/src/chat/ingress/slack-webhook.ts @@ -0,0 +1,580 @@ +import type { SlackAdapter, SlackEvent } from "@chat-adapter/slack"; +import { + ChannelImpl, + ThreadImpl, + type Author, + type Message, + type SlashCommandEvent, + type StateAdapter, +} from "chat"; +import type { SlackTurnRuntime } from "@/chat/runtime/slack-runtime"; +import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; +import { appendAndEnqueueInboundMessage } from "@/chat/task-execution/store"; +import { + buildSlackInboundMessage, + type SlackConversationRoute, +} from "@/chat/task-execution/slack-work"; +import { + ensureSlackAdapterInitialized, + runWithSlackInstallation, + verifySlackSignature, + type SlackInstallationContext, +} from "@/chat/slack/adapter-context"; +import { + extractMessageChangedMention, + isMessageChangedEnvelope, +} from "@/chat/ingress/message-changed"; +import { normalizeIncomingSlackThreadId } from "@/chat/ingress/message-router"; +import { isExternalSlackUser } from "@/chat/ingress/workspace-membership"; +import { runWithWorkspaceTeamId } from "@/chat/slack/workspace-context"; +import { getStateAdapter } from "@/chat/state/adapter"; +import { handleSlashCommand } from "@/chat/ingress/slash-command"; +import { createUserTokenStore } from "@/chat/capabilities/factory"; +import { unlinkProvider } from "@/chat/credentials/unlink-provider"; +import { publishAppHomeView } from "@/chat/slack/app-home"; +import { getSlackClient } from "@/chat/slack/client"; +import { logException, withSpan } from "@/chat/logging"; +import type { WaitUntilFn } from "@/handlers/types"; + +type SlackMessageEvent = { + bot_id?: string; + channel?: string; + channel_type?: string; + event_ts?: string; + subtype?: string; + text?: string; + thread_ts?: string; + ts?: string; + type?: string; + user?: string; +}; + +type SlackEventEnvelope = { + enterprise_id?: string; + event?: SlackMessageEvent & Record; + is_enterprise_install?: boolean; + team_id?: string; + type?: string; +}; + +interface SlackInteractivePayload { + actions?: Array<{ + action_id?: string; + selected_option?: { value?: string }; + value?: string; + }>; + team?: { id?: string }; + type?: string; + user?: { id?: string; name?: string; team_id?: string; username?: string }; +} + +export interface SlackWebhookServices { + getSlackAdapter: () => SlackAdapter; + queue: ConversationWorkQueue; + runtime: Pick< + SlackTurnRuntime, + | "handleAssistantContextChanged" + | "handleAssistantThreadStarted" + | "handleNewMention" + | "handleSubscribedMessage" + >; + state?: StateAdapter; +} + +function enqueue(waitUntil: WaitUntilFn, task: Promise): void { + waitUntil(task); +} + +function parseJson(body: string): unknown { + try { + return JSON.parse(body); + } catch { + return undefined; + } +} + +function installationFromEnvelope( + body: SlackEventEnvelope, +): SlackInstallationContext { + return { + teamId: body.team_id, + enterpriseId: body.enterprise_id, + isEnterpriseInstall: body.is_enterprise_install === true, + }; +} + +function isDmEvent(event: SlackMessageEvent): boolean { + return event.channel_type === "im" || event.channel?.startsWith("D") === true; +} + +function textMentionsBot( + event: SlackMessageEvent, + botUserId: string | undefined, +): boolean { + return Boolean(botUserId && event.text?.includes(`<@${botUserId}>`)); +} + +function normalizeMessageThreadId(message: Message): string { + const normalized = normalizeIncomingSlackThreadId(message.threadId, message); + if (normalized !== message.threadId) { + (message as unknown as { threadId: string }).threadId = normalized; + } + return normalized; +} + +async function buildThread(args: { + adapter: SlackAdapter; + message: Message; + route: SlackConversationRoute; + state: StateAdapter; +}): Promise { + const threadId = normalizeMessageThreadId(args.message); + return new ThreadImpl({ + adapter: args.adapter, + stateAdapter: args.state, + id: threadId, + channelId: args.adapter.channelIdFromThreadId(threadId), + channelVisibility: args.adapter.getChannelVisibility(threadId), + currentMessage: args.message, + initialMessage: args.message, + isDM: args.adapter.isDM(threadId), + isSubscribedContext: args.route === "subscribed", + }); +} + +function shouldIgnoreMessage(message: Message): boolean { + return ( + message.author.isMe === true || + message.author.isBot === true || + isExternalSlackUser(message.raw as Record | undefined) + ); +} + +async function persistSlackMessage(args: { + adapter: SlackAdapter; + installation: SlackInstallationContext; + message: Message; + queue: ConversationWorkQueue; + receivedAtMs: number; + route: SlackConversationRoute; + state: StateAdapter; +}): Promise { + const thread = await buildThread(args); + const inbound = buildSlackInboundMessage({ + conversationId: thread.id, + installation: args.installation, + message: args.message, + receivedAtMs: args.receivedAtMs, + route: args.route, + thread, + }); + await appendAndEnqueueInboundMessage({ + message: inbound, + queue: args.queue, + state: args.state, + }); +} + +async function routeParsedMessage(args: { + adapter: SlackAdapter; + event: SlackMessageEvent; + installation: SlackInstallationContext; + message: Message; + queue: ConversationWorkQueue; + receivedAtMs: number; + state: StateAdapter; +}): Promise { + if (shouldIgnoreMessage(args.message)) { + return; + } + + const threadId = normalizeMessageThreadId(args.message); + const isMention = + args.event.type === "app_mention" || + textMentionsBot(args.event, args.adapter.botUserId); + if (isMention) { + args.message.isMention = true; + } + + const route: SlackConversationRoute | undefined = + isDmEvent(args.event) || isMention + ? "mention" + : (await args.state.isSubscribed(threadId)) + ? "subscribed" + : undefined; + if (!route) { + return; + } + + await persistSlackMessage({ + adapter: args.adapter, + installation: args.installation, + message: args.message, + queue: args.queue, + receivedAtMs: args.receivedAtMs, + route, + state: args.state, + }); +} + +async function handleMessageChanged(args: { + adapter: SlackAdapter; + body: unknown; + installation: SlackInstallationContext; + queue: ConversationWorkQueue; + receivedAtMs: number; + state: StateAdapter; +}): Promise { + if (!isMessageChangedEnvelope(args.body)) { + return false; + } + const botUserId = args.adapter.botUserId; + if (!botUserId) { + return false; + } + + const result = extractMessageChangedMention( + args.body, + botUserId, + args.adapter, + ); + if (!result) { + return true; + } + + await persistSlackMessage({ + adapter: args.adapter, + installation: args.installation, + message: result.message, + queue: args.queue, + receivedAtMs: args.receivedAtMs, + route: "mention", + state: args.state, + }); + return true; +} + +async function handleSlackEvent(args: { + body: SlackEventEnvelope; + services: SlackWebhookServices; +}): Promise { + const event = args.body.event; + if (!event) { + return; + } + + const adapter = args.services.getSlackAdapter(); + const state = args.services.state ?? getStateAdapter(); + await state.connect(); + const installation = installationFromEnvelope(args.body); + const receivedAtMs = Date.now(); + + await runWithWorkspaceTeamId(installation.teamId, () => + runWithSlackInstallation({ + adapter, + installation, + state, + task: async () => { + if ( + await handleMessageChanged({ + adapter, + body: args.body, + installation, + queue: args.services.queue, + receivedAtMs, + state, + }) + ) { + return; + } + + if (event.type === "assistant_thread_started") { + const assistantThread = (event as Record) + .assistant_thread as + | { + channel_id?: string; + context?: { channel_id?: string }; + thread_ts?: string; + user_id?: string; + } + | undefined; + if (assistantThread?.channel_id && assistantThread.thread_ts) { + await args.services.runtime.handleAssistantThreadStarted({ + channelId: assistantThread.channel_id, + context: { channelId: assistantThread.context?.channel_id }, + threadId: adapter.encodeThreadId({ + channel: assistantThread.channel_id, + threadTs: assistantThread.thread_ts, + }), + threadTs: assistantThread.thread_ts, + userId: assistantThread.user_id, + }); + } + return; + } + + if (event.type === "assistant_thread_context_changed") { + const assistantThread = (event as Record) + .assistant_thread as + | { + channel_id?: string; + context?: { channel_id?: string }; + thread_ts?: string; + user_id?: string; + } + | undefined; + if (assistantThread?.channel_id && assistantThread.thread_ts) { + await args.services.runtime.handleAssistantContextChanged({ + channelId: assistantThread.channel_id, + context: { channelId: assistantThread.context?.channel_id }, + threadId: adapter.encodeThreadId({ + channel: assistantThread.channel_id, + threadTs: assistantThread.thread_ts, + }), + threadTs: assistantThread.thread_ts, + userId: assistantThread.user_id, + }); + } + return; + } + + if (event.type === "app_home_opened" && event.user) { + await publishAppHomeView( + getSlackClient(), + event.user, + createUserTokenStore(), + ); + return; + } + + if ( + (event.type === "message" || event.type === "app_mention") && + !event.subtype && + event.channel && + event.ts + ) { + const message = adapter.parseMessage(event as SlackEvent); + await routeParsedMessage({ + adapter, + event, + installation, + message, + queue: args.services.queue, + receivedAtMs, + state, + }); + } + }, + }), + ); +} + +function buildAuthorFromInteractive( + user: SlackInteractivePayload["user"], +): Author { + const userId = user?.id ?? "unknown"; + return { + userId, + userName: user?.username ?? user?.name ?? userId, + fullName: user?.name ?? user?.username ?? userId, + isBot: false, + isMe: false, + }; +} + +async function handleSlashCommandForm(args: { + adapter: SlackAdapter; + params: URLSearchParams; + state: StateAdapter; +}): Promise { + const raw = Object.fromEntries(args.params); + const channelId = args.params.get("channel_id") ?? ""; + const channel = new ChannelImpl({ + id: channelId ? `slack:${channelId}` : "", + adapter: args.adapter, + stateAdapter: args.state, + }); + const userId = args.params.get("user_id") || "unknown"; + await withSpan( + "chat.slash_command", + "chat.slash_command", + { slackUserId: userId }, + async () => { + await handleSlashCommand({ + adapter: args.adapter, + channel, + command: args.params.get("command") || "", + text: args.params.get("text") || "", + triggerId: args.params.get("trigger_id") || undefined, + raw, + user: { + userId, + userName: args.params.get("user_name") || userId, + fullName: args.params.get("user_name") || userId, + isBot: false, + isMe: false, + }, + openModal: async () => undefined, + } satisfies SlashCommandEvent); + }, + ); +} + +async function handleInteractivePayload(args: { + adapter: SlackAdapter; + payload: SlackInteractivePayload; +}): Promise { + if (args.payload.type !== "block_actions") { + return; + } + const action = args.payload.actions?.find( + (candidate) => candidate.action_id === "app_home_disconnect", + ); + const provider = action?.selected_option?.value ?? action?.value; + const userId = args.payload.user?.id; + if (!provider || !userId) { + return; + } + + await withSpan( + "chat.app_home_disconnect", + "chat.app_home_disconnect", + { slackUserId: userId }, + async () => { + await unlinkProvider(userId, provider, createUserTokenStore()); + await publishAppHomeView( + getSlackClient(), + userId, + createUserTokenStore(), + ); + }, + ); +} + +function installationFromForm( + params: URLSearchParams, +): SlackInstallationContext { + const isEnterpriseInstall = params.get("is_enterprise_install") === "true"; + return { + teamId: params.get("team_id") ?? undefined, + enterpriseId: params.get("enterprise_id") ?? undefined, + isEnterpriseInstall, + }; +} + +function installationFromInteractive( + payload: SlackInteractivePayload, +): SlackInstallationContext { + return { + teamId: payload.team?.id ?? payload.user?.team_id, + }; +} + +async function handleSlackForm(args: { + body: string; + services: SlackWebhookServices; + waitUntil: WaitUntilFn; +}): Promise { + const params = new URLSearchParams(args.body); + const adapter = args.services.getSlackAdapter(); + const state = args.services.state ?? getStateAdapter(); + await state.connect(); + + if (params.has("command") && !params.has("payload")) { + const installation = installationFromForm(params); + enqueue( + args.waitUntil, + runWithSlackInstallation({ + adapter, + installation, + state, + task: () => + handleSlashCommandForm({ + adapter, + params, + state, + }), + }).catch((error) => { + logException(error, "slash_command_failed", { + slackUserId: params.get("user_id") ?? undefined, + }); + }), + ); + return new Response("", { status: 200 }); + } + + const rawPayload = params.get("payload"); + if (!rawPayload) { + return new Response("Missing payload", { status: 400 }); + } + const payload = parseJson(rawPayload) as SlackInteractivePayload | undefined; + if (!payload) { + return new Response("Invalid payload JSON", { status: 400 }); + } + + enqueue( + args.waitUntil, + runWithSlackInstallation({ + adapter, + installation: installationFromInteractive(payload), + state, + task: () => handleInteractivePayload({ adapter, payload }), + }).catch((error) => { + logException(error, "slack_interactive_payload_failed", { + slackUserId: buildAuthorFromInteractive(payload.user).userId, + }); + }), + ); + return new Response("", { status: 200 }); +} + +/** Handle Slack webhooks by enqueueing durable conversation work. */ +export async function handleSlackWebhook(args: { + request: Request; + services: SlackWebhookServices; + waitUntil: WaitUntilFn; +}): Promise { + const adapter = args.services.getSlackAdapter(); + const body = await args.request.text(); + await ensureSlackAdapterInitialized({ + adapter, + state: args.services.state, + }); + + if (!verifySlackSignature({ adapter, body, request: args.request })) { + return new Response("Invalid signature", { status: 401 }); + } + + const contentType = args.request.headers.get("content-type") || ""; + if (contentType.includes("application/x-www-form-urlencoded")) { + return await handleSlackForm({ + body, + services: args.services, + waitUntil: args.waitUntil, + }); + } + + const parsed = parseJson(body) as SlackEventEnvelope | undefined; + if (!parsed) { + return new Response("Invalid JSON", { status: 400 }); + } + + if (parsed.type === "url_verification") { + const challenge = (parsed as { challenge?: unknown }).challenge; + return Response.json({ challenge }); + } + + if (parsed.type === "event_callback") { + try { + await handleSlackEvent({ + body: parsed, + services: args.services, + }); + } catch (error) { + logException(error, "slack_event_enqueue_failed"); + throw error; + } + } + + return new Response("ok", { status: 200 }); +} diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index ffeec8411..6ae696856 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -209,6 +209,8 @@ export interface ReplyRequestContext { configuration?: Record; /** Durable Pi transcript for this conversation, excluding ephemeral turn context. */ piMessages?: PiMessage[]; + /** Absolute wall-clock deadline for this host request, in milliseconds. */ + turnDeadlineAtMs?: number; channelConfiguration?: ChannelConfigurationService; userAttachments?: Array<{ data?: Buffer; @@ -409,6 +411,16 @@ export async function generateAssistantReply( context: ReplyRequestContext = {}, ): Promise { const replyStartedAtMs = Date.now(); + const configuredTurnDeadlineAtMs = replyStartedAtMs + botConfig.turnTimeoutMs; + const contextTurnDeadlineAtMs = + typeof context.turnDeadlineAtMs === "number" && + Number.isFinite(context.turnDeadlineAtMs) + ? Math.floor(context.turnDeadlineAtMs) + : undefined; + const turnDeadlineAtMs = + contextTurnDeadlineAtMs === undefined + ? configuredTurnDeadlineAtMs + : Math.min(configuredTurnDeadlineAtMs, contextTurnDeadlineAtMs); let timeoutResumeConversationId: string | undefined; let timeoutResumeSessionId: string | undefined; let timeoutResumeSliceId = 1; @@ -1166,7 +1178,7 @@ export async function generateAssistantReply( ): Promise => { let timeoutId: ReturnType | undefined; const timeoutPromise = new Promise((_, reject) => { - timeoutId = setTimeout(() => { + const rejectWithTimeout = () => { timedOut = true; agent.abort(); reject( @@ -1174,7 +1186,13 @@ export async function generateAssistantReply( `Agent turn timed out after ${botConfig.turnTimeoutMs}ms`, ), ); - }, botConfig.turnTimeoutMs); + }; + const remainingTimeoutMs = turnDeadlineAtMs - Date.now(); + if (remainingTimeoutMs <= 0) { + rejectWithTimeout(); + return; + } + timeoutId = setTimeout(rejectWithTimeout, remainingTimeoutMs); }); try { @@ -1195,6 +1213,10 @@ export async function generateAssistantReply( } : {}), "app.ai.turn_timeout_ms": botConfig.turnTimeoutMs, + "app.ai.turn_deadline_remaining_ms": Math.max( + 0, + turnDeadlineAtMs - Date.now(), + ), }, "Agent turn timed out and was aborted", ); diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 69c85b37d..b79e673f9 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -41,6 +41,7 @@ import { } from "@/chat/runtime/thread-context"; import { persistThreadState } from "@/chat/runtime/thread-state"; import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; +import { getTurnRequestDeadline } from "@/chat/runtime/request-deadline"; import { completeAuthPauseTurn } from "@/chat/runtime/auth-pause-state"; import type { PreparedTurnState } from "@/chat/runtime/turn-preparation"; import { @@ -79,18 +80,16 @@ import { appendSlackLegacyAttachmentText } from "@/chat/slack/legacy-attachments import { type ThreadArtifactsState } from "@/chat/state/artifacts"; import { lookupSlackUser } from "@/chat/slack/user"; import type { TurnContinuationRequest } from "@/chat/services/timeout-resume"; -import { canScheduleTurnTimeoutResume } from "@/chat/services/timeout-resume"; import { isRetryableTurnError } from "@/chat/runtime/turn"; import { buildDeterministicTurnId } from "@/chat/runtime/turn"; import { markTurnClosed, markTurnFailed } from "@/chat/runtime/turn"; import { startActiveTurn } from "@/chat/runtime/turn"; import { isRedundantReactionAckText } from "@/chat/services/reply-delivery-plan"; -import { deleteSlackMessage, postSlackMessage } from "@/chat/slack/outbound"; +import { deleteSlackMessage } from "@/chat/slack/outbound"; import { finalizeFailedTurnReply, getAgentTurnDiagnosticsAttributes, } from "@/chat/services/turn-failure-response"; -import { buildSlackTurnContinuationNotice } from "@/chat/slack/turn-continuation-notice"; import { buildAuthPauseResponse } from "@/chat/services/auth-pause-response"; import { maybeApplyProviderDefaultConfigRequest } from "@/chat/services/provider-default-config"; import type { PiMessage } from "@/chat/pi/messages"; @@ -366,41 +365,6 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { beforeFirstResponsePostCalled = true; await options.beforeFirstResponsePost?.(); }; - const postTurnContinuationNotice = async (): Promise => { - try { - await beforeFirstResponsePost(); - const notice = buildSlackTurnContinuationNotice({ conversationId }); - const shouldUseSlackFooter = - Boolean(notice.blocks?.length) && - Boolean(channelId && threadTs) && - (thread.adapter as { name?: string } | undefined)?.name === - "slack"; - if (shouldUseSlackFooter && channelId && threadTs) { - await postSlackMessage({ - channelId, - threadTs, - ...notice, - }); - return; - } - - await thread.post(buildSlackOutputMessage(notice.text)); - } catch (error) { - logException( - error, - "slack_turn_continuation_notice_post_failed", - turnTraceContext, - { - "app.slack.reply_stage": - "thread_reply_turn_continuation_notice", - ...(messageTs ? { "messaging.message.id": messageTs } : {}), - ...getSlackErrorObservabilityAttributes(error), - }, - "Failed to post turn continuation notice", - ); - throw error; - } - }; const postAuthPauseNotice = async (): Promise => { try { await beforeFirstResponsePost(); @@ -447,7 +411,6 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { throw error; } - await postTurnContinuationNotice(); markConversationMessage( preparedState.conversation, preparedState.userMessageId, @@ -682,6 +645,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { omittedImageAttachmentCount, userAttachments, slackConversation, + turnDeadlineAtMs: getTurnRequestDeadline()?.deadlineAtMs, correlation: { conversationId, threadId, @@ -906,12 +870,10 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { const conversationIdForResume = error.metadata?.conversationId; const sessionIdForResume = error.metadata?.sessionId; const version = error.metadata?.version; - const nextSliceId = error.metadata?.sliceId; if ( conversationIdForResume && sessionIdForResume && - typeof version === "number" && - canScheduleTurnTimeoutResume(nextSliceId) + typeof version === "number" ) { try { await deps.services.scheduleTurnTimeoutResume({ @@ -934,24 +896,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { shouldPersistFailureState = true; throw scheduleError; } - await postTurnContinuationNotice(); return; - } else if ( - conversationIdForResume && - sessionIdForResume && - typeof version === "number" - ) { - logWarn( - "agent_turn_timeout_resume_slice_limit_reached", - turnTraceContext, - { - ...(messageTs ? { "messaging.message.id": messageTs } : {}), - ...(typeof nextSliceId === "number" - ? { "app.ai.resume_slice_id": nextSliceId } - : {}), - }, - "Skipped automatic timeout resume because the turn exceeded the slice limit", - ); } else { logWarn( "agent_turn_timeout_resume_metadata_missing", diff --git a/packages/junior/src/chat/runtime/request-deadline.ts b/packages/junior/src/chat/runtime/request-deadline.ts new file mode 100644 index 000000000..a5fe95eb6 --- /dev/null +++ b/packages/junior/src/chat/runtime/request-deadline.ts @@ -0,0 +1,42 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { FUNCTION_TIMEOUT_BUFFER_SECONDS, getChatConfig } from "@/chat/config"; + +export interface TurnRequestDeadline { + deadlineAtMs: number; + startedAtMs: number; +} + +const requestDeadlineStorage = new AsyncLocalStorage(); + +function createTurnRequestDeadline(startedAtMs: number): TurnRequestDeadline { + const requestBudgetMs = Math.max( + 1, + getChatConfig().functionMaxDurationSeconds * 1000 - + FUNCTION_TIMEOUT_BUFFER_SECONDS * 1000, + ); + return { + startedAtMs, + deadlineAtMs: startedAtMs + requestBudgetMs, + }; +} + +/** Return the host request deadline inherited by the current async turn. */ +export function getTurnRequestDeadline(): TurnRequestDeadline | undefined { + return requestDeadlineStorage.getStore(); +} + +/** Run work with one host request deadline shared by nested queued turns. */ +export function runWithTurnRequestDeadline( + callback: () => T, + startedAtMs = Date.now(), +): T { + const existingDeadline = requestDeadlineStorage.getStore(); + if (existingDeadline) { + return callback(); + } + + return requestDeadlineStorage.run( + createTurnRequestDeadline(startedAtMs), + callback, + ); +} diff --git a/packages/junior/src/chat/runtime/slack-resume.ts b/packages/junior/src/chat/runtime/slack-resume.ts index 6e847160f..62e670062 100644 --- a/packages/junior/src/chat/runtime/slack-resume.ts +++ b/packages/junior/src/chat/runtime/slack-resume.ts @@ -26,7 +26,6 @@ import { postSlackApiReplyPosts, } from "@/chat/slack/reply"; import { postSlackMessage as postSlackApiMessage } from "@/chat/slack/outbound"; -import { buildSlackTurnContinuationNotice } from "@/chat/slack/turn-continuation-notice"; import { ACTIVE_LOCK_TTL_MS, getStateAdapter } from "@/chat/state/adapter"; import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; import { addAgentTurnUsage } from "@/chat/usage"; @@ -172,33 +171,6 @@ async function postResumeFailureReply(args: { } } -async function postTurnContinuationNoticeBestEffort(args: { - lockKey: string; - resumeArgs: ResumeSlackTurnArgs; -}): Promise { - const notice = buildSlackTurnContinuationNotice({ - conversationId: - args.resumeArgs.replyContext?.correlation?.conversationId ?? args.lockKey, - }); - try { - await postSlackApiMessage({ - channelId: args.resumeArgs.channelId, - threadTs: args.resumeArgs.threadTs, - ...notice, - }); - } catch (error) { - logException( - error, - "slack_turn_continuation_notice_post_failed", - getResumeLogContext(args.resumeArgs, args.lockKey), - { - "app.slack.reply_stage": "thread_reply_turn_continuation_notice", - }, - "Failed to post turn continuation notice", - ); - } -} - async function handleResumeFailure(args: { body: string; error: unknown; @@ -470,12 +442,6 @@ export async function resumeSlackTurn(args: ResumeSlackTurnArgs) { buildAuthPauseResponse(), ); } - if (deferredPauseKind === "timeout") { - await postTurnContinuationNoticeBestEffort({ - lockKey, - resumeArgs: runArgs, - }); - } return; } catch (pauseError) { await handleResumeFailure({ diff --git a/packages/junior/src/chat/runtime/timeout-resume-runner.ts b/packages/junior/src/chat/runtime/timeout-resume-runner.ts new file mode 100644 index 000000000..f733bbd99 --- /dev/null +++ b/packages/junior/src/chat/runtime/timeout-resume-runner.ts @@ -0,0 +1,313 @@ +import { logException, logWarn } from "@/chat/logging"; +import { + ResumeTurnBusyError, + resumeSlackTurn, +} from "@/chat/runtime/slack-resume"; +import { coerceThreadConversationState } from "@/chat/state/conversation"; +import { + failAgentTurnSessionRecord, + getAgentTurnSessionRecord, + type AgentTurnSessionRecord, +} from "@/chat/state/turn-session"; +import { + getPersistedThreadState, + getPersistedSandboxState, + persistThreadStateById, + getChannelConfigurationServiceById, +} from "@/chat/runtime/thread-state"; +import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; +import { + getTurnUserMessage, + getTurnUserReplyAttachmentContext, + getTurnUserSlackMessageTs, +} from "@/chat/runtime/turn-user-message"; +import { + buildConversationContext, + markConversationMessage, + updateConversationStats, +} from "@/chat/services/conversation-memory"; +import { coerceThreadArtifactsState } from "@/chat/state/artifacts"; +import { isRetryableTurnError, markTurnFailed } from "@/chat/runtime/turn"; +import { + scheduleTurnTimeoutResume, + type TurnContinuationRequest, +} from "@/chat/services/timeout-resume"; +import { parseSlackThreadId } from "@/chat/slack/context"; +import type { AssistantReply } from "@/chat/respond"; +import { persistAuthPauseTurnState } from "@/chat/runtime/auth-pause-state"; +import { + applyPendingAuthUpdate, + clearPendingAuth, +} from "@/chat/services/pending-auth"; + +const TIMEOUT_RESUME_LOCK_RETRY_DELAYS_MS = [250, 1_000, 2_000] as const; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function persistCompletedReplyState(args: { + sessionRecord: AgentTurnSessionRecord; + reply: AssistantReply; +}): Promise { + const currentState = await getPersistedThreadState( + args.sessionRecord.conversationId, + ); + const conversation = coerceThreadConversationState(currentState); + const artifacts = coerceThreadArtifactsState(currentState); + const userMessage = getTurnUserMessage( + conversation, + args.sessionRecord.sessionId, + ); + const statePatch = buildDeliveredTurnStatePatch({ + artifacts, + conversation, + reply: args.reply, + sessionId: args.sessionRecord.sessionId, + userMessageId: userMessage?.id, + }); + + await persistThreadStateById(args.sessionRecord.conversationId, { + ...statePatch, + }); +} + +async function failSessionRecordBestEffort(args: { + sessionRecord: AgentTurnSessionRecord; + errorMessage: string; +}): Promise { + try { + await failAgentTurnSessionRecord({ + conversationId: args.sessionRecord.conversationId, + expectedVersion: args.sessionRecord.version, + sessionId: args.sessionRecord.sessionId, + errorMessage: args.errorMessage, + }); + } catch (error) { + logException( + error, + "timeout_resume_session_record_fail_persist_failed", + {}, + { + "app.ai.conversation_id": args.sessionRecord.conversationId, + "app.ai.session_id": args.sessionRecord.sessionId, + }, + "Failed to mark timed-out turn session record failed", + ); + } +} + +async function persistFailedReplyState( + sessionRecord: AgentTurnSessionRecord, +): Promise { + const currentState = await getPersistedThreadState( + sessionRecord.conversationId, + ); + const conversation = coerceThreadConversationState(currentState); + clearPendingAuth(conversation, sessionRecord.sessionId); + + markTurnFailed({ + conversation, + nowMs: Date.now(), + sessionId: sessionRecord.sessionId, + userMessageId: getTurnUserMessage(conversation, sessionRecord.sessionId) + ?.id, + markConversationMessage, + updateConversationStats, + }); + + await failSessionRecordBestEffort({ + sessionRecord, + errorMessage: "Timed-out turn failed while resuming", + }); + await persistThreadStateById(sessionRecord.conversationId, { + conversation, + }); +} + +/** Resume one durable timeout continuation for a Slack thread. */ +export async function resumeTimedOutTurn( + payload: TurnContinuationRequest, +): Promise { + const thread = parseSlackThreadId(payload.conversationId); + if (!thread) { + throw new Error( + `Timeout resume requires a Slack thread conversation id, got "${payload.conversationId}"`, + ); + } + + await resumeSlackTurn({ + messageText: "", + channelId: thread.channelId, + threadTs: thread.threadTs, + lockKey: payload.conversationId, + beforeStart: async () => { + const sessionRecord = await getAgentTurnSessionRecord( + payload.conversationId, + payload.sessionId, + ); + if ( + !sessionRecord || + sessionRecord.state !== "awaiting_resume" || + sessionRecord.resumeReason !== "timeout" || + sessionRecord.version !== payload.expectedVersion + ) { + return false; + } + + const currentState = await getPersistedThreadState( + payload.conversationId, + ); + const conversation = coerceThreadConversationState(currentState); + const artifacts = coerceThreadArtifactsState(currentState); + const userMessage = getTurnUserMessage(conversation, payload.sessionId); + if (!userMessage?.author?.userId) { + throw new Error( + `Unable to locate the persisted user message for timeout resume session "${payload.sessionId}"`, + ); + } + if (conversation.processing.activeTurnId !== payload.sessionId) { + return false; + } + + const channelConfiguration = getChannelConfigurationServiceById( + thread.channelId, + ); + const conversationContext = buildConversationContext(conversation, { + excludeMessageId: userMessage.id, + }); + const sandbox = getPersistedSandboxState(currentState); + + return { + messageText: userMessage.text, + messageTs: getTurnUserSlackMessageTs(userMessage), + replyContext: { + requester: { + userId: userMessage.author.userId, + userName: userMessage.author.userName, + fullName: userMessage.author.fullName, + }, + correlation: { + conversationId: payload.conversationId, + turnId: payload.sessionId, + channelId: thread.channelId, + threadTs: thread.threadTs, + requesterId: userMessage.author.userId, + }, + toolChannelId: + artifacts.assistantContextChannelId ?? thread.channelId, + artifactState: artifacts, + pendingAuth: conversation.processing.pendingAuth, + conversationContext, + channelConfiguration, + piMessages: conversation.piMessages, + sandbox, + onAuthPending: async (nextPendingAuth) => { + await applyPendingAuthUpdate({ + conversation, + conversationId: payload.conversationId, + nextPendingAuth, + }); + await persistThreadStateById(payload.conversationId, { + conversation, + }); + }, + ...getTurnUserReplyAttachmentContext(userMessage), + }, + onSuccess: async (reply: AssistantReply) => { + await persistCompletedReplyState({ sessionRecord, reply }); + }, + onFailure: async () => { + await persistFailedReplyState(sessionRecord); + }, + onPostDeliveryCommitFailure: async () => { + await failAgentTurnSessionRecord({ + conversationId: sessionRecord.conversationId, + expectedVersion: sessionRecord.version, + sessionId: sessionRecord.sessionId, + errorMessage: + "Timed-out turn reply was delivered but completion state did not persist", + }); + }, + onAuthPause: async () => { + await persistAuthPauseTurnState({ + sessionId: payload.sessionId, + threadStateId: payload.conversationId, + }); + logWarn( + "timeout_resume_reparked_for_auth", + {}, + { + "app.ai.conversation_id": payload.conversationId, + "app.ai.session_id": payload.sessionId, + }, + "Resumed timed-out turn parked for auth", + ); + }, + onTimeoutPause: async (error: unknown) => { + if (!isRetryableTurnError(error, "turn_timeout_resume")) { + throw error; + } + const version = error.metadata?.version; + if (typeof version !== "number") { + throw new Error( + "Timed-out resume turn did not include a turn-session version", + ); + } + + await scheduleTurnTimeoutResume({ + conversationId: payload.conversationId, + sessionId: payload.sessionId, + expectedVersion: version, + }); + }, + }; + }, + }); +} + +/** Retry timeout continuation when the normal Slack thread lock is briefly busy. */ +export async function resumeTimedOutTurnWithLockRetry( + payload: TurnContinuationRequest, +): Promise { + for (const [attempt, delayMs] of [ + ...TIMEOUT_RESUME_LOCK_RETRY_DELAYS_MS, + undefined, + ].entries()) { + try { + await resumeTimedOutTurn(payload); + return; + } catch (error) { + if (!(error instanceof ResumeTurnBusyError)) { + throw error; + } + if (typeof delayMs !== "number") { + logWarn( + "timeout_resume_lock_busy", + {}, + { + "app.ai.conversation_id": payload.conversationId, + "app.ai.session_id": payload.sessionId, + "app.ai.resume_lock_retry_count": attempt, + }, + "Rescheduling timeout resume because another turn still owns the thread lock", + ); + await scheduleTurnTimeoutResume(payload); + return; + } + + logWarn( + "timeout_resume_lock_busy_retrying", + {}, + { + "app.ai.conversation_id": payload.conversationId, + "app.ai.session_id": payload.sessionId, + "app.ai.resume_lock_retry_attempt": attempt + 1, + "app.ai.resume_lock_retry_delay_ms": delayMs, + }, + "Timeout resume lock was busy; retrying", + ); + await sleep(delayMs); + } + } +} diff --git a/packages/junior/src/chat/services/subscribed-decision.ts b/packages/junior/src/chat/services/subscribed-decision.ts index 441feb5ad..ad247c38a 100644 --- a/packages/junior/src/chat/services/subscribed-decision.ts +++ b/packages/junior/src/chat/services/subscribed-decision.ts @@ -77,6 +77,7 @@ const replyDecisionSchema = z.object({ }); const ROUTER_CONFIDENCE_THRESHOLD = 0.8; +const ROUTER_CLASSIFIER_MAX_TOKENS = 240; const LEADING_SLACK_MENTION_RE = /^\s*<@([A-Z0-9]+)(?:\|([^>]+))?>[\s,:-]*/i; const LEADING_NAMED_MENTION_RE = /^\s*@([a-z0-9._-]+)\b[\s,:-]*/i; const TRANSCRIPT_MESSAGE_LINE_RE = @@ -363,7 +364,7 @@ function buildRouterSystemPrompt(botUserName: string): string { "If the latest message clearly tells Junior to stop watching, replying, or participating, set should_unsubscribe=true and should_reply=false.", "When uncertain, prefer should_reply=false with low confidence.", "", - "Return JSON with should_reply, should_unsubscribe, confidence, and a short reason.", + "Return JSON with should_reply, should_unsubscribe, confidence, and a reason under 160 characters.", "Do not return any extra keys.", "", `${escapeXml(botUserName)}`, @@ -466,7 +467,7 @@ export async function decideSubscribedThreadReply(args: { const result = await args.completeObject({ modelId: args.modelId, schema: replyDecisionSchema, - maxTokens: 120, + maxTokens: ROUTER_CLASSIFIER_MAX_TOKENS, temperature: 0, system: buildRouterSystemPrompt(args.botUserName), prompt: buildRouterPrompt(rawText, signals), diff --git a/packages/junior/src/chat/services/timeout-resume.ts b/packages/junior/src/chat/services/timeout-resume.ts index 17a864d50..29e304bdb 100644 --- a/packages/junior/src/chat/services/timeout-resume.ts +++ b/packages/junior/src/chat/services/timeout-resume.ts @@ -1,21 +1,25 @@ /** - * Timeout resume callback signing. + * Timeout resume continuation scheduling. * - * This module owns the internal HTTP handoff used when a turn times out but has - * a safe Pi continuation boundary. It emits and verifies a small signed request - * so only current deployment code can resume the parked turn. + * This module owns the durable queue handoff used when a turn times out but has + * a safe Pi continuation boundary. The signed request verifier remains for + * callbacks that were already in flight during a deployment rollover. */ import { createHmac, timingSafeEqual } from "node:crypto"; -import { resolveBaseUrl } from "@/chat/oauth-flow"; +import type { StateAdapter } from "chat"; import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; +import { + markConversationWorkEnqueued, + requestConversationWork, +} from "@/chat/task-execution/store"; +import { getVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; -const TURN_TIMEOUT_RESUME_PATH = "/api/internal/turn-resume"; const TURN_TIMEOUT_RESUME_HMAC_CONTEXT = "junior.turn_timeout_resume.v1"; const TURN_TIMEOUT_RESUME_SIGNATURE_VERSION = "v1"; const TURN_TIMEOUT_RESUME_MAX_SKEW_MS = 5 * 60 * 1000; const TURN_TIMEOUT_RESUME_TIMESTAMP_HEADER = "x-junior-resume-timestamp"; const TURN_TIMEOUT_RESUME_SIGNATURE_HEADER = "x-junior-resume-signature"; -const MAX_TURN_TIMEOUT_RESUME_SLICE_ID = 5; export interface TurnContinuationRequest { conversationId: string; @@ -23,15 +27,10 @@ export interface TurnContinuationRequest { sessionId: string; } -/** Bound automatic timeout continuation so one bad turn cannot loop forever. */ -export function canScheduleTurnTimeoutResume( - nextSliceId: number | undefined, -): boolean { - return ( - typeof nextSliceId === "number" && - nextSliceId > 1 && - nextSliceId <= MAX_TURN_TIMEOUT_RESUME_SLICE_ID - ); +export interface ScheduleTurnTimeoutResumeOptions { + nowMs?: number; + queue?: ConversationWorkQueue; + state?: StateAdapter; } /** Build the callback request for an awaiting automatic turn continuation. */ @@ -47,7 +46,7 @@ export async function getAwaitingTurnContinuationRequest(args: { !sessionRecord || sessionRecord.state !== "awaiting_resume" || sessionRecord.resumeReason !== "timeout" || - !canScheduleTurnTimeoutResume(sessionRecord.sliceId) + sessionRecord.sliceId < 2 ) { return undefined; } @@ -118,44 +117,34 @@ function parseTurnTimeoutResumeRequest( }; } -/** Schedule an authenticated internal callback to resume a timed-out turn. */ +/** Schedule durable conversation work to resume a timed-out turn. */ export async function scheduleTurnTimeoutResume( request: TurnContinuationRequest, + options: ScheduleTurnTimeoutResumeOptions = {}, ): Promise { - const baseUrl = resolveBaseUrl(); - if (!baseUrl) { - throw new Error( - "Cannot determine base URL for timeout resume callback (set JUNIOR_BASE_URL or deploy to Vercel)", - ); - } - - const secret = getTurnTimeoutResumeSecret(); - if (!secret) { - throw new Error( - "Cannot determine timeout resume secret (set JUNIOR_SECRET)", - ); - } - - const body = JSON.stringify(request); - const timestamp = Date.now().toString(); - const response = await fetch(`${baseUrl}${TURN_TIMEOUT_RESUME_PATH}`, { - method: "POST", - headers: { - "content-type": "application/json", - [TURN_TIMEOUT_RESUME_TIMESTAMP_HEADER]: timestamp, - [TURN_TIMEOUT_RESUME_SIGNATURE_HEADER]: signTurnTimeoutResumeBody( - secret, - timestamp, - body, - ), + const nowMs = options.nowMs ?? Date.now(); + await requestConversationWork({ + conversationId: request.conversationId, + nowMs, + state: options.state, + }); + const queue = options.queue ?? getVercelConversationWorkQueue(); + await queue.send( + { conversationId: request.conversationId }, + { + idempotencyKey: [ + "timeout", + request.conversationId, + request.sessionId, + request.expectedVersion, + ].join(":"), }, - body, + ); + await markConversationWorkEnqueued({ + conversationId: request.conversationId, + nowMs, + state: options.state, }); - if (!response.ok) { - throw new Error( - `Timeout resume callback failed with status ${response.status}`, - ); - } } /** Verify and parse an authenticated timeout resume callback request. */ diff --git a/packages/junior/src/chat/services/turn-continuation-response.ts b/packages/junior/src/chat/services/turn-continuation-response.ts deleted file mode 100644 index f69409120..000000000 --- a/packages/junior/src/chat/services/turn-continuation-response.ts +++ /dev/null @@ -1,7 +0,0 @@ -const TURN_CONTINUATION_RESPONSE = - "I'm still working on this in the background. I'll post the final response here when it finishes."; - -/** Build the user-facing response for a turn parked for continuation. */ -export function buildTurnContinuationResponse(): string { - return TURN_CONTINUATION_RESPONSE; -} diff --git a/packages/junior/src/chat/slack/adapter-context.ts b/packages/junior/src/chat/slack/adapter-context.ts new file mode 100644 index 000000000..062dfaab1 --- /dev/null +++ b/packages/junior/src/chat/slack/adapter-context.ts @@ -0,0 +1,124 @@ +import type { SlackAdapter } from "@chat-adapter/slack"; +import type { ChatInstance, StateAdapter } from "chat"; +import { getStateAdapter } from "@/chat/state/adapter"; + +interface SlackAdapterInternals { + defaultBotTokenProvider?: () => string | Promise; + requestContext?: { + run(context: SlackTokenContext, fn: () => T): T; + }; + resolveTokenForTeam?: ( + installationId: string, + isEnterpriseInstall?: boolean, + ) => Promise<{ botUserId?: string; token: string } | null>; + verifySignature?: ( + body: string, + timestamp: string | null, + signature: string | null, + ) => boolean; +} + +interface SlackTokenContext { + botUserId?: string; + enterpriseId?: string; + isEnterpriseInstall?: boolean; + token: string; +} + +export interface SlackInstallationContext { + enterpriseId?: string; + isEnterpriseInstall?: boolean; + teamId?: string; +} + +const initializedAdapters = new WeakSet(); + +async function getConnectedState( + stateAdapter?: StateAdapter, +): Promise { + const state = stateAdapter ?? getStateAdapter(); + await state.connect(); + return state; +} + +/** Initialize the Slack adapter against the repository state adapter. */ +export async function ensureSlackAdapterInitialized(args: { + adapter: SlackAdapter; + state?: StateAdapter; +}): Promise { + if (initializedAdapters.has(args.adapter)) { + return; + } + const state = await getConnectedState(args.state); + await args.adapter.initialize({ + getState: () => state, + } as unknown as ChatInstance); + initializedAdapters.add(args.adapter); +} + +/** Verify a Slack request using the adapter's configured signing secret. */ +export function verifySlackSignature(args: { + adapter: SlackAdapter; + body: string; + request: Request; +}): boolean { + const internals = args.adapter as unknown as SlackAdapterInternals; + const verifySignature = internals.verifySignature; + if (!verifySignature) { + throw new Error("Slack adapter does not expose signature verification"); + } + return verifySignature.call( + args.adapter, + args.body, + args.request.headers.get("x-slack-request-timestamp"), + args.request.headers.get("x-slack-signature"), + ); +} + +/** Run Slack work with the installation token that matches the inbound event. */ +export async function runWithSlackInstallation(args: { + adapter: SlackAdapter; + installation: SlackInstallationContext; + state?: StateAdapter; + task: () => T | Promise; +}): Promise { + await ensureSlackAdapterInitialized({ + adapter: args.adapter, + state: args.state, + }); + + const internals = args.adapter as unknown as SlackAdapterInternals; + if (internals.defaultBotTokenProvider) { + return await args.task(); + } + + const installationId = args.installation.isEnterpriseInstall + ? args.installation.enterpriseId + : args.installation.teamId; + if (!installationId) { + throw new Error("Slack installation context is missing team id"); + } + if (!internals.resolveTokenForTeam || !internals.requestContext) { + throw new Error("Slack adapter cannot resolve workspace installations"); + } + + const tokenContext = await internals.resolveTokenForTeam.call( + args.adapter, + installationId, + args.installation.isEnterpriseInstall, + ); + if (!tokenContext) { + throw new Error( + `Slack installation token was not found for ${installationId}`, + ); + } + + return await internals.requestContext.run( + { + ...tokenContext, + enterpriseId: args.installation.enterpriseId, + isEnterpriseInstall: args.installation.isEnterpriseInstall, + }, + args.task, + ); +} diff --git a/packages/junior/src/chat/slack/turn-continuation-notice.ts b/packages/junior/src/chat/slack/turn-continuation-notice.ts deleted file mode 100644 index 533293805..000000000 --- a/packages/junior/src/chat/slack/turn-continuation-notice.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { buildTurnContinuationResponse } from "@/chat/services/turn-continuation-response"; -import { - buildSlackReplyBlocks, - buildSlackReplyFooter, - type SlackMessageBlock, -} from "@/chat/slack/footer"; - -/** Build the Slack timeout-continuation acknowledgement with correlation-only metadata. */ -export function buildSlackTurnContinuationNotice(args: { - conversationId?: string; -}): { - blocks?: SlackMessageBlock[]; - text: string; -} { - const text = buildTurnContinuationResponse(); - const footer = buildSlackReplyFooter({ - conversationId: args.conversationId, - }); - const blocks = footer ? buildSlackReplyBlocks(text, footer) : undefined; - return { - text, - ...(blocks ? { blocks } : {}), - }; -} diff --git a/packages/junior/src/chat/task-execution/heartbeat.ts b/packages/junior/src/chat/task-execution/heartbeat.ts new file mode 100644 index 000000000..15aa1ab0b --- /dev/null +++ b/packages/junior/src/chat/task-execution/heartbeat.ts @@ -0,0 +1,133 @@ +import type { StateAdapter } from "chat"; +import { logException, logInfo } from "@/chat/logging"; +import type { ConversationWorkQueue } from "./queue"; +import { + clearExpiredConversationLease, + CONVERSATION_WORK_STALE_ENQUEUE_MS, + getConversationWorkState, + hasRunnableConversationWork, + listConversationWorkIds, + markConversationWorkEnqueued, +} from "./store"; + +const DEFAULT_RECOVERY_LIMIT = 25; + +export interface ConversationWorkRecoveryResult { + expiredLeaseCount: number; + pendingCount: number; +} + +function heartbeatIdempotencyKey( + reason: string, + conversationId: string, +): string { + return `heartbeat:${reason}:${conversationId}`; +} + +async function sendRecoveryNudge(args: { + conversationId: string; + idempotencyKey: string; + nowMs: number; + queue: ConversationWorkQueue; + state?: StateAdapter; +}): Promise { + await args.queue.send( + { conversationId: args.conversationId }, + { idempotencyKey: args.idempotencyKey }, + ); + await markConversationWorkEnqueued({ + conversationId: args.conversationId, + nowMs: args.nowMs, + state: args.state, + }); +} + +/** Requeue expired leases and stranded mailbox work without running the agent. */ +export async function recoverConversationWork(args: { + limit?: number; + nowMs: number; + queue: ConversationWorkQueue; + state?: StateAdapter; +}): Promise { + const result: ConversationWorkRecoveryResult = { + expiredLeaseCount: 0, + pendingCount: 0, + }; + const ids = await listConversationWorkIds({ + limit: args.limit ?? DEFAULT_RECOVERY_LIMIT, + state: args.state, + }); + + for (const conversationId of ids) { + try { + const work = await getConversationWorkState({ + conversationId, + state: args.state, + }); + if (!work) { + continue; + } + + if (work.lease && work.lease.leaseExpiresAtMs <= args.nowMs) { + const cleared = await clearExpiredConversationLease({ + conversationId, + nowMs: args.nowMs, + state: args.state, + }); + if (!cleared) { + continue; + } + await sendRecoveryNudge({ + conversationId, + idempotencyKey: heartbeatIdempotencyKey("lease", conversationId), + nowMs: args.nowMs, + queue: args.queue, + state: args.state, + }); + result.expiredLeaseCount += 1; + logInfo( + "conversation_work_lease_expired_requeued", + { conversationId }, + {}, + "Heartbeat requeued expired conversation work lease", + ); + continue; + } + + if (work.lease || !hasRunnableConversationWork(work)) { + continue; + } + if ( + typeof work.lastEnqueuedAtMs === "number" && + work.lastEnqueuedAtMs + CONVERSATION_WORK_STALE_ENQUEUE_MS > args.nowMs + ) { + continue; + } + + await sendRecoveryNudge({ + conversationId, + idempotencyKey: heartbeatIdempotencyKey("pending", conversationId), + nowMs: args.nowMs, + queue: args.queue, + state: args.state, + }); + result.pendingCount += 1; + logInfo( + "conversation_work_pending_requeued", + { conversationId }, + {}, + "Heartbeat requeued pending conversation work", + ); + } catch (error) { + logException( + error, + "conversation_work_recovery_failed", + { conversationId }, + {}, + "Conversation work heartbeat recovery failed", + ); + } + } + + return result; +} diff --git a/packages/junior/src/chat/task-execution/queue.ts b/packages/junior/src/chat/task-execution/queue.ts new file mode 100644 index 000000000..cf76def02 --- /dev/null +++ b/packages/junior/src/chat/task-execution/queue.ts @@ -0,0 +1,19 @@ +export interface ConversationQueueMessage { + conversationId: string; +} + +export interface ConversationQueueSendOptions { + delayMs?: number; + idempotencyKey?: string; +} + +export interface ConversationQueueSendResult { + messageId?: string; +} + +export interface ConversationWorkQueue { + send( + message: ConversationQueueMessage, + options?: ConversationQueueSendOptions, + ): Promise; +} diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts new file mode 100644 index 000000000..ad5860ca1 --- /dev/null +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -0,0 +1,306 @@ +import type { SlackAdapter } from "@chat-adapter/slack"; +import { + Message, + ThreadImpl, + type MessageContext, + type SerializedMessage, + type SerializedThread, + type StateAdapter, +} from "chat"; +import type { SlackTurnRuntime } from "@/chat/runtime/slack-runtime"; +import { normalizeIncomingSlackThreadId } from "@/chat/ingress/message-router"; +import { rehydrateAttachmentFetchers } from "@/chat/queue/thread-message-dispatcher"; +import { getAwaitingTurnContinuationRequest } from "@/chat/services/timeout-resume"; +import { resumeTimedOutTurnWithLockRetry } from "@/chat/runtime/timeout-resume-runner"; +import { + listAgentTurnSessionSummariesForConversation, + type AgentTurnSessionSummary, +} from "@/chat/state/turn-session"; +import { getStateAdapter } from "@/chat/state/adapter"; +import type { + AgentInputMessage, + InboundMessageRecord, +} from "@/chat/task-execution/store"; +import { + getConversationWorkState, + countPendingConversationMessages, +} from "@/chat/task-execution/store"; +import type { + ConversationWorkerContext, + ConversationWorkerResult, +} from "@/chat/task-execution/worker"; +import { + runWithSlackInstallation, + type SlackInstallationContext, +} from "@/chat/slack/adapter-context"; + +export type SlackConversationRoute = "mention" | "subscribed"; + +export interface SlackConversationMessageMetadata { + [key: string]: unknown; + installation?: SlackInstallationContext; + message: SerializedMessage; + platform: "slack"; + route: SlackConversationRoute; + thread: SerializedThread; +} + +export interface CreateSlackConversationWorkerOptions { + getSlackAdapter: () => SlackAdapter; + runtime: Pick< + SlackTurnRuntime, + "handleNewMention" | "handleSubscribedMessage" + >; + state?: StateAdapter; +} + +function getConnectedState(stateAdapter?: StateAdapter): StateAdapter { + return stateAdapter ?? getStateAdapter(); +} + +function isSlackMetadata( + value: AgentInputMessage["metadata"], +): value is SlackConversationMessageMetadata { + return ( + Boolean(value) && + value?.platform === "slack" && + (value.route === "mention" || value.route === "subscribed") && + Boolean(value.thread) && + Boolean(value.message) + ); +} + +function compareInboundMessages( + left: InboundMessageRecord, + right: InboundMessageRecord, +): number { + return ( + left.createdAtMs - right.createdAtMs || + left.receivedAtMs - right.receivedAtMs || + left.inboundMessageId.localeCompare(right.inboundMessageId) + ); +} + +function routeForRecords( + records: InboundMessageRecord[], +): SlackConversationRoute { + return records.some((record) => record.input.metadata?.route === "mention") + ? "mention" + : "subscribed"; +} + +function restoreMessage(args: { + adapter: SlackAdapter; + record: InboundMessageRecord; +}): Message { + const metadata = args.record.input.metadata; + if (!isSlackMetadata(metadata)) { + throw new Error("Conversation mailbox record is not a Slack message"); + } + + const message = Message.fromJSON(metadata.message); + message.attachments = message.attachments.map((attachment) => + args.adapter.rehydrateAttachment(attachment), + ); + rehydrateAttachmentFetchers(message); + return message; +} + +function restoreThread(args: { + adapter: SlackAdapter; + isSubscribedContext: boolean; + message: Message; + state: StateAdapter; + threadJson: SerializedThread; +}): ThreadImpl { + const threadId = normalizeIncomingSlackThreadId( + args.threadJson.id, + args.message, + ); + if (args.message.threadId !== threadId) { + (args.message as unknown as { threadId: string }).threadId = threadId; + } + return new ThreadImpl({ + adapter: args.adapter, + stateAdapter: args.state, + id: threadId, + channelId: args.threadJson.channelId, + channelVisibility: args.threadJson.channelVisibility, + currentMessage: args.message, + initialMessage: args.message, + isDM: args.threadJson.isDM, + isSubscribedContext: args.isSubscribedContext, + }); +} + +function latestTimeoutResume( + summaries: AgentTurnSessionSummary[], +): AgentTurnSessionSummary | undefined { + return summaries.find( + (summary) => + summary.state === "awaiting_resume" && summary.resumeReason === "timeout", + ); +} + +async function resumeAwaitingTimeout(conversationId: string): Promise { + const summary = latestTimeoutResume( + await listAgentTurnSessionSummariesForConversation(conversationId), + ); + if (!summary) { + return false; + } + + const request = await getAwaitingTurnContinuationRequest({ + conversationId, + sessionId: summary.sessionId, + }); + if (!request) { + return false; + } + + await resumeTimedOutTurnWithLockRetry(request); + return true; +} + +async function getCrashRecoveryRecords(args: { + conversationId: string; + state?: StateAdapter; +}): Promise { + const work = await getConversationWorkState(args); + if (!work || countPendingConversationMessages(work) > 0) { + return []; + } + return work.messages + .filter((message) => message.source === "slack") + .sort(compareInboundMessages); +} + +function getInstallation( + records: InboundMessageRecord[], +): SlackInstallationContext { + for (let index = records.length - 1; index >= 0; index -= 1) { + const metadata = records[index]?.input.metadata; + if (isSlackMetadata(metadata) && metadata.installation) { + return metadata.installation; + } + } + return {}; +} + +/** Build the worker run function for queued Slack conversation work. */ +export function createSlackConversationWorker( + options: CreateSlackConversationWorkerOptions, +): (context: ConversationWorkerContext) => Promise { + return async (context) => { + const adapter = options.getSlackAdapter(); + const state = getConnectedState(options.state); + await state.connect(); + + if (await resumeAwaitingTimeout(context.conversationId)) { + return context.shouldYield() + ? { status: "yielded" } + : { status: "completed" }; + } + + let records = await context.drainMailbox(async () => {}); + if (records.length === 0) { + records = await getCrashRecoveryRecords({ + conversationId: context.conversationId, + state, + }); + } + if (records.length === 0) { + return { status: "completed" }; + } + + records = records.sort(compareInboundMessages); + const latestRecord = records[records.length - 1]; + if (!latestRecord) { + return { status: "completed" }; + } + + const latestMetadata = latestRecord.input.metadata; + if (!isSlackMetadata(latestMetadata)) { + throw new Error( + "Latest conversation mailbox record is not Slack metadata", + ); + } + + await runWithSlackInstallation({ + adapter, + installation: getInstallation(records), + state, + task: async () => { + const messages = records.map((record) => + restoreMessage({ adapter, record }), + ); + const latestMessage = messages[messages.length - 1]; + if (!latestMessage) { + return; + } + const thread = restoreThread({ + adapter, + isSubscribedContext: latestMetadata.route === "subscribed", + message: latestMessage, + state, + threadJson: latestMetadata.thread, + }); + const skipped = messages.slice(0, -1); + const messageContext: MessageContext = { + skipped, + totalSinceLastHandler: messages.length, + }; + + if (routeForRecords(records) === "mention") { + await options.runtime.handleNewMention(thread, latestMessage, { + messageContext, + }); + return; + } + + await options.runtime.handleSubscribedMessage(thread, latestMessage, { + messageContext, + }); + }, + }); + + return context.shouldYield() + ? { status: "yielded" } + : { status: "completed" }; + }; +} + +/** Serialize a Slack message into the generic durable conversation mailbox. */ +export function buildSlackInboundMessage(args: { + conversationId: string; + installation?: SlackInstallationContext; + message: Message; + receivedAtMs: number; + route: SlackConversationRoute; + thread: ThreadImpl; +}): InboundMessageRecord { + return { + conversationId: args.conversationId, + inboundMessageId: [ + "slack", + args.installation?.teamId ?? args.installation?.enterpriseId ?? "unknown", + args.conversationId, + args.message.id, + ].join(":"), + source: "slack", + createdAtMs: args.message.metadata.dateSent.getTime(), + receivedAtMs: args.receivedAtMs, + input: { + text: args.message.text || " ", + authorId: args.message.author.userId, + attachments: args.message.attachments, + metadata: { + platform: "slack", + route: args.route, + installation: args.installation, + thread: args.thread.toJSON(), + message: args.message.toJSON(), + } satisfies SlackConversationMessageMetadata, + }, + }; +} diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts new file mode 100644 index 000000000..bdb576223 --- /dev/null +++ b/packages/junior/src/chat/task-execution/store.ts @@ -0,0 +1,759 @@ +import { randomUUID } from "node:crypto"; +import type { Lock, StateAdapter } from "chat"; +import { isRecord, toOptionalNumber, toOptionalString } from "@/chat/coerce"; +import { getStateAdapter } from "@/chat/state/adapter"; +import { JUNIOR_THREAD_STATE_TTL_MS } from "@/chat/state/ttl"; +import type { ConversationWorkQueue } from "./queue"; + +const CONVERSATION_WORK_PREFIX = "junior:conversation-work"; +const CONVERSATION_WORK_SCHEMA_VERSION = 1; +const CONVERSATION_WORK_INDEX_MAX_LENGTH = 10_000; +const CONVERSATION_WORK_INDEX_LOCK_TTL_MS = 10_000; +const CONVERSATION_WORK_MUTATION_LOCK_TTL_MS = 10_000; +const CONVERSATION_WORK_MUTATION_WAIT_MS = 2_000; +const CONVERSATION_WORK_MUTATION_RETRY_MS = 25; + +export const CONVERSATION_WORK_LEASE_TTL_MS = 90_000; +export const CONVERSATION_WORK_CHECK_IN_INTERVAL_MS = 15_000; +export const CONVERSATION_WORK_STALE_ENQUEUE_MS = 60_000; + +export type InboundMessageSource = "plugin" | "scheduler" | "slack"; + +export interface AgentInputMessage { + attachments?: unknown[]; + authorId?: string; + metadata?: Record; + text: string; +} + +export interface InboundMessageRecord { + conversationId: string; + createdAtMs: number; + inboundMessageId: string; + injectedAtMs?: number; + input: AgentInputMessage; + receivedAtMs: number; + source: InboundMessageSource; +} + +export interface ConversationLease { + acquiredAtMs: number; + lastCheckInAtMs: number; + leaseExpiresAtMs: number; + leaseToken: string; +} + +export interface ConversationWorkState { + conversationId: string; + lastEnqueuedAtMs?: number; + lease?: ConversationLease; + messages: InboundMessageRecord[]; + needsRun: boolean; + schemaVersion: 1; + updatedAtMs: number; +} + +export interface ConversationLeaseAcquired { + leaseExpiresAtMs: number; + leaseToken: string; + status: "acquired"; +} + +export interface ConversationLeaseActive { + leaseExpiresAtMs: number; + status: "active"; +} + +export interface ConversationLeaseNoWork { + status: "no_work"; +} + +export type ConversationLeaseStartResult = + | ConversationLeaseAcquired + | ConversationLeaseActive + | ConversationLeaseNoWork; + +export interface AppendInboundMessageResult { + status: "appended" | "duplicate"; +} + +export interface AppendAndEnqueueInboundMessageResult extends AppendInboundMessageResult { + queueMessageId?: string; +} + +export interface RequestConversationWorkResult { + status: "created" | "updated"; +} + +function stateKey(conversationId: string): string { + return `${CONVERSATION_WORK_PREFIX}:state:${conversationId}`; +} + +function indexKey(): string { + return `${CONVERSATION_WORK_PREFIX}:index`; +} + +function indexLockKey(): string { + return `${CONVERSATION_WORK_PREFIX}:index:lock`; +} + +function mutationLockKey(conversationId: string): string { + return `${CONVERSATION_WORK_PREFIX}:mutation:${conversationId}`; +} + +function now(): number { + return Date.now(); +} + +function uniqueStrings(values: unknown[]): string[] { + return [ + ...new Set( + values.filter((value): value is string => { + return typeof value === "string" && value.trim().length > 0; + }), + ), + ]; +} + +function compareMessages( + left: InboundMessageRecord, + right: InboundMessageRecord, +): number { + return ( + left.createdAtMs - right.createdAtMs || + left.receivedAtMs - right.receivedAtMs || + left.inboundMessageId.localeCompare(right.inboundMessageId) + ); +} + +function normalizeSource(value: unknown): InboundMessageSource | undefined { + if (value === "plugin" || value === "scheduler" || value === "slack") { + return value; + } + return undefined; +} + +function normalizeMetadata( + value: unknown, +): Record | undefined { + if (!isRecord(value)) { + return undefined; + } + return value; +} + +function normalizeInput(value: unknown): AgentInputMessage | undefined { + if (!isRecord(value)) { + return undefined; + } + const text = toOptionalString(value.text); + if (!text) { + return undefined; + } + return { + text, + authorId: toOptionalString(value.authorId), + attachments: Array.isArray(value.attachments) + ? [...value.attachments] + : undefined, + metadata: normalizeMetadata(value.metadata), + }; +} + +function normalizeMessage(value: unknown): InboundMessageRecord | undefined { + if (!isRecord(value)) { + return undefined; + } + const conversationId = toOptionalString(value.conversationId); + const inboundMessageId = toOptionalString(value.inboundMessageId); + const source = normalizeSource(value.source); + const createdAtMs = toOptionalNumber(value.createdAtMs); + const receivedAtMs = toOptionalNumber(value.receivedAtMs); + const input = normalizeInput(value.input); + if ( + !conversationId || + !inboundMessageId || + !source || + typeof createdAtMs !== "number" || + typeof receivedAtMs !== "number" || + !input + ) { + return undefined; + } + return { + conversationId, + inboundMessageId, + source, + createdAtMs, + receivedAtMs, + input, + injectedAtMs: toOptionalNumber(value.injectedAtMs), + }; +} + +function normalizeLease(value: unknown): ConversationLease | undefined { + if (!isRecord(value)) { + return undefined; + } + const leaseToken = toOptionalString(value.leaseToken); + const acquiredAtMs = toOptionalNumber(value.acquiredAtMs); + const lastCheckInAtMs = toOptionalNumber(value.lastCheckInAtMs); + const leaseExpiresAtMs = toOptionalNumber(value.leaseExpiresAtMs); + if ( + !leaseToken || + typeof acquiredAtMs !== "number" || + typeof lastCheckInAtMs !== "number" || + typeof leaseExpiresAtMs !== "number" + ) { + return undefined; + } + return { + leaseToken, + acquiredAtMs, + lastCheckInAtMs, + leaseExpiresAtMs, + }; +} + +function normalizeWorkState( + conversationId: string, + value: unknown, +): ConversationWorkState | undefined { + if ( + !isRecord(value) || + value.schemaVersion !== CONVERSATION_WORK_SCHEMA_VERSION + ) { + return undefined; + } + const storedConversationId = toOptionalString(value.conversationId); + const updatedAtMs = toOptionalNumber(value.updatedAtMs); + if ( + storedConversationId !== conversationId || + typeof updatedAtMs !== "number" + ) { + return undefined; + } + const messages = Array.isArray(value.messages) + ? value.messages + .map(normalizeMessage) + .filter((message): message is InboundMessageRecord => Boolean(message)) + .filter((message) => message.conversationId === conversationId) + .sort(compareMessages) + : []; + return { + schemaVersion: CONVERSATION_WORK_SCHEMA_VERSION, + conversationId, + messages, + needsRun: value.needsRun === true, + updatedAtMs, + lastEnqueuedAtMs: toOptionalNumber(value.lastEnqueuedAtMs), + lease: normalizeLease(value.lease), + }; +} + +function emptyWorkState(args: { + conversationId: string; + nowMs: number; +}): ConversationWorkState { + return { + schemaVersion: CONVERSATION_WORK_SCHEMA_VERSION, + conversationId: args.conversationId, + messages: [], + needsRun: false, + updatedAtMs: args.nowMs, + }; +} + +function isLeaseActive( + lease: ConversationLease | undefined, + nowMs: number, +): boolean { + return Boolean(lease && lease.leaseExpiresAtMs > nowMs); +} + +function pendingMessages(state: ConversationWorkState): InboundMessageRecord[] { + return state.messages + .filter((message) => message.injectedAtMs === undefined) + .sort(compareMessages); +} + +function shouldKeepIndexed(state: ConversationWorkState): boolean { + return ( + state.needsRun || Boolean(state.lease) || pendingMessages(state).length > 0 + ); +} + +async function getConnectedState( + stateAdapter?: StateAdapter, +): Promise { + const state = stateAdapter ?? getStateAdapter(); + await state.connect(); + return state; +} + +async function sleep(ms: number): Promise { + await new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + (timer as { unref?: () => void }).unref?.(); + }); +} + +async function withIndexLock( + state: StateAdapter, + callback: () => Promise, +): Promise { + const lock = await state.acquireLock( + indexLockKey(), + CONVERSATION_WORK_INDEX_LOCK_TTL_MS, + ); + if (!lock) { + throw new Error("Could not acquire conversation work index lock"); + } + try { + return await callback(); + } finally { + await state.releaseLock(lock); + } +} + +async function addToIndex( + state: StateAdapter, + conversationId: string, +): Promise { + await withIndexLock(state, async () => { + const existing = uniqueStrings( + (await state.get(indexKey())) ?? [], + ); + if (existing.includes(conversationId)) { + return; + } + await state.set( + indexKey(), + [...existing, conversationId].slice(-CONVERSATION_WORK_INDEX_MAX_LENGTH), + JUNIOR_THREAD_STATE_TTL_MS, + ); + }); +} + +async function removeFromIndex( + state: StateAdapter, + conversationId: string, +): Promise { + await withIndexLock(state, async () => { + const existing = uniqueStrings( + (await state.get(indexKey())) ?? [], + ); + const next = existing.filter((id) => id !== conversationId); + if (next.length === existing.length) { + return; + } + await state.set(indexKey(), next, JUNIOR_THREAD_STATE_TTL_MS); + }); +} + +async function acquireMutationLock( + state: StateAdapter, + conversationId: string, +): Promise { + const startedAtMs = now(); + while (true) { + const lock = await state.acquireLock( + mutationLockKey(conversationId), + CONVERSATION_WORK_MUTATION_LOCK_TTL_MS, + ); + if (lock) { + return lock; + } + if (now() - startedAtMs >= CONVERSATION_WORK_MUTATION_WAIT_MS) { + throw new Error( + `Could not acquire conversation work mutation lock for ${conversationId}`, + ); + } + await sleep(CONVERSATION_WORK_MUTATION_RETRY_MS); + } +} + +async function withConversationMutation( + args: { + conversationId: string; + state?: StateAdapter; + }, + callback: (state: StateAdapter) => Promise, +): Promise { + const state = await getConnectedState(args.state); + const lock = await acquireMutationLock(state, args.conversationId); + try { + return await callback(state); + } finally { + await state.releaseLock(lock); + } +} + +async function readWorkState( + state: StateAdapter, + conversationId: string, +): Promise { + return normalizeWorkState( + conversationId, + await state.get(stateKey(conversationId)), + ); +} + +async function writeWorkState( + state: StateAdapter, + work: ConversationWorkState, +): Promise { + await state.set( + stateKey(work.conversationId), + work, + JUNIOR_THREAD_STATE_TTL_MS, + ); + if (shouldKeepIndexed(work)) { + await addToIndex(state, work.conversationId); + } else { + await removeFromIndex(state, work.conversationId); + } +} + +function hasRunnableWork(state: ConversationWorkState): boolean { + return state.needsRun || pendingMessages(state).length > 0; +} + +/** Return a persisted conversation work record, if one exists. */ +export async function getConversationWorkState(args: { + conversationId: string; + state?: StateAdapter; +}): Promise { + const state = await getConnectedState(args.state); + return await readWorkState(state, args.conversationId); +} + +/** Count mailbox messages that have not yet reached the session log. */ +export function countPendingConversationMessages( + state: ConversationWorkState, +): number { + return pendingMessages(state).length; +} + +/** Return whether a conversation has pending or resumable execution work. */ +export function hasRunnableConversationWork( + state: ConversationWorkState, +): boolean { + return hasRunnableWork(state); +} + +/** Persist one inbound message idempotently in its conversation mailbox. */ +export async function appendInboundMessage(args: { + message: InboundMessageRecord; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation( + { conversationId: args.message.conversationId, state: args.state }, + async (state) => { + const current = + (await readWorkState(state, args.message.conversationId)) ?? + emptyWorkState({ + conversationId: args.message.conversationId, + nowMs, + }); + const existing = current.messages.find( + (message) => message.inboundMessageId === args.message.inboundMessageId, + ); + if (existing) { + const next: ConversationWorkState = { + ...current, + needsRun: current.needsRun || existing.injectedAtMs === undefined, + updatedAtMs: nowMs, + }; + await writeWorkState(state, next); + return { status: "duplicate" }; + } + + const next: ConversationWorkState = { + ...current, + messages: [...current.messages, args.message].sort(compareMessages), + needsRun: true, + updatedAtMs: nowMs, + }; + await writeWorkState(state, next); + return { status: "appended" }; + }, + ); +} + +/** Persist inbound work and send the queue nudge that wakes a worker. */ +export async function appendAndEnqueueInboundMessage(args: { + message: InboundMessageRecord; + nowMs?: number; + queue: ConversationWorkQueue; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + const appendResult = await appendInboundMessage({ + message: args.message, + nowMs, + state: args.state, + }); + const queueResult = await args.queue.send( + { conversationId: args.message.conversationId }, + { idempotencyKey: args.message.inboundMessageId }, + ); + await markConversationWorkEnqueued({ + conversationId: args.message.conversationId, + nowMs, + state: args.state, + }); + return { + ...appendResult, + queueMessageId: queueResult?.messageId, + }; +} + +/** Mark a conversation runnable when there is no new mailbox message. */ +export async function requestConversationWork(args: { + conversationId: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation(args, async (state) => { + const current = + (await readWorkState(state, args.conversationId)) ?? + emptyWorkState({ + conversationId: args.conversationId, + nowMs, + }); + await writeWorkState(state, { + ...current, + needsRun: true, + updatedAtMs: nowMs, + }); + return { status: current === undefined ? "created" : "updated" }; + }); +} + +/** Record that a wake-up nudge was accepted for the conversation. */ +export async function markConversationWorkEnqueued(args: { + conversationId: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current) { + return; + } + await writeWorkState(state, { + ...current, + lastEnqueuedAtMs: nowMs, + updatedAtMs: nowMs, + }); + }); +} + +/** Try to acquire the durable execution lease for one conversation. */ +export async function startConversationWork(args: { + conversationId: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current || !hasRunnableWork(current)) { + return { status: "no_work" }; + } + if (isLeaseActive(current.lease, nowMs)) { + return { + status: "active", + leaseExpiresAtMs: current.lease!.leaseExpiresAtMs, + }; + } + + const lease: ConversationLease = { + leaseToken: randomUUID(), + acquiredAtMs: nowMs, + lastCheckInAtMs: nowMs, + leaseExpiresAtMs: nowMs + CONVERSATION_WORK_LEASE_TTL_MS, + }; + await writeWorkState(state, { + ...current, + lease, + needsRun: true, + updatedAtMs: nowMs, + }); + return { + status: "acquired", + leaseToken: lease.leaseToken, + leaseExpiresAtMs: lease.leaseExpiresAtMs, + }; + }); +} + +/** Extend the durable execution lease when the worker checks in. */ +export async function checkInConversationWork(args: { + conversationId: string; + leaseToken: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current || current.lease?.leaseToken !== args.leaseToken) { + return false; + } + await writeWorkState(state, { + ...current, + lease: { + ...current.lease, + lastCheckInAtMs: nowMs, + leaseExpiresAtMs: nowMs + CONVERSATION_WORK_LEASE_TTL_MS, + }, + updatedAtMs: nowMs, + }); + return true; + }); +} + +/** Drain pending mailbox entries after the caller has durably injected them. */ +export async function drainConversationMailbox(args: { + conversationId: string; + inject: (messages: InboundMessageRecord[]) => Promise; + leaseToken: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current || current.lease?.leaseToken !== args.leaseToken) { + throw new Error( + `Conversation work lease is not held for ${args.conversationId}`, + ); + } + const pending = pendingMessages(current); + if (pending.length === 0) { + return []; + } + + await args.inject(pending); + const drainedIds = new Set( + pending.map((message) => message.inboundMessageId), + ); + await writeWorkState(state, { + ...current, + messages: current.messages.map((message) => + drainedIds.has(message.inboundMessageId) + ? { ...message, injectedAtMs: nowMs } + : message, + ), + updatedAtMs: nowMs, + }); + return pending; + }); +} + +/** Mark the leased conversation as needing another queue-delivered slice. */ +export async function requestConversationContinuation(args: { + conversationId: string; + leaseToken: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current || current.lease?.leaseToken !== args.leaseToken) { + return false; + } + await writeWorkState(state, { + ...current, + needsRun: true, + updatedAtMs: nowMs, + }); + return true; + }); +} + +/** Release the durable execution lease without changing completion state. */ +export async function releaseConversationWork(args: { + conversationId: string; + leaseToken: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current || current.lease?.leaseToken !== args.leaseToken) { + return false; + } + await writeWorkState(state, { + ...current, + lease: undefined, + updatedAtMs: nowMs, + }); + return true; + }); +} + +/** Finish a leased conversation when no pending mailbox work remains. */ +export async function completeConversationWork(args: { + conversationId: string; + leaseToken: string; + nowMs?: number; + state?: StateAdapter; +}): Promise<"completed" | "lost_lease" | "pending"> { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current || current.lease?.leaseToken !== args.leaseToken) { + return "lost_lease"; + } + const hasPending = pendingMessages(current).length > 0; + await writeWorkState(state, { + ...current, + lease: undefined, + needsRun: hasPending, + updatedAtMs: nowMs, + }); + return hasPending ? "pending" : "completed"; + }); +} + +/** Clear an expired durable lease so a later worker can resume safely. */ +export async function clearExpiredConversationLease(args: { + conversationId: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + return await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current?.lease || current.lease.leaseExpiresAtMs > nowMs) { + return false; + } + await writeWorkState(state, { + ...current, + lease: undefined, + updatedAtMs: nowMs, + }); + return true; + }); +} + +/** List bounded conversation ids that may need heartbeat recovery. */ +export async function listConversationWorkIds( + args: { + limit?: number; + state?: StateAdapter; + } = {}, +): Promise { + const state = await getConnectedState(args.state); + const ids = uniqueStrings((await state.get(indexKey())) ?? []); + return ids.slice(0, args.limit ?? ids.length); +} diff --git a/packages/junior/src/chat/task-execution/vercel-callback.ts b/packages/junior/src/chat/task-execution/vercel-callback.ts new file mode 100644 index 000000000..fd5dc746e --- /dev/null +++ b/packages/junior/src/chat/task-execution/vercel-callback.ts @@ -0,0 +1,77 @@ +import { handleCallback } from "@vercel/queue"; +import type { StateAdapter } from "chat"; +import { runWithTurnRequestDeadline } from "@/chat/runtime/request-deadline"; +import type { ConversationQueueMessage, ConversationWorkQueue } from "./queue"; +import { getVercelConversationWorkQueue } from "./vercel-queue"; +import { + processConversationWork, + type ConversationWorkProcessResult, + type ConversationWorkerResult, + type ConversationWorkerContext, +} from "./worker"; + +export const CONVERSATION_WORK_VISIBILITY_TIMEOUT_SECONDS = 300; + +export interface ProcessConversationQueueMessageOptions { + checkInIntervalMs?: number; + nowMs?: () => number; + queue?: ConversationWorkQueue; + run(context: ConversationWorkerContext): Promise; + softYieldAfterMs?: number; + state?: StateAdapter; +} + +export interface VercelConversationWorkCallbackOptions extends ProcessConversationQueueMessageOptions { + visibilityTimeoutSeconds?: number; +} + +function parseConversationQueueMessage( + message: unknown, +): ConversationQueueMessage { + if ( + !message || + typeof message !== "object" || + typeof (message as { conversationId?: unknown }).conversationId !== + "string" || + !(message as { conversationId: string }).conversationId.trim() + ) { + throw new Error("Conversation queue message is missing conversationId"); + } + return { + conversationId: (message as { conversationId: string }).conversationId, + }; +} + +/** Process one Vercel Queue payload with the generic conversation worker. */ +export async function processConversationQueueMessage( + message: unknown, + options: ProcessConversationQueueMessageOptions, +): Promise { + const parsed = parseConversationQueueMessage(message); + return await processConversationWork(parsed.conversationId, { + checkInIntervalMs: options.checkInIntervalMs, + nowMs: options.nowMs, + queue: options.queue ?? getVercelConversationWorkQueue(), + run: options.run, + softYieldAfterMs: options.softYieldAfterMs, + state: options.state, + }); +} + +/** Create the Vercel Queue push callback for conversation work nudges. */ +export function createVercelConversationWorkCallback( + options: VercelConversationWorkCallbackOptions, +): (request: Request) => Promise { + return handleCallback( + async (message: unknown) => { + await runWithTurnRequestDeadline(() => + processConversationQueueMessage(message, options), + ); + }, + { + visibilityTimeoutSeconds: + options.visibilityTimeoutSeconds ?? + CONVERSATION_WORK_VISIBILITY_TIMEOUT_SECONDS, + }, + ); +} diff --git a/packages/junior/src/chat/task-execution/vercel-queue.ts b/packages/junior/src/chat/task-execution/vercel-queue.ts new file mode 100644 index 000000000..874385863 --- /dev/null +++ b/packages/junior/src/chat/task-execution/vercel-queue.ts @@ -0,0 +1,71 @@ +import { QueueClient } from "@vercel/queue"; +import type { SendOptions, SendResult } from "@vercel/queue"; +import type { + ConversationQueueMessage, + ConversationQueueSendOptions, + ConversationQueueSendResult, + ConversationWorkQueue, +} from "./queue"; + +export const DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC = "junior_conversation_work"; + +interface QueueSender { + send( + topicName: string, + payload: T, + options?: SendOptions, + ): Promise; +} + +export interface VercelConversationWorkQueueOptions { + client?: QueueSender; + retentionSeconds?: number; + topic?: string; +} + +let defaultQueue: ConversationWorkQueue | undefined; + +function getTopic(options: VercelConversationWorkQueueOptions): string { + return ( + options.topic ?? + process.env.JUNIOR_CONVERSATION_WORK_QUEUE_TOPIC?.trim() ?? + DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC + ); +} + +function toDelaySeconds( + options: ConversationQueueSendOptions | undefined, +): number | undefined { + if (!options?.delayMs || options.delayMs <= 0) { + return undefined; + } + return Math.ceil(options.delayMs / 1000); +} + +/** Create the Vercel Queue implementation for conversation wake-up nudges. */ +export function createVercelConversationWorkQueue( + options: VercelConversationWorkQueueOptions = {}, +): ConversationWorkQueue { + const topic = getTopic(options); + const client = options.client ?? new QueueClient(); + + return { + async send( + message: ConversationQueueMessage, + sendOptions?: ConversationQueueSendOptions, + ): Promise { + const result = await client.send(topic, message, { + idempotencyKey: sendOptions?.idempotencyKey, + delaySeconds: toDelaySeconds(sendOptions), + retentionSeconds: options.retentionSeconds, + }); + return result.messageId ? { messageId: result.messageId } : {}; + }, + }; +} + +/** Return the default production conversation work queue. */ +export function getVercelConversationWorkQueue(): ConversationWorkQueue { + defaultQueue ??= createVercelConversationWorkQueue(); + return defaultQueue; +} diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts new file mode 100644 index 000000000..8d0eea486 --- /dev/null +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -0,0 +1,299 @@ +import type { StateAdapter } from "chat"; +import { logException, logInfo, logWarn } from "@/chat/logging"; +import type { ConversationWorkQueue } from "./queue"; +import { + checkInConversationWork, + completeConversationWork, + CONVERSATION_WORK_CHECK_IN_INTERVAL_MS, + countPendingConversationMessages, + drainConversationMailbox, + getConversationWorkState, + markConversationWorkEnqueued, + releaseConversationWork, + requestConversationContinuation, + startConversationWork, + type InboundMessageRecord, +} from "./store"; + +export const CONVERSATION_WORK_DEFER_DELAY_MS = 15_000; +export const CONVERSATION_WORK_SOFT_YIELD_AFTER_MS = 240_000; +export const CONVERSATION_WORK_MIN_NEXT_ITERATION_BUDGET_MS = 120_000; + +export interface ConversationWorkerContext { + checkIn(): Promise; + conversationId: string; + drainMailbox( + inject: (messages: InboundMessageRecord[]) => Promise, + ): Promise; + leaseToken: string; + shouldYield(): boolean; +} + +export interface ConversationWorkerResult { + status: "completed" | "yielded"; +} + +export interface ConversationWorkProcessResult { + status: + | "active" + | "completed" + | "lost_lease" + | "no_work" + | "pending_requeued" + | "yielded"; +} + +export interface ProcessConversationWorkOptions { + checkInIntervalMs?: number; + nowMs?: () => number; + queue: ConversationWorkQueue; + run(context: ConversationWorkerContext): Promise; + softYieldAfterMs?: number; + state?: StateAdapter; +} + +function now(options: ProcessConversationWorkOptions): number { + return options.nowMs?.() ?? Date.now(); +} + +function nudgeIdempotencyKey(reason: string, conversationId: string): string { + return `${reason}:${conversationId}`; +} + +async function sendWakeNudge(args: { + conversationId: string; + delayMs?: number; + idempotencyKey: string; + nowMs: number; + options: ProcessConversationWorkOptions; +}): Promise { + await args.options.queue.send( + { conversationId: args.conversationId }, + { + delayMs: args.delayMs, + idempotencyKey: args.idempotencyKey, + }, + ); + await markConversationWorkEnqueued({ + conversationId: args.conversationId, + nowMs: args.nowMs, + state: args.options.state, + }); +} + +function startLeaseCheckIn(args: { + conversationId: string; + leaseToken: string; + options: ProcessConversationWorkOptions; +}): ReturnType { + const timer = setInterval(() => { + const nowMs = now(args.options); + void checkInConversationWork({ + conversationId: args.conversationId, + leaseToken: args.leaseToken, + nowMs, + state: args.options.state, + }).then( + (checkedIn) => { + if (!checkedIn) { + logWarn( + "conversation_work_check_in_failed", + { conversationId: args.conversationId }, + {}, + "Conversation work check-in lost its lease", + ); + } + }, + (error) => { + logException( + error, + "conversation_work_check_in_failed", + { conversationId: args.conversationId }, + {}, + "Conversation work check-in failed", + ); + }, + ); + }, args.options.checkInIntervalMs ?? CONVERSATION_WORK_CHECK_IN_INTERVAL_MS); + (timer as { unref?: () => void }).unref?.(); + return timer; +} + +/** Process one queue wake-up for a conversation. */ +export async function processConversationWork( + conversationId: string, + options: ProcessConversationWorkOptions, +): Promise { + const initial = await getConversationWorkState({ + conversationId, + state: options.state, + }); + if ( + !initial || + (countPendingConversationMessages(initial) === 0 && !initial.needsRun) + ) { + return { status: "no_work" }; + } + + const lease = await startConversationWork({ + conversationId, + nowMs: now(options), + state: options.state, + }); + if (lease.status === "no_work") { + return { status: "no_work" }; + } + if (lease.status === "active") { + await sendWakeNudge({ + conversationId, + delayMs: CONVERSATION_WORK_DEFER_DELAY_MS, + idempotencyKey: nudgeIdempotencyKey("active", conversationId), + nowMs: now(options), + options, + }); + logInfo( + "conversation_work_nudge_deferred_for_active_lease", + { conversationId }, + { + "app.lease.expires_at_ms": lease.leaseExpiresAtMs, + }, + "Conversation work nudge deferred for active lease", + ); + return { status: "active" }; + } + + const startedAtMs = now(options); + const softYieldDeadlineMs = + startedAtMs + + (options.softYieldAfterMs ?? CONVERSATION_WORK_SOFT_YIELD_AFTER_MS); + const timer = startLeaseCheckIn({ + conversationId, + leaseToken: lease.leaseToken, + options, + }); + logInfo( + "conversation_work_lease_acquired", + { conversationId }, + { + "app.lease.expires_at_ms": lease.leaseExpiresAtMs, + "app.worker.soft_yield_deadline_ms": softYieldDeadlineMs, + }, + "Conversation work lease acquired", + ); + + const workerContext: ConversationWorkerContext = { + conversationId, + leaseToken: lease.leaseToken, + shouldYield: () => now(options) >= softYieldDeadlineMs, + checkIn: () => + checkInConversationWork({ + conversationId, + leaseToken: lease.leaseToken, + nowMs: now(options), + state: options.state, + }), + drainMailbox: (inject) => + drainConversationMailbox({ + conversationId, + leaseToken: lease.leaseToken, + inject, + nowMs: now(options), + state: options.state, + }), + }; + + try { + const result = await options.run(workerContext); + if (result.status === "yielded") { + const continuationMarked = await requestConversationContinuation({ + conversationId, + leaseToken: lease.leaseToken, + nowMs: now(options), + state: options.state, + }); + if (!continuationMarked) { + return { status: "lost_lease" }; + } + await sendWakeNudge({ + conversationId, + idempotencyKey: nudgeIdempotencyKey("yield", conversationId), + nowMs: now(options), + options, + }); + await releaseConversationWork({ + conversationId, + leaseToken: lease.leaseToken, + nowMs: now(options), + state: options.state, + }); + logInfo( + "conversation_work_cooperative_yield", + { conversationId }, + { + "app.worker.elapsed_ms": now(options) - startedAtMs, + "app.worker.soft_yield_deadline_ms": softYieldDeadlineMs, + }, + "Conversation work yielded cooperatively", + ); + return { status: "yielded" }; + } + + const completion = await completeConversationWork({ + conversationId, + leaseToken: lease.leaseToken, + nowMs: now(options), + state: options.state, + }); + if (completion === "lost_lease") { + return { status: "lost_lease" }; + } + if (completion === "pending") { + await sendWakeNudge({ + conversationId, + idempotencyKey: nudgeIdempotencyKey("pending", conversationId), + nowMs: now(options), + options, + }); + return { status: "pending_requeued" }; + } + + logInfo( + "conversation_work_completed", + { conversationId }, + { + "app.worker.elapsed_ms": now(options) - startedAtMs, + }, + "Conversation work completed", + ); + return { status: "completed" }; + } catch (error) { + try { + await releaseConversationWork({ + conversationId, + leaseToken: lease.leaseToken, + nowMs: now(options), + state: options.state, + }); + } catch (releaseError) { + logException( + releaseError, + "conversation_work_release_failed", + { conversationId }, + {}, + "Conversation work release failed after runner error", + ); + } + logException( + error, + "conversation_work_failed", + { conversationId }, + { + "app.worker.elapsed_ms": now(options) - startedAtMs, + }, + "Conversation work failed", + ); + throw error; + } finally { + clearInterval(timer); + } +} diff --git a/packages/junior/src/handlers/heartbeat.ts b/packages/junior/src/handlers/heartbeat.ts index f4e3ef1f0..84850a015 100644 --- a/packages/junior/src/handlers/heartbeat.ts +++ b/packages/junior/src/handlers/heartbeat.ts @@ -1,8 +1,13 @@ import { timingSafeEqual } from "node:crypto"; import { runHeartbeat } from "@/chat/agent-dispatch/heartbeat"; +import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { logException } from "@/chat/logging"; import type { WaitUntilFn } from "@/handlers/types"; +export interface HeartbeatHandlerOptions { + conversationWorkQueue?: ConversationWorkQueue; +} + function getHeartbeatSecret(): string | undefined { return ( process.env.JUNIOR_SCHEDULER_SECRET?.trim() || @@ -29,6 +34,7 @@ function verifyHeartbeatRequest(request: Request): boolean { export async function GET( request: Request, waitUntil: WaitUntilFn, + options: HeartbeatHandlerOptions = {}, ): Promise { if (!verifyHeartbeatRequest(request)) { return new Response("Unauthorized", { status: 401 }); @@ -36,7 +42,10 @@ export async function GET( const nowMs = Date.now(); waitUntil(() => - runHeartbeat({ nowMs }).catch((error) => { + runHeartbeat({ + conversationWorkQueue: options.conversationWorkQueue, + nowMs, + }).catch((error) => { logException( error, "heartbeat_failed", diff --git a/packages/junior/src/handlers/mcp-oauth-callback.ts b/packages/junior/src/handlers/mcp-oauth-callback.ts index f4fdecd37..50363b426 100644 --- a/packages/junior/src/handlers/mcp-oauth-callback.ts +++ b/packages/junior/src/handlers/mcp-oauth-callback.ts @@ -49,10 +49,7 @@ import { } from "@/chat/state/turn-session"; import { recordAuthorizationCompleted } from "@/chat/state/session-log"; import { isRetryableTurnError, markTurnFailed } from "@/chat/runtime/turn"; -import { - canScheduleTurnTimeoutResume, - scheduleTurnTimeoutResume, -} from "@/chat/services/timeout-resume"; +import { scheduleTurnTimeoutResume } from "@/chat/services/timeout-resume"; import { htmlCallbackResponse } from "@/handlers/oauth-html"; import type { WaitUntilFn } from "@/handlers/types"; @@ -403,28 +400,11 @@ async function resumeAuthorizedMcpTurn(args: { throw error; } const version = error.metadata?.version; - const nextSliceId = error.metadata?.sliceId; if (typeof version !== "number") { throw new Error( "Timed-out MCP resume did not include a turn-session version", ); } - if (!canScheduleTurnTimeoutResume(nextSliceId)) { - logWarn( - "mcp_oauth_callback_resume_slice_limit_reached", - {}, - { - "app.credential.provider": provider, - ...(typeof nextSliceId === "number" - ? { "app.ai.resume_slice_id": nextSliceId } - : {}), - }, - "Skipped automatic timeout resume because the turn exceeded the slice limit", - ); - throw new Error( - "Timed-out turn exceeded the automatic resume slice limit", - ); - } await scheduleTurnTimeoutResume({ conversationId: authSession.conversationId, sessionId: lockedSessionId, diff --git a/packages/junior/src/handlers/oauth-callback.ts b/packages/junior/src/handlers/oauth-callback.ts index 0bb27604a..96465611a 100644 --- a/packages/junior/src/handlers/oauth-callback.ts +++ b/packages/junior/src/handlers/oauth-callback.ts @@ -57,10 +57,7 @@ import { } from "@/chat/services/pending-auth"; import { escapeXml } from "@/chat/xml"; import type { WaitUntilFn } from "@/handlers/types"; -import { - canScheduleTurnTimeoutResume, - scheduleTurnTimeoutResume, -} from "@/chat/services/timeout-resume"; +import { scheduleTurnTimeoutResume } from "@/chat/services/timeout-resume"; import type { AssistantReply } from "@/chat/respond"; /** @@ -413,17 +410,11 @@ async function resumeOAuthSessionRecordTurn( throw error; } const version = error.metadata?.version; - const nextSliceId = error.metadata?.sliceId; if (typeof version !== "number") { throw new Error( "Timed-out OAuth resume did not include a turn-session version", ); } - if (!canScheduleTurnTimeoutResume(nextSliceId)) { - throw new Error( - "Timed-out turn exceeded the automatic resume slice limit", - ); - } await scheduleTurnTimeoutResume({ conversationId: stored.resumeConversationId!, sessionId: lockedSessionId, diff --git a/packages/junior/src/handlers/turn-resume.ts b/packages/junior/src/handlers/turn-resume.ts index d1f2584fc..cd918ee44 100644 --- a/packages/junior/src/handlers/turn-resume.ts +++ b/packages/junior/src/handlers/turn-resume.ts @@ -1,351 +1,16 @@ /** * Internal timeout-resume handler. * - * This handler receives signed continuation callbacks for turns that outlived a - * request slice. It reloads the turn session, reacquires Slack/thread context, - * and either resumes the agent or schedules the next slice while preserving the - * same conversation/session identity. + * This route remains for signed callbacks that were already in flight during a + * deployment rollover. New timeout continuations are delivered through the + * durable conversation queue. */ -import { logException, logWarn } from "@/chat/logging"; -import { - ResumeTurnBusyError, - resumeSlackTurn, -} from "@/chat/runtime/slack-resume"; -import { coerceThreadConversationState } from "@/chat/state/conversation"; -import { - failAgentTurnSessionRecord, - getAgentTurnSessionRecord, - type AgentTurnSessionRecord, -} from "@/chat/state/turn-session"; -import { - getPersistedThreadState, - getPersistedSandboxState, - persistThreadStateById, - getChannelConfigurationServiceById, -} from "@/chat/runtime/thread-state"; -import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; -import { - getTurnUserMessage, - getTurnUserReplyAttachmentContext, - getTurnUserSlackMessageTs, -} from "@/chat/runtime/turn-user-message"; -import { - buildConversationContext, - markConversationMessage, - updateConversationStats, -} from "@/chat/services/conversation-memory"; -import { coerceThreadArtifactsState } from "@/chat/state/artifacts"; -import { isRetryableTurnError, markTurnFailed } from "@/chat/runtime/turn"; -import { - canScheduleTurnTimeoutResume, - scheduleTurnTimeoutResume, - verifyTurnTimeoutResumeRequest, - type TurnContinuationRequest, -} from "@/chat/services/timeout-resume"; -import { parseSlackThreadId } from "@/chat/slack/context"; -import type { AssistantReply } from "@/chat/respond"; -import { persistAuthPauseTurnState } from "@/chat/runtime/auth-pause-state"; -import { - applyPendingAuthUpdate, - clearPendingAuth, -} from "@/chat/services/pending-auth"; +import { logException } from "@/chat/logging"; +import { resumeTimedOutTurnWithLockRetry } from "@/chat/runtime/timeout-resume-runner"; +import { verifyTurnTimeoutResumeRequest } from "@/chat/services/timeout-resume"; import type { WaitUntilFn } from "@/handlers/types"; -const TIMEOUT_RESUME_LOCK_RETRY_DELAYS_MS = [250, 1_000, 2_000] as const; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function persistCompletedReplyState(args: { - sessionRecord: AgentTurnSessionRecord; - reply: AssistantReply; -}): Promise { - const currentState = await getPersistedThreadState( - args.sessionRecord.conversationId, - ); - const conversation = coerceThreadConversationState(currentState); - const artifacts = coerceThreadArtifactsState(currentState); - const userMessage = getTurnUserMessage( - conversation, - args.sessionRecord.sessionId, - ); - const statePatch = buildDeliveredTurnStatePatch({ - artifacts, - conversation, - reply: args.reply, - sessionId: args.sessionRecord.sessionId, - userMessageId: userMessage?.id, - }); - - await persistThreadStateById(args.sessionRecord.conversationId, { - ...statePatch, - }); -} - -async function failSessionRecordBestEffort(args: { - sessionRecord: AgentTurnSessionRecord; - errorMessage: string; -}): Promise { - try { - await failAgentTurnSessionRecord({ - conversationId: args.sessionRecord.conversationId, - expectedVersion: args.sessionRecord.version, - sessionId: args.sessionRecord.sessionId, - errorMessage: args.errorMessage, - }); - } catch (error) { - logException( - error, - "timeout_resume_session_record_fail_persist_failed", - {}, - { - "app.ai.conversation_id": args.sessionRecord.conversationId, - "app.ai.session_id": args.sessionRecord.sessionId, - }, - "Failed to mark timed-out turn session record failed", - ); - } -} - -async function persistFailedReplyState( - sessionRecord: AgentTurnSessionRecord, -): Promise { - const currentState = await getPersistedThreadState( - sessionRecord.conversationId, - ); - const conversation = coerceThreadConversationState(currentState); - clearPendingAuth(conversation, sessionRecord.sessionId); - - markTurnFailed({ - conversation, - nowMs: Date.now(), - sessionId: sessionRecord.sessionId, - userMessageId: getTurnUserMessage(conversation, sessionRecord.sessionId) - ?.id, - markConversationMessage, - updateConversationStats, - }); - - await failSessionRecordBestEffort({ - sessionRecord, - errorMessage: "Timed-out turn failed while resuming", - }); - await persistThreadStateById(sessionRecord.conversationId, { - conversation, - }); -} - -async function resumeTimedOutTurn( - payload: TurnContinuationRequest, -): Promise { - const thread = parseSlackThreadId(payload.conversationId); - if (!thread) { - throw new Error( - `Timeout resume requires a Slack thread conversation id, got "${payload.conversationId}"`, - ); - } - - await resumeSlackTurn({ - messageText: "", - channelId: thread.channelId, - threadTs: thread.threadTs, - lockKey: payload.conversationId, - beforeStart: async () => { - const sessionRecord = await getAgentTurnSessionRecord( - payload.conversationId, - payload.sessionId, - ); - if ( - !sessionRecord || - sessionRecord.state !== "awaiting_resume" || - sessionRecord.resumeReason !== "timeout" || - sessionRecord.version !== payload.expectedVersion - ) { - return false; - } - - const currentState = await getPersistedThreadState( - payload.conversationId, - ); - const conversation = coerceThreadConversationState(currentState); - const artifacts = coerceThreadArtifactsState(currentState); - const userMessage = getTurnUserMessage(conversation, payload.sessionId); - if (!userMessage?.author?.userId) { - throw new Error( - `Unable to locate the persisted user message for timeout resume session "${payload.sessionId}"`, - ); - } - if (conversation.processing.activeTurnId !== payload.sessionId) { - return false; - } - - const channelConfiguration = getChannelConfigurationServiceById( - thread.channelId, - ); - const conversationContext = buildConversationContext(conversation, { - excludeMessageId: userMessage.id, - }); - const sandbox = getPersistedSandboxState(currentState); - - return { - messageText: userMessage.text, - messageTs: getTurnUserSlackMessageTs(userMessage), - replyContext: { - credentialContext: { - actor: { - type: "user", - userId: userMessage.author.userId, - }, - }, - requester: { - userId: userMessage.author.userId, - userName: userMessage.author.userName, - fullName: userMessage.author.fullName, - }, - correlation: { - conversationId: payload.conversationId, - turnId: payload.sessionId, - channelId: thread.channelId, - threadTs: thread.threadTs, - requesterId: userMessage.author.userId, - }, - toolChannelId: - artifacts.assistantContextChannelId ?? thread.channelId, - artifactState: artifacts, - pendingAuth: conversation.processing.pendingAuth, - conversationContext, - channelConfiguration, - piMessages: conversation.piMessages, - sandbox, - onAuthPending: async (nextPendingAuth) => { - await applyPendingAuthUpdate({ - conversation, - conversationId: payload.conversationId, - nextPendingAuth, - }); - await persistThreadStateById(payload.conversationId, { - conversation, - }); - }, - ...getTurnUserReplyAttachmentContext(userMessage), - }, - onSuccess: async (reply: AssistantReply) => { - await persistCompletedReplyState({ sessionRecord, reply }); - }, - onFailure: async () => { - await persistFailedReplyState(sessionRecord); - }, - onPostDeliveryCommitFailure: async () => { - await failAgentTurnSessionRecord({ - conversationId: sessionRecord.conversationId, - expectedVersion: sessionRecord.version, - sessionId: sessionRecord.sessionId, - errorMessage: - "Timed-out turn reply was delivered but completion state did not persist", - }); - }, - onAuthPause: async () => { - await persistAuthPauseTurnState({ - sessionId: payload.sessionId, - threadStateId: payload.conversationId, - }); - logWarn( - "timeout_resume_reparked_for_auth", - {}, - { - "app.ai.conversation_id": payload.conversationId, - "app.ai.session_id": payload.sessionId, - }, - "Resumed timed-out turn parked for auth", - ); - }, - onTimeoutPause: async (error: unknown) => { - if (!isRetryableTurnError(error, "turn_timeout_resume")) { - throw error; - } - const version = error.metadata?.version; - const nextSliceId = error.metadata?.sliceId; - if (typeof version !== "number") { - throw new Error( - "Timed-out resume turn did not include a turn-session version", - ); - } - if (!canScheduleTurnTimeoutResume(nextSliceId)) { - logWarn( - "timeout_resume_slice_limit_reached", - {}, - { - "app.ai.conversation_id": payload.conversationId, - "app.ai.session_id": payload.sessionId, - ...(typeof nextSliceId === "number" - ? { "app.ai.resume_slice_id": nextSliceId } - : {}), - }, - "Skipped automatic timeout resume because the turn exceeded the slice limit", - ); - throw new Error( - "Timed-out turn exceeded the automatic resume slice limit", - ); - } - - await scheduleTurnTimeoutResume({ - conversationId: payload.conversationId, - sessionId: payload.sessionId, - expectedVersion: version, - }); - }, - }; - }, - }); -} - -async function resumeTimedOutTurnWithLockRetry( - payload: TurnContinuationRequest, -): Promise { - for (const [attempt, delayMs] of [ - ...TIMEOUT_RESUME_LOCK_RETRY_DELAYS_MS, - undefined, - ].entries()) { - try { - await resumeTimedOutTurn(payload); - return; - } catch (error) { - if (!(error instanceof ResumeTurnBusyError)) { - throw error; - } - if (typeof delayMs !== "number") { - logWarn( - "timeout_resume_lock_busy", - {}, - { - "app.ai.conversation_id": payload.conversationId, - "app.ai.session_id": payload.sessionId, - "app.ai.resume_lock_retry_count": attempt, - }, - "Rescheduling timeout resume because another turn still owns the thread lock", - ); - await scheduleTurnTimeoutResume(payload); - return; - } - - logWarn( - "timeout_resume_lock_busy_retrying", - {}, - { - "app.ai.conversation_id": payload.conversationId, - "app.ai.session_id": payload.sessionId, - "app.ai.resume_lock_retry_attempt": attempt + 1, - "app.ai.resume_lock_retry_delay_ms": delayMs, - }, - "Timeout resume lock was busy; retrying", - ); - await sleep(delayMs); - } - } -} - -/** Handle the authenticated internal timeout-resume callback. */ +/** Handle an authenticated internal timeout-resume callback. */ export async function POST( request: Request, waitUntil: WaitUntilFn, diff --git a/packages/junior/src/handlers/webhooks.ts b/packages/junior/src/handlers/webhooks.ts index e74da956d..5f2c764e0 100644 --- a/packages/junior/src/handlers/webhooks.ts +++ b/packages/junior/src/handlers/webhooks.ts @@ -1,4 +1,7 @@ -import { getProductionBot } from "@/chat/app/production"; +import type { SlackAdapter } from "@chat-adapter/slack"; +import { getProductionSlackWebhookServices } from "@/chat/app/production"; +import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; +import { JuniorChat } from "@/chat/ingress/junior-chat"; import { extractMessageChangedMention, isMessageChangedEnvelope, @@ -30,6 +33,8 @@ interface SlackWebhookAuthAdapter { ) => boolean; } +type LegacyChatSdkBot = JuniorChat<{ slack: SlackAdapter }>; + function getSlackPayloadTeamId(body: unknown): string | undefined { if (!body || typeof body !== "object") { return undefined; @@ -41,7 +46,7 @@ function getSlackPayloadTeamId(body: unknown): string | undefined { async function handleAuthenticatedSlackMessageChangedMention(args: { body: unknown; - bot: ReturnType; + bot: LegacyChatSdkBot; rawBody: string; request: Request; waitUntil: WaitUntilFn; @@ -51,14 +56,10 @@ async function handleAuthenticatedSlackMessageChangedMention(args: { const timestamp = args.request.headers.get("x-slack-request-timestamp"); const signature = args.request.headers.get("x-slack-signature"); - // Reuse the adapter's own Slack signature verification before dispatching - // the synthetic edit event so this side-channel cannot bypass auth. if (!authAdapter.verifySignature(args.rawBody, timestamp, signature)) { return; } - // Chat SDK initializes adapters automatically inside webhook handling. This - // side-channel runs before the SDK handler, so it must join that lifecycle. await args.bot.initialize(); const webhookOptions = { @@ -111,81 +112,76 @@ async function handleAuthenticatedSlackMessageChangedMention(args: { authAdapter.requestContext.run(context, dispatch); } +async function handleLegacyChatSdkWebhook(args: { + bot: LegacyChatSdkBot; + platform: string; + request: Request; + waitUntil: WaitUntilFn; +}): Promise { + const handler = + args.bot.webhooks[args.platform as keyof typeof args.bot.webhooks]; + if (!handler) { + return new Response(`Unknown platform: ${args.platform}`, { status: 404 }); + } + + let request = args.request; + let slackWorkspaceTeamId: string | undefined; + if (args.platform === "slack") { + const rawBody = await args.request.text(); + const parsedBody = parseJson(rawBody); + slackWorkspaceTeamId = getSlackPayloadTeamId(parsedBody); + + if (parsedBody && isMessageChangedEnvelope(parsedBody)) { + await runWithWorkspaceTeamId(slackWorkspaceTeamId, () => + handleAuthenticatedSlackMessageChangedMention({ + body: parsedBody, + bot: args.bot, + rawBody, + request: args.request, + waitUntil: args.waitUntil, + }), + ); + } + + request = new Request(args.request.url, { + method: args.request.method, + headers: args.request.headers, + body: rawBody, + }); + } + + return await runWithWorkspaceTeamId(slackWorkspaceTeamId, () => + handler(request, { + waitUntil: (task: Promise) => args.waitUntil(task), + } as Parameters[1]), + ); +} + +function parseJson(body: string): unknown { + try { + return JSON.parse(body); + } catch { + return undefined; + } +} + /** * Handles `POST /api/webhooks/:platform`. * - * The router only resolves the platform and delegates to the adapter webhook - * implementation; request semantics stay owned by the adapter package. - * - * For Slack, the body is read once and used to detect `message_changed` events - * that introduce a new bot @mention, which the Slack adapter silently ignores. - * The request is then reconstructed so the adapter can consume it normally. + * Slack production ingress persists messages into the durable conversation + * mailbox and wakes the queue worker. The optional `legacyBot` parameter is + * kept for integration tests that still exercise Chat SDK fixtures directly. */ export async function handlePlatformWebhook( request: Request, platform: string, waitUntil: WaitUntilFn, - bot = getProductionBot(), + legacyBot?: LegacyChatSdkBot, ): Promise { - const handler = bot.webhooks[platform as keyof typeof bot.webhooks]; const requestContext = createRequestContext(request, { platform }); const requestUrl = new URL(request.url); - return withContext(requestContext, async () => { - if (!handler) { - const error = new Error(`Unknown platform: ${platform}`); - logException( - error, - "webhook_platform_unknown", - {}, - { - "http.response.status_code": 404, - }, - `Unknown platform: ${platform}`, - ); - return new Response(`Unknown platform: ${platform}`, { status: 404 }); - } - - // For Slack webhooks, peek the body to handle `message_changed` events - // that introduce a new bot @mention. The Slack adapter drops these subtypes, - // so we dispatch them as a synthesized mention before forwarding to the adapter. - let rebuiltRequest = request; - let slackWorkspaceTeamId: string | undefined; - if (platform === "slack") { - const rawBody = await request.text(); - let parsedBody: unknown; - try { - parsedBody = JSON.parse(rawBody); - } catch { - parsedBody = undefined; - } - - slackWorkspaceTeamId = getSlackPayloadTeamId(parsedBody); - - if (parsedBody && isMessageChangedEnvelope(parsedBody)) { - try { - await runWithWorkspaceTeamId(slackWorkspaceTeamId, () => - handleAuthenticatedSlackMessageChangedMention({ - body: parsedBody, - bot, - rawBody, - request, - waitUntil, - }), - ); - } catch (error) { - logException(error, "slack_message_changed_side_channel_failed"); - } - } - - // Reconstruct the request so the adapter can read the body. - rebuiltRequest = new Request(request.url, { - method: request.method, - headers: request.headers, - body: rawBody, - }); - } - + return await withContext(requestContext, async () => { try { return await withSpan( "http.server.request", @@ -193,13 +189,26 @@ export async function handlePlatformWebhook( requestContext, async () => { try { - const response = await runWithWorkspaceTeamId( - slackWorkspaceTeamId, - () => - handler(rebuiltRequest, { - waitUntil: (task: Promise) => waitUntil(task), - } as Parameters[1]), - ); + let response: Response; + if (legacyBot) { + response = await handleLegacyChatSdkWebhook({ + bot: legacyBot, + platform, + request, + waitUntil, + }); + } else if (platform === "slack") { + response = await handleSlackWebhook({ + request, + services: getProductionSlackWebhookServices(), + waitUntil, + }); + } else { + response = new Response(`Unknown platform: ${platform}`, { + status: 404, + }); + } + if (response.status >= 400) { let responseBodySnippet: string | undefined; try { @@ -227,6 +236,7 @@ export async function handlePlatformWebhook( `Webhook ${platform} returned ${response.status}`, ); } + setSpanAttributes({ "http.response.status_code": response.status, }); @@ -249,6 +259,7 @@ export async function handlePlatformWebhook( }); } +/** Handle a platform webhook request from the app route. */ export async function POST( request: Request, platform: string, diff --git a/packages/junior/src/nitro.ts b/packages/junior/src/nitro.ts index a04757bf9..fc30b2a25 100644 --- a/packages/junior/src/nitro.ts +++ b/packages/junior/src/nitro.ts @@ -193,7 +193,7 @@ export function juniorNitro(options: JuniorNitroOptions = {}): { nitro.options.vercel ??= {}; nitro.options.vercel.functions ??= {}; nitro.options.vercel.functions.maxDuration ??= - options.maxDuration ?? 800; + options.maxDuration ?? 300; applyRolldownTreeshakeWorkaround(nitro); const pluginSource = options.plugins; diff --git a/packages/junior/src/vercel.ts b/packages/junior/src/vercel.ts index 4fcc62b04..1a2ebb9a0 100644 --- a/packages/junior/src/vercel.ts +++ b/packages/junior/src/vercel.ts @@ -1,3 +1,5 @@ +import { DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC } from "@/chat/task-execution/vercel-queue"; + export interface JuniorVercelConfigOptions { buildCommand?: string | null; } @@ -15,6 +17,17 @@ export function juniorVercelConfig(options: JuniorVercelConfigOptions = {}) { schedule: "* * * * *", }, ], + functions: { + "server.ts": { + maxDuration: 300, + experimentalTriggers: [ + { + type: "queue/v2beta", + topic: DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC, + }, + ], + }, + }, }; if (buildCommand !== null) { diff --git a/packages/junior/tests/integration/conversation-work.test.ts b/packages/junior/tests/integration/conversation-work.test.ts new file mode 100644 index 000000000..9c1720dc8 --- /dev/null +++ b/packages/junior/tests/integration/conversation-work.test.ts @@ -0,0 +1,702 @@ +import { createHmac } from "node:crypto"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Message, Thread } from "chat"; +import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; +import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; +import { runHeartbeat } from "@/chat/agent-dispatch/heartbeat"; +import { + appendAndEnqueueInboundMessage, + appendInboundMessage, + checkInConversationWork, + completeConversationWork, + CONVERSATION_WORK_LEASE_TTL_MS, + countPendingConversationMessages, + drainConversationMailbox, + getConversationWorkState, + startConversationWork, + type InboundMessageRecord, +} from "@/chat/task-execution/store"; +import { + CONVERSATION_WORK_DEFER_DELAY_MS, + processConversationWork, +} from "@/chat/task-execution/worker"; +import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; +import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; +import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; +import { createVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; +import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; +import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; + +vi.hoisted(() => { + process.env.JUNIOR_STATE_ADAPTER = "memory"; +}); + +const CONVERSATION_ID = "slack:C123:1712345.0001"; +const SLACK_SIGNING_SECRET = "test-signing-secret"; +const SLACK_BOT_USER_ID = "U_BOT"; + +class FakeQueue implements ConversationWorkQueue { + fail = false; + sent: Array<{ + conversationId: string; + delayMs?: number; + idempotencyKey?: string; + }> = []; + + async send( + message: { conversationId: string }, + options?: { delayMs?: number; idempotencyKey?: string }, + ): Promise<{ messageId: string }> { + if (this.fail) { + throw new Error("queue unavailable"); + } + this.sent.push({ + conversationId: message.conversationId, + delayMs: options?.delayMs, + idempotencyKey: options?.idempotencyKey, + }); + return { messageId: `queue-${this.sent.length}` }; + } +} + +function inboundMessage( + inboundMessageId: string, + overrides: Partial = {}, +): InboundMessageRecord { + return { + conversationId: CONVERSATION_ID, + inboundMessageId, + source: "slack", + createdAtMs: 1_000, + receivedAtMs: 1_100, + input: { + text: `message ${inboundMessageId}`, + authorId: "U123", + }, + ...overrides, + }; +} + +function signSlackBody(body: string, timestamp: string): string { + return `v0=${createHmac("sha256", SLACK_SIGNING_SECRET) + .update(`v0:${timestamp}:${body}`) + .digest("hex")}`; +} + +function slackWebhookRequest(body: unknown): Request { + const serialized = JSON.stringify(body); + const timestamp = String(Math.floor(Date.now() / 1000)); + return new Request("https://example.test/api/webhooks/slack", { + method: "POST", + headers: { + "content-type": "application/json", + "x-slack-request-timestamp": timestamp, + "x-slack-signature": signSlackBody(serialized, timestamp), + }, + body: serialized, + }); +} + +function slackEnvelope(input: { + channel?: string; + eventType?: "app_mention" | "message"; + text?: string; + threadTs?: string; + ts?: string; +}) { + const channel = input.channel ?? "C123"; + const ts = input.ts ?? "1712345.0001"; + return { + team_id: "T123", + type: "event_callback", + event: { + type: input.eventType ?? "app_mention", + user: "U123", + text: input.text ?? `<@${SLACK_BOT_USER_ID}> hello`, + channel, + ts, + event_ts: ts, + channel_type: channel.startsWith("D") ? "im" : "channel", + ...(input.threadTs ? { thread_ts: input.threadTs } : {}), + }, + }; +} + +function deferred(): { + promise: Promise; + reject(error: unknown): void; + resolve(value: T): void; +} { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +describe("conversation work execution", () => { + beforeEach(async () => { + await disconnectStateAdapter(); + }); + + afterEach(async () => { + await disconnectStateAdapter(); + vi.useRealTimers(); + }); + + it("stores inbound mailbox messages idempotently", async () => { + const queue = new FakeQueue(); + await expect( + appendAndEnqueueInboundMessage({ + message: inboundMessage("m1"), + nowMs: 2_000, + queue, + }), + ).resolves.toMatchObject({ status: "appended", queueMessageId: "queue-1" }); + await expect( + appendAndEnqueueInboundMessage({ + message: inboundMessage("m1"), + nowMs: 3_000, + queue, + }), + ).resolves.toMatchObject({ + status: "duplicate", + queueMessageId: "queue-2", + }); + + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(state?.messages).toHaveLength(1); + expect(state ? countPendingConversationMessages(state) : 0).toBe(1); + expect(queue.sent).toHaveLength(2); + }); + + it("repairs pending mailbox work when the initial queue send fails", async () => { + const queue = new FakeQueue(); + queue.fail = true; + await expect( + appendAndEnqueueInboundMessage({ + message: inboundMessage("m1"), + nowMs: 2_000, + queue, + }), + ).rejects.toThrow("queue unavailable"); + + queue.fail = false; + await expect( + recoverConversationWork({ + nowMs: 62_000, + queue, + }), + ).resolves.toEqual({ expiredLeaseCount: 0, pendingCount: 1 }); + expect(queue.sent).toEqual([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `heartbeat:pending:${CONVERSATION_ID}`, + }, + ]); + }); + + it("defers duplicate queue nudges while a conversation lease is active", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + const entered = deferred(); + const finish = deferred(); + let runs = 0; + + const first = processConversationWork(CONVERSATION_ID, { + queue, + run: async (context) => { + runs += 1; + await context.drainMailbox(async () => {}); + entered.resolve(); + await finish.promise; + return { status: "completed" }; + }, + }); + await entered.promise; + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + run: async () => { + runs += 1; + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "active" }); + expect(runs).toBe(1); + expect(queue.sent).toMatchObject([ + { + conversationId: CONVERSATION_ID, + delayMs: CONVERSATION_WORK_DEFER_DELAY_MS, + }, + ]); + + finish.resolve(); + await expect(first).resolves.toEqual({ status: "completed" }); + }); + + it("drains pending messages and completes the leased conversation", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + const injected: InboundMessageRecord[][] = []; + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + run: async (context) => { + injected.push(await context.drainMailbox(async () => {})); + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "completed" }); + + expect(injected).toEqual([ + [expect.objectContaining({ inboundMessageId: "m1" })], + ]); + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(state?.lease).toBeUndefined(); + expect(state?.needsRun).toBe(false); + expect(state ? countPendingConversationMessages(state) : 0).toBe(0); + }); + + it("extends the lease with worker check-ins during long execution", async () => { + vi.useFakeTimers({ now: 1_000 }); + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + const entered = deferred(); + const finish = deferred(); + + const running = processConversationWork(CONVERSATION_ID, { + checkInIntervalMs: 15_000, + queue, + run: async (context) => { + await context.drainMailbox(async () => {}); + entered.resolve(); + await finish.promise; + return { status: "completed" }; + }, + }); + await entered.promise; + const before = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + + await vi.advanceTimersByTimeAsync(15_000); + const after = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + + expect(before?.lease?.leaseExpiresAtMs).toBe( + 1_000 + CONVERSATION_WORK_LEASE_TTL_MS, + ); + expect(after?.lease?.leaseExpiresAtMs).toBe( + 16_000 + CONVERSATION_WORK_LEASE_TTL_MS, + ); + + finish.resolve(); + await expect(running).resolves.toEqual({ status: "completed" }); + }); + + it("requeues an expired conversation lease from heartbeat", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + await expect( + startConversationWork({ conversationId: CONVERSATION_ID, nowMs: 2_000 }), + ).resolves.toMatchObject({ status: "acquired" }); + + await expect( + recoverConversationWork({ + nowMs: 2_000 + CONVERSATION_WORK_LEASE_TTL_MS, + queue, + }), + ).resolves.toEqual({ expiredLeaseCount: 1, pendingCount: 0 }); + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(state?.lease).toBeUndefined(); + expect(queue.sent).toMatchObject([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `heartbeat:lease:${CONVERSATION_ID}`, + }, + ]); + }); + + it("requeues pending mailbox work with no recent queue marker", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + recoverConversationWork({ + nowMs: 62_000, + queue, + }), + ).resolves.toEqual({ expiredLeaseCount: 0, pendingCount: 1 }); + expect(queue.sent).toHaveLength(1); + }); + + it("runs conversation work recovery from the core heartbeat", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await runHeartbeat({ + nowMs: 62_000, + conversationWorkQueue: queue, + }); + + expect(queue.sent).toEqual([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `heartbeat:pending:${CONVERSATION_ID}`, + }, + ]); + }); + + it("injects messages that arrive during active execution at a safe boundary", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + const injected: string[][] = []; + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + run: async (context) => { + const first = await context.drainMailbox(async () => {}); + injected.push(first.map((message) => message.inboundMessageId)); + await appendInboundMessage({ + message: inboundMessage("m2", { + createdAtMs: 2_000, + receivedAtMs: 2_100, + }), + nowMs: 2_100, + }); + const second = await context.drainMailbox(async () => {}); + injected.push(second.map((message) => message.inboundMessageId)); + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "completed" }); + + expect(injected).toEqual([["m1"], ["m2"]]); + }); + + it("requeues instead of completing when final mailbox work remains", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + run: async (context) => { + await context.drainMailbox(async () => {}); + await appendInboundMessage({ + message: inboundMessage("m2", { + createdAtMs: 2_000, + receivedAtMs: 2_100, + }), + nowMs: 2_100, + }); + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "pending_requeued" }); + expect(queue.sent).toMatchObject([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `pending:${CONVERSATION_ID}`, + }, + ]); + }); + + it("yields cooperatively and leaves the conversation resumable", async () => { + const queue = new FakeQueue(); + let currentNowMs = 1_000; + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, + queue, + run: async (context) => { + await context.drainMailbox(async () => {}); + currentNowMs = 242_000; + expect(context.shouldYield()).toBe(true); + return { status: "yielded" }; + }, + }), + ).resolves.toEqual({ status: "yielded" }); + + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(state?.lease).toBeUndefined(); + expect(state?.needsRun).toBe(true); + expect(queue.sent).toMatchObject([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `yield:${CONVERSATION_ID}`, + }, + ]); + }); + + it("keeps lease mutations token-bound", async () => { + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + const lease = await startConversationWork({ + conversationId: CONVERSATION_ID, + nowMs: 2_000, + }); + expect(lease.status).toBe("acquired"); + if (lease.status !== "acquired") { + return; + } + + await expect( + checkInConversationWork({ + conversationId: CONVERSATION_ID, + leaseToken: "wrong-token", + nowMs: 3_000, + }), + ).resolves.toBe(false); + await expect( + drainConversationMailbox({ + conversationId: CONVERSATION_ID, + leaseToken: "wrong-token", + inject: async () => {}, + nowMs: 3_000, + }), + ).rejects.toThrow("lease is not held"); + await expect( + completeConversationWork({ + conversationId: CONVERSATION_ID, + leaseToken: "wrong-token", + nowMs: 3_000, + }), + ).resolves.toBe("lost_lease"); + }); + + it("maps the generic queue port to Vercel Queue send options", async () => { + const sends: Array<{ + message: unknown; + options: unknown; + topic: string; + }> = []; + const queue = createVercelConversationWorkQueue({ + topic: "junior_test_work", + client: { + async send(topic, message, options) { + sends.push({ topic, message, options }); + return { messageId: "msg_123" }; + }, + }, + }); + + await expect( + queue.send( + { conversationId: CONVERSATION_ID }, + { delayMs: 15_001, idempotencyKey: "idem-1" }, + ), + ).resolves.toEqual({ messageId: "msg_123" }); + + expect(sends).toEqual([ + { + topic: "junior_test_work", + message: { conversationId: CONVERSATION_ID }, + options: { + delaySeconds: 16, + idempotencyKey: "idem-1", + retentionSeconds: undefined, + }, + }, + ]); + }); + + it("processes Vercel Queue payloads through the leased worker", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + const injected: string[] = []; + + await expect( + processConversationQueueMessage( + { conversationId: CONVERSATION_ID }, + { + queue, + run: async (context) => { + const messages = await context.drainMailbox(async () => {}); + injected.push( + ...messages.map((message) => message.inboundMessageId), + ); + return { status: "completed" }; + }, + }, + ), + ).resolves.toEqual({ status: "completed" }); + + expect(injected).toEqual(["m1"]); + }); + + it("persists Slack mentions into the durable mailbox and wakes the queue", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + + const response = await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> deploy status`, + }), + ), + waitUntil: () => {}, + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }, + }); + + expect(response.status).toBe(200); + expect(queue.sent).toEqual([ + expect.objectContaining({ + conversationId: CONVERSATION_ID, + }), + ]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.needsRun).toBe(true); + expect(work?.messages).toEqual([ + expect.objectContaining({ + conversationId: CONVERSATION_ID, + source: "slack", + input: expect.objectContaining({ + authorId: "U123", + metadata: expect.objectContaining({ + platform: "slack", + route: "mention", + }), + }), + }), + ]); + }); + + it("runs queued Slack mailbox work through the Slack runtime", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + const calls: Array<{ + message: Message; + skipped: Message[]; + thread: Thread; + }> = []; + + await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + ts: "1712345.0001", + }), + ), + waitUntil: () => {}, + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }, + }); + await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> second`, + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + waitUntil: () => {}, + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }, + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (thread, message, hooks) => { + calls.push({ + thread, + message, + skipped: hooks?.messageContext?.skipped ?? [], + }); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.thread.id).toBe(CONVERSATION_ID); + expect(calls[0]?.message.id).toBe("1712345.0002"); + expect(calls[0]?.message.text).toContain("second"); + expect(calls[0]?.skipped.map((message) => message.id)).toEqual([ + "1712345.0001", + ]); + }); + + it("rejects malformed Vercel Queue payloads", async () => { + const queue = new FakeQueue(); + + await expect( + processConversationQueueMessage( + { wrong: CONVERSATION_ID }, + { + queue, + run: async () => ({ status: "completed" }), + }, + ), + ).rejects.toThrow("missing conversationId"); + }); +}); diff --git a/packages/junior/tests/integration/heartbeat.test.ts b/packages/junior/tests/integration/heartbeat.test.ts index 70e4cddb9..1bf3c92df 100644 --- a/packages/junior/tests/integration/heartbeat.test.ts +++ b/packages/junior/tests/integration/heartbeat.test.ts @@ -18,8 +18,12 @@ import { } from "@/chat/agent-dispatch/store"; import type { DispatchRecord } from "@/chat/agent-dispatch/types"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { getConversationWorkState } from "@/chat/task-execution/store"; +import type { PiMessage } from "@/chat/pi/messages"; import { setAgentPlugins } from "@/chat/plugins/agent-hooks"; import { GET as heartbeat } from "@/handlers/heartbeat"; +import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import type { WaitUntilFn } from "@/handlers/types"; import { createSlackDirectCredentialSubject } from "@/chat/credentials/subject"; import { getCapturedSlackApiCalls } from "../msw/handlers/slack-api"; @@ -37,6 +41,21 @@ function collectWaitUntil(tasks: Promise[]): WaitUntilFn { }; } +class FakeConversationWorkQueue implements ConversationWorkQueue { + sent: Array<{ conversationId: string; idempotencyKey?: string }> = []; + + async send( + message: { conversationId: string }, + options?: { idempotencyKey?: string }, + ): Promise<{ messageId: string }> { + this.sent.push({ + conversationId: message.conversationId, + idempotencyKey: options?.idempotencyKey, + }); + return { messageId: `queue-${this.sent.length}` }; + } +} + function schedulerStore() { return createSchedulerStore(createPluginState("scheduler")); } @@ -194,6 +213,55 @@ describe("trusted plugin heartbeat", () => { expect(seen).toHaveLength(1); }); + it("reschedules stale timeout resume records", async () => { + const queue = new FakeConversationWorkQueue(); + const conversationId = "slack:C123:1712345.0001"; + const sessionId = "turn-timeout"; + const staleNowMs = TEST_NOW_MS - 3 * 60 * 1000; + vi.setSystemTime(staleNowMs); + await upsertAgentTurnSessionRecord({ + conversationId, + sessionId, + sliceId: 2, + state: "awaiting_resume", + resumeReason: "timeout", + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "finish this" }], + timestamp: staleNowMs, + } as PiMessage, + ], + }); + vi.setSystemTime(TEST_NOW_MS); + + const waitUntilTasks: Promise[] = []; + const response = await heartbeat( + new Request("https://example.invalid/api/internal/heartbeat", { + headers: { authorization: "Bearer heartbeat-secret" }, + }), + collectWaitUntil(waitUntilTasks), + { conversationWorkQueue: queue }, + ); + + expect(response.status).toBe(202); + await Promise.all(waitUntilTasks); + expect(queue.sent).toEqual([ + { + conversationId, + idempotencyKey: expect.stringContaining( + `timeout:${conversationId}:${sessionId}:`, + ), + }, + ]); + await expect( + getConversationWorkState({ conversationId }), + ).resolves.toMatchObject({ + conversationId, + needsRun: true, + }); + }); + it("scopes dispatch lookup to the plugin that created it", async () => { const fetchMock = vi.fn(async () => { return new Response("Accepted", { status: 202 }); diff --git a/packages/junior/tests/integration/slack/bot-handlers.test.ts b/packages/junior/tests/integration/slack/bot-handlers.test.ts index 2ed901f47..9e0267352 100644 --- a/packages/junior/tests/integration/slack/bot-handlers.test.ts +++ b/packages/junior/tests/integration/slack/bot-handlers.test.ts @@ -3,7 +3,6 @@ import type { JuniorRuntimeServiceOverrides } from "@/chat/app/services"; import { makeAssistantStatus } from "@/chat/slack/assistant-thread/status"; import { getSlackInterruptionMarker } from "@/chat/slack/output"; import { RetryableTurnError } from "@/chat/runtime/turn"; -import { buildTurnContinuationResponse } from "@/chat/services/turn-continuation-response"; import { getCapturedSlackApiCalls, resetSlackApiMockState, @@ -594,7 +593,7 @@ describe("bot handlers (integration)", () => { }); }); - it("posts a durable notice when a turn is scheduled for continuation", async () => { + it("schedules durable continuation without posting a notice", async () => { const scheduleTurnTimeoutResume = vi.fn().mockResolvedValue(undefined); const conversationId = "slack:C_TIMEOUT:1700000000.000"; const sessionId = "turn_msg-timeout"; @@ -636,11 +635,7 @@ describe("bot handlers (integration)", () => { sessionId, expectedVersion: 3, }); - expect(thread.posts).toEqual([ - expect.objectContaining({ - markdown: buildTurnContinuationResponse(), - }), - ]); + expect(thread.posts).toEqual([]); const state = thread.getState(); const conversation = ( @@ -653,7 +648,7 @@ describe("bot handlers (integration)", () => { expect(conversation?.processing?.activeTurnId).toBe(sessionId); }); - it("posts a Slack continuation notice with a correlation footer when a live turn times out", async () => { + it("does not post a Slack continuation notice when a live turn times out", async () => { resetSlackApiMockState(); const scheduleTurnTimeoutResume = vi.fn().mockResolvedValue(undefined); const conversationId = "slack:C_TIMEOUT_API:1700000000.000"; @@ -699,30 +694,7 @@ describe("bot handlers (integration)", () => { expectedVersion: 3, }); expect(thread.posts).toEqual([]); - expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ - expect.objectContaining({ - params: expect.objectContaining({ - channel: "C_TIMEOUT_API", - thread_ts: "1700000000.000", - text: buildTurnContinuationResponse(), - blocks: [ - { - type: "markdown", - text: buildTurnContinuationResponse(), - }, - { - type: "context", - elements: [ - { - type: "mrkdwn", - text: `*ID:* ${conversationId}`, - }, - ], - }, - ], - }), - }), - ]); + expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([]); }); it("reschedules an awaiting turn continuation instead of starting a new turn", async () => { @@ -772,11 +744,7 @@ describe("bot handlers (integration)", () => { expectedVersion: 4, }); expect(generateAssistantReply).not.toHaveBeenCalled(); - expect(thread.posts).toEqual([ - expect.objectContaining({ - markdown: buildTurnContinuationResponse(), - }), - ]); + expect(thread.posts).toEqual([]); const state = thread.getState(); const conversation = ( @@ -847,7 +815,7 @@ describe("bot handlers (integration)", () => { expect(generateAssistantReply).not.toHaveBeenCalled(); }); - it("does not silently complete when continuation acknowledgement fails", async () => { + it("keeps awaiting continuation state without a visible acknowledgement", async () => { const conversationId = "slack:C_TIMEOUT_NOTICE_FAIL:1700000000.000"; const activeSessionId = "turn_msg-original"; const scheduleTurnTimeoutResume = vi.fn().mockResolvedValue(undefined); @@ -871,9 +839,6 @@ describe("bot handlers (integration)", () => { id: conversationId, state: createAwaitingContinuationState({ activeSessionId }), }); - vi.spyOn(thread, "post").mockRejectedValueOnce( - new Error("slack unavailable"), - ); await slackRuntime.handleNewMention( thread, @@ -891,11 +856,7 @@ describe("bot handlers (integration)", () => { expectedVersion: 4, }); expect(generateAssistantReply).not.toHaveBeenCalled(); - expect(thread.posts).toEqual([ - expect.stringContaining( - "I ran into an internal error while processing that.", - ), - ]); + expect(thread.posts).toEqual([]); const state = thread.getState(); const conversation = ( diff --git a/packages/junior/tests/integration/turn-resume-slack.test.ts b/packages/junior/tests/integration/turn-resume-slack.test.ts index 689086d11..ca2b3bdf1 100644 --- a/packages/junior/tests/integration/turn-resume-slack.test.ts +++ b/packages/junior/tests/integration/turn-resume-slack.test.ts @@ -1,21 +1,41 @@ import { Buffer } from "node:buffer"; +import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { WaitUntilFn } from "@/handlers/types"; -import { buildTurnContinuationResponse } from "@/chat/services/turn-continuation-response"; import { getCapturedSlackApiCalls, getCapturedSlackFileUploadCalls, resetSlackApiMockState, } from "../msw/handlers/slack-api"; -const { generateAssistantReplyMock } = vi.hoisted(() => ({ +const { generateAssistantReplyMock, queueSends } = vi.hoisted(() => ({ generateAssistantReplyMock: vi.fn(), + queueSends: [] as Array<{ + conversationId: string; + idempotencyKey?: string; + }>, })); vi.mock("@/chat/respond", () => ({ generateAssistantReply: generateAssistantReplyMock, })); +vi.mock("@/chat/task-execution/vercel-queue", () => ({ + DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC: "junior_conversation_work", + getVercelConversationWorkQueue: () => ({ + send: async ( + message: { conversationId: string }, + options?: { idempotencyKey?: string }, + ) => { + queueSends.push({ + conversationId: message.conversationId, + idempotencyKey: options?.idempotencyKey, + }); + return { messageId: `queue-${queueSends.length}` }; + }, + }), +})); + const ORIGINAL_ENV = { ...process.env }; type StateAdapterModule = typeof import("@/chat/state/adapter"); @@ -39,35 +59,26 @@ async function buildSignedTurnResumeRequest(args: { sessionId: string; expectedVersion: number; }): Promise { - const originalFetch = global.fetch; - const fetchMock = vi.fn( - async () => new Response("Accepted", { status: 202 }), - ); - global.fetch = fetchMock as typeof fetch; - - try { - const { scheduleTurnTimeoutResume } = - await import("@/chat/services/timeout-resume"); - await scheduleTurnTimeoutResume(args); - } finally { - global.fetch = originalFetch; - } - - const firstCall = fetchMock.mock.calls[0]; - if (!firstCall) { - throw new Error("Expected scheduleTurnTimeoutResume to issue one fetch"); - } - const [url, init] = firstCall as unknown as [string, RequestInit]; - return new Request(url, { - method: init.method, - headers: init.headers, - body: init.body, + const timestamp = Date.now().toString(); + const body = JSON.stringify(args); + const digest = createHmac("sha256", "resume-secret") + .update(`junior.turn_timeout_resume.v1:${timestamp}:${body}`) + .digest("hex"); + return new Request("https://junior.example.com/api/internal/turn-resume", { + method: "POST", + headers: { + "content-type": "application/json", + "x-junior-resume-timestamp": timestamp, + "x-junior-resume-signature": `v1=${digest}`, + }, + body, }); } describe("turn resume slack integration", () => { beforeEach(async () => { waitUntilCallbacks.length = 0; + queueSends.length = 0; generateAssistantReplyMock.mockReset(); generateAssistantReplyMock.mockResolvedValue({ text: "Final resumed answer", @@ -250,7 +261,7 @@ describe("turn resume slack integration", () => { }); }); - it("posts the failure message when timeout resume depth is exhausted", async () => { + it("schedules another continuation for high timeout resume slice ids", async () => { const conversationId = "slack:C123:1712345.0002"; const sessionId = "turn_msg_2"; const sessionRecord = @@ -330,16 +341,14 @@ describe("turn resume slack integration", () => { await waitUntilCallbacks[0]?.(); - expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ - expect.objectContaining({ - params: expect.objectContaining({ - channel: "C123", - thread_ts: "1712345.0002", - text: expect.stringContaining( - "I ran into an internal error while processing that. Reference: `event_id=", - ), - }), - }), + expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([]); + expect(queueSends).toEqual([ + { + conversationId, + idempotencyKey: expect.stringContaining( + `timeout:${conversationId}:${sessionId}:`, + ), + }, ]); const persisted = @@ -347,10 +356,10 @@ describe("turn resume slack integration", () => { const conversation = (persisted.conversation ?? {}) as { processing?: { activeTurnId?: string }; }; - expect(conversation.processing?.activeTurnId).toBeUndefined(); + expect(conversation.processing?.activeTurnId).toBe(sessionId); }); - it("posts a continuation notice with a correlation footer when a resumed slice times out again", async () => { + it("schedules a durable continuation without posting a notice when a resumed slice times out again", async () => { const conversationId = "slack:C123:1712345.0006"; const sessionId = "turn_msg_6"; const sessionRecord = @@ -428,43 +437,18 @@ describe("turn resume slack integration", () => { expect(response.status).toBe(202); expect(waitUntilCallbacks).toHaveLength(1); - const originalFetch = global.fetch; - const fetchMock = vi.fn( - async () => new Response("Accepted", { status: 202 }), - ); - global.fetch = fetchMock as typeof fetch; - try { - await waitUntilCallbacks[0]?.(); - } finally { - global.fetch = originalFetch; - } + await waitUntilCallbacks[0]?.(); const postCalls = getCapturedSlackApiCalls("chat.postMessage"); - expect(postCalls).toEqual([ - expect.objectContaining({ - params: expect.objectContaining({ - channel: "C123", - thread_ts: "1712345.0006", - text: buildTurnContinuationResponse(), - blocks: [ - { - type: "markdown", - text: buildTurnContinuationResponse(), - }, - { - type: "context", - elements: [ - { - type: "mrkdwn", - text: `*ID:* ${conversationId}`, - }, - ], - }, - ], - }), - }), + expect(postCalls).toEqual([]); + expect(queueSends).toEqual([ + { + conversationId, + idempotencyKey: expect.stringContaining( + `timeout:${conversationId}:${sessionId}:`, + ), + }, ]); - expect(fetchMock).toHaveBeenCalledOnce(); }); it("uploads resumed reply files through the shared delivery path", async () => { diff --git a/packages/junior/tests/unit/cli/init-cli.test.ts b/packages/junior/tests/unit/cli/init-cli.test.ts index ae4a469ab..80a58a96a 100644 --- a/packages/junior/tests/unit/cli/init-cli.test.ts +++ b/packages/junior/tests/unit/cli/init-cli.test.ts @@ -57,7 +57,17 @@ describe("init cli", () => { schedule: "* * * * *", }, ]); - expect(vercelConfig.functions).toBeUndefined(); + expect(vercelConfig.functions).toEqual({ + "server.ts": { + maxDuration: 300, + experimentalTriggers: [ + { + type: "queue/v2beta", + topic: "junior_conversation_work", + }, + ], + }, + }); const pkg = JSON.parse( fs.readFileSync(path.join(target, "package.json"), "utf8"), diff --git a/packages/junior/tests/unit/config/chat-config.test.ts b/packages/junior/tests/unit/config/chat-config.test.ts index 050d72931..38026f627 100644 --- a/packages/junior/tests/unit/config/chat-config.test.ts +++ b/packages/junior/tests/unit/config/chat-config.test.ts @@ -158,25 +158,25 @@ describe("chat config", () => { it("uses default AGENT_TURN_TIMEOUT_MS when env var is unset", async () => { delete process.env.AGENT_TURN_TIMEOUT_MS; const { botConfig } = await loadConfig(); - expect(botConfig.turnTimeoutMs).toBe(720000); + expect(botConfig.turnTimeoutMs).toBe(280000); }); it("uses AGENT_TURN_TIMEOUT_MS from env var when valid", async () => { - process.env.AGENT_TURN_TIMEOUT_MS = "600000"; + process.env.AGENT_TURN_TIMEOUT_MS = "240000"; const { botConfig } = await loadConfig(); - expect(botConfig.turnTimeoutMs).toBe(600000); + expect(botConfig.turnTimeoutMs).toBe(240000); }); it("falls back to default AGENT_TURN_TIMEOUT_MS when env var is invalid", async () => { process.env.AGENT_TURN_TIMEOUT_MS = "not-a-number"; const { botConfig } = await loadConfig(); - expect(botConfig.turnTimeoutMs).toBe(720000); + expect(botConfig.turnTimeoutMs).toBe(280000); }); it("caps AGENT_TURN_TIMEOUT_MS to configured max", async () => { process.env.AGENT_TURN_TIMEOUT_MS = "999999"; const { botConfig } = await loadConfig(); - expect(botConfig.turnTimeoutMs).toBe(780000); + expect(botConfig.turnTimeoutMs).toBe(280000); }); it("derives AGENT_TURN_TIMEOUT_MS cap from FUNCTION_MAX_DURATION_SECONDS", async () => { diff --git a/packages/junior/tests/unit/config/turn-timeout-matrix.test.ts b/packages/junior/tests/unit/config/turn-timeout-matrix.test.ts index fc9a04023..9731fc24f 100644 --- a/packages/junior/tests/unit/config/turn-timeout-matrix.test.ts +++ b/packages/junior/tests/unit/config/turn-timeout-matrix.test.ts @@ -22,29 +22,29 @@ describe("turnTimeoutMs decision matrix", () => { turnMs: undefined, funcMax: undefined, queueMax: undefined, - expected: 720000, + expected: 280000, }, { label: "explicit turn timeout", - turnMs: "600000", + turnMs: "240000", funcMax: undefined, queueMax: undefined, - expected: 600000, + expected: 240000, }, { label: "invalid turn timeout falls back to default", turnMs: "not-a-number", funcMax: undefined, queueMax: undefined, - expected: 720000, + expected: 280000, }, { label: - "turn timeout capped by default function max (800s - 20s buffer = 780s)", + "turn timeout capped by default function max (300s - 20s buffer = 280s)", turnMs: "999999", funcMax: undefined, queueMax: undefined, - expected: 780000, + expected: 280000, }, { label: "turn timeout capped by FUNCTION_MAX_DURATION_SECONDS", diff --git a/packages/junior/tests/unit/handlers/oauth-resume.test.ts b/packages/junior/tests/unit/handlers/oauth-resume.test.ts index 53b4ed559..3a67c6404 100644 --- a/packages/junior/tests/unit/handlers/oauth-resume.test.ts +++ b/packages/junior/tests/unit/handlers/oauth-resume.test.ts @@ -240,12 +240,7 @@ describe("resumeAuthorizedRequest", () => { }); expect(onTimeoutPause).toHaveBeenCalledTimes(1); - expect(postMessageMock).toHaveBeenCalledWith( - expect.objectContaining({ - channel: "C-test", - thread_ts: "1700000000.0002", - }), - ); + expect(postMessageMock).not.toHaveBeenCalled(); }); it("posts the canonical failure response when timeout pause handling throws", async () => { @@ -270,7 +265,7 @@ describe("resumeAuthorizedRequest", () => { }); }, onTimeoutPause: async () => { - throw new Error("slice limit reached"); + throw new Error("continuation scheduling failed"); }, onFailure, }); diff --git a/packages/junior/tests/unit/handlers/turn-resume.test.ts b/packages/junior/tests/unit/handlers/turn-resume.test.ts index 3db5eeea4..457a3cef6 100644 --- a/packages/junior/tests/unit/handlers/turn-resume.test.ts +++ b/packages/junior/tests/unit/handlers/turn-resume.test.ts @@ -533,7 +533,7 @@ describe("turn resume handler", () => { expect(conversation.messages).toHaveLength(1); }); - it("persists timeout-resume failure state when terminalization fails", async () => { + it("persists timeout-resume failure state when continuation scheduling fails", async () => { const conversationId = "slack:C123:1712345.0001"; const sessionId = "turn_msg_1"; const sessionRecord = await upsertAgentTurnSessionRecord({ @@ -593,6 +593,9 @@ describe("turn resume handler", () => { sessionId, expectedVersion: sessionRecord.version, }); + scheduleTurnTimeoutResumeMock.mockRejectedValueOnce( + new Error("queue unavailable"), + ); resumeSlackTurnMock.mockImplementationOnce(async (args) => { const prepared = await args.beforeStart?.(); @@ -630,7 +633,11 @@ describe("turn resume handler", () => { expect(response.status).toBe(202); await waitUntilCallbacks[0]?.(); - expect(scheduleTurnTimeoutResumeMock).not.toHaveBeenCalled(); + expect(scheduleTurnTimeoutResumeMock).toHaveBeenCalledWith({ + conversationId, + sessionId, + expectedVersion: sessionRecord.version + 1, + }); const persisted = await getPersistedThreadState(conversationId); const conversation = (persisted.conversation ?? {}) as { diff --git a/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts b/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts index 0fe434345..fe2b442eb 100644 --- a/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts +++ b/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts @@ -4,7 +4,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const { promptAborted, promptMode } = vi.hoisted(() => ({ promptAborted: { value: false }, promptMode: { - value: "settlesAfterAbort" as "settlesAfterAbort" | "hangsAfterAbort", + value: "settlesAfterAbort" as + | "settlesAfterAbort" + | "hangsAfterAbort" + | "providerRetryThenHangs", }, })); @@ -43,6 +46,18 @@ vi.mock("@earendil-works/pi-agent-core", () => { } async continue() { + if (promptMode.value === "providerRetryThenHangs") { + await new Promise((resolve) => { + this.resolveAbort = resolve; + }); + this.state.messages.push({ + role: "assistant", + content: [{ type: "text", text: "continued partial" }], + stopReason: "stop", + }); + return {}; + } + this.state.messages.push({ role: "assistant", content: [{ type: "text", text: "continued" }], @@ -53,6 +68,16 @@ vi.mock("@earendil-works/pi-agent-core", () => { async prompt(message: unknown) { this.state.messages.push(message); + if (promptMode.value === "providerRetryThenHangs") { + await new Promise((resolve) => setTimeout(resolve, 8_000)); + this.state.messages.push({ + role: "assistant", + content: [{ type: "text", text: "provider error" }], + stopReason: "error", + errorMessage: "Provider returned error: 503 service unavailable", + }); + return {}; + } if (promptMode.value === "hangsAfterAbort") { await new Promise(() => undefined); return {}; @@ -298,4 +323,38 @@ describe("generateAssistantReply timeout resume", () => { }), ]); }); + + it("uses one wall-clock timeout budget across provider retries", async () => { + promptMode.value = "providerRetryThenHangs"; + const replyPromise = generateAssistantReply("help me", { + requester: { userId: "U123" }, + correlation: { + conversationId: "conversation-retry", + turnId: "turn-retry", + channelId: "C123", + threadTs: "1712345.0004", + }, + }).catch((caught) => caught); + + await vi.advanceTimersByTimeAsync(10_000); + const error = await replyPromise; + + expect(promptAborted.value).toBe(true); + expect(isRetryableTurnError(error, "turn_timeout_resume")).toBe(true); + const sessionRecord = await getAgentTurnSessionRecord( + "conversation-retry", + "turn-retry", + ); + expect(sessionRecord).toMatchObject({ + state: "awaiting_resume", + resumeReason: "timeout", + resumedFromSliceId: 1, + sliceId: 2, + }); + expect(sessionRecord?.piMessages).toEqual([ + expect.objectContaining({ + role: "user", + }), + ]); + }); }); diff --git a/packages/junior/tests/unit/runtime/timeout-resume.test.ts b/packages/junior/tests/unit/runtime/timeout-resume.test.ts index 15ea8bd45..6b540baf3 100644 --- a/packages/junior/tests/unit/runtime/timeout-resume.test.ts +++ b/packages/junior/tests/unit/runtime/timeout-resume.test.ts @@ -1,10 +1,32 @@ import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { - canScheduleTurnTimeoutResume, scheduleTurnTimeoutResume, verifyTurnTimeoutResumeRequest, } from "@/chat/services/timeout-resume"; +import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; +import { getConversationWorkState } from "@/chat/task-execution/store"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; + +class FakeQueue implements ConversationWorkQueue { + sent: Array<{ + conversationId: string; + delayMs?: number; + idempotencyKey?: string; + }> = []; + + async send( + message: { conversationId: string }, + options?: { delayMs?: number; idempotencyKey?: string }, + ): Promise<{ messageId: string }> { + this.sent.push({ + conversationId: message.conversationId, + delayMs: options?.delayMs, + idempotencyKey: options?.idempotencyKey, + }); + return { messageId: `queue-${this.sent.length}` }; + } +} function makeSignedResumeRequest(body: Record): Request { const timestamp = Date.now().toString(); @@ -24,17 +46,17 @@ function makeSignedResumeRequest(body: Record): Request { } describe("timeout resume callback signing", () => { - const originalFetch = global.fetch; const originalSlackSigningSecret = process.env.SLACK_SIGNING_SECRET; - beforeEach(() => { - process.env.JUNIOR_BASE_URL = "https://junior.example.com"; + beforeEach(async () => { + process.env.JUNIOR_STATE_ADAPTER = "memory"; process.env.JUNIOR_SECRET = "resume-secret"; + await disconnectStateAdapter(); }); - afterEach(() => { - global.fetch = originalFetch; - delete process.env.JUNIOR_BASE_URL; + afterEach(async () => { + await disconnectStateAdapter(); + delete process.env.JUNIOR_STATE_ADAPTER; delete process.env.JUNIOR_SECRET; if (originalSlackSigningSecret === undefined) { delete process.env.SLACK_SIGNING_SECRET; @@ -44,27 +66,41 @@ describe("timeout resume callback signing", () => { vi.restoreAllMocks(); }); - it("signs scheduled callbacks so the handler can verify them", async () => { - const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => { - return new Response("Accepted", { status: 202 }); + it("marks timeout continuations runnable and wakes the durable queue", async () => { + const queue = new FakeQueue(); + const conversationId = "slack:C123:1712345.0001"; + + await scheduleTurnTimeoutResume( + { + conversationId, + sessionId: "turn_msg_1", + expectedVersion: 3, + }, + { queue, nowMs: 1_000 }, + ); + + expect(queue.sent).toEqual([ + { + conversationId, + idempotencyKey: `timeout:${conversationId}:turn_msg_1:3`, + }, + ]); + await expect( + getConversationWorkState({ conversationId }), + ).resolves.toMatchObject({ + conversationId, + needsRun: true, + lastEnqueuedAtMs: 1_000, }); - global.fetch = fetchMock as typeof fetch; + }); - await scheduleTurnTimeoutResume({ + it("still verifies signed callbacks that were already in flight", async () => { + const request = makeSignedResumeRequest({ conversationId: "slack:C123:1712345.0001", sessionId: "turn_msg_1", expectedVersion: 3, }); - expect(fetchMock).toHaveBeenCalledTimes(1); - const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; - expect(url).toBe("https://junior.example.com/api/internal/turn-resume"); - - const request = new Request(url, { - method: init.method, - headers: init.headers, - body: init.body, - }); await expect(verifyTurnTimeoutResumeRequest(request)).resolves.toEqual({ conversationId: "slack:C123:1712345.0001", sessionId: "turn_msg_1", @@ -87,53 +123,35 @@ describe("timeout resume callback signing", () => { }); it("rejects requests whose signature does not match the body", async () => { - const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => { - return new Response("Accepted", { status: 202 }); - }); - global.fetch = fetchMock as typeof fetch; - - await scheduleTurnTimeoutResume({ + const request = makeSignedResumeRequest({ conversationId: "slack:C123:1712345.0001", sessionId: "turn_msg_1", expectedVersion: 3, }); - - const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; - const headers = new Headers(init.headers); + const headers = new Headers(request.headers); headers.set("x-junior-resume-signature", "v1=deadbeef"); - const request = new Request(url, { - method: init.method, + const tampered = new Request(request.url, { + method: request.method, headers, - body: init.body, + body: await request.text(), }); await expect( - verifyTurnTimeoutResumeRequest(request), + verifyTurnTimeoutResumeRequest(tampered), ).resolves.toBeUndefined(); }); - it("requires the Junior secret instead of the Slack signing secret", async () => { + it("requires the Junior secret to verify legacy callbacks", async () => { + const request = makeSignedResumeRequest({ + conversationId: "slack:C123:1712345.0001", + sessionId: "turn_msg_1", + expectedVersion: 3, + }); delete process.env.JUNIOR_SECRET; process.env.SLACK_SIGNING_SECRET = "slack-secret"; - const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => { - return new Response("Accepted", { status: 202 }); - }); - global.fetch = fetchMock as typeof fetch; await expect( - scheduleTurnTimeoutResume({ - conversationId: "slack:C123:1712345.0001", - sessionId: "turn_msg_1", - expectedVersion: 3, - }), - ).rejects.toThrow("JUNIOR_SECRET"); - expect(fetchMock).not.toHaveBeenCalled(); - }); - - it("caps automatic timeout resume depth", () => { - expect(canScheduleTurnTimeoutResume(2)).toBe(true); - expect(canScheduleTurnTimeoutResume(5)).toBe(true); - expect(canScheduleTurnTimeoutResume(6)).toBe(false); - expect(canScheduleTurnTimeoutResume(undefined)).toBe(false); + verifyTurnTimeoutResumeRequest(request), + ).resolves.toBeUndefined(); }); }); diff --git a/packages/junior/tests/unit/vercel.test.ts b/packages/junior/tests/unit/vercel.test.ts index d6ff4c799..7f07d348f 100644 --- a/packages/junior/tests/unit/vercel.test.ts +++ b/packages/junior/tests/unit/vercel.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC } from "@/chat/task-execution/vercel-queue"; import { juniorVercelConfig } from "@/vercel"; describe("juniorVercelConfig", () => { @@ -13,6 +14,17 @@ describe("juniorVercelConfig", () => { schedule: "* * * * *", }, ]); + expect(config.functions).toEqual({ + "server.ts": { + maxDuration: 300, + experimentalTriggers: [ + { + type: "queue/v2beta", + topic: DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC, + }, + ], + }, + }); }); it("omits buildCommand when set to null", () => { diff --git a/packages/junior/tsup.config.ts b/packages/junior/tsup.config.ts index a0069f299..ee128de9e 100644 --- a/packages/junior/tsup.config.ts +++ b/packages/junior/tsup.config.ts @@ -33,6 +33,7 @@ export default defineConfig({ "@sinclair/typebox", "@slack/web-api", "@vercel/functions", + "@vercel/queue", "@vercel/sandbox", "ai", "bash-tool", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e484a8c55..b1e966471 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -159,6 +159,9 @@ importers: "@vercel/functions": specifier: ^3.6.0 version: 3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43) + "@vercel/queue": + specifier: ^0.2.0 + version: 0.2.0 "@vercel/sandbox": specifier: 2.0.0 version: 2.0.0 @@ -204,7 +207,7 @@ importers: version: 2.14.6(@types/node@25.9.1)(typescript@6.0.3) nitro: specifier: 3.0.260522-beta - version: 3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.7)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)) + version: 3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(@vercel/queue@0.2.0)(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.7)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)) oxlint: specifier: ^1.66.0 version: 1.66.0 @@ -245,7 +248,7 @@ importers: version: 1.17.0(react@19.2.6) nitro: specifier: 3.0.260522-beta - version: 3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)) + version: 3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(@vercel/queue@0.2.0)(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)) react: specifier: ^19.2.6 version: 19.2.6 @@ -4661,6 +4664,13 @@ packages: integrity: sha512-kAykqslePS5jRgBsIui8m4BK7/7+Iqztb7X88qeTDqg0eP/JRCxg0kxC7HkX9ZkW9lpQr6hn6CNzMAdKH8ihFg==, } + "@vercel/queue@0.2.0": + resolution: + { + integrity: sha512-+iOwbz9wHWwcr8kLyB6m6seE/Kw2mn0Mxatj+6ko6HwBaCZGSYRNgII7JwrKMPSaxaqE1iZ3BZRbhLwYQATaew==, + } + engines: { node: ">=20.0.0" } + "@vercel/redwood@2.4.13": resolution: { @@ -13572,6 +13582,7 @@ snapshots: "@sinclair/typebox": 0.34.49 "@slack/web-api": 7.16.0 "@vercel/functions": 3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43) + "@vercel/queue": 0.2.0 "@vercel/sandbox": 2.0.0 ai: 6.0.190(zod@4.4.3) bash-tool: 1.3.16(@vercel/sandbox@2.0.0)(ai@6.0.190(zod@4.4.3))(just-bash@3.0.1) @@ -15222,12 +15233,14 @@ snapshots: entities@6.0.1: {} - env-runner@0.1.9: + env-runner@0.1.9(@vercel/queue@0.2.0): dependencies: crossws: 0.4.5(srvx@0.11.16) exsolve: 1.0.8 httpxy: 0.5.3 srvx: 0.11.16 + optionalDependencies: + "@vercel/queue": 0.2.0 environment@1.1.0: {} @@ -16965,12 +16978,12 @@ snapshots: nf3@0.3.17: {} - nitro@3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.7)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)): + nitro@3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(@vercel/queue@0.2.0)(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.7)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)): dependencies: consola: 3.4.2 crossws: 0.4.5(srvx@0.11.16) db0: 0.3.4 - env-runner: 0.1.9 + env-runner: 0.1.9(@vercel/queue@0.2.0) h3: 2.0.1-rc.22(crossws@0.4.5(srvx@0.11.16)) hookable: 6.1.1 nf3: 0.3.17 @@ -16982,6 +16995,7 @@ snapshots: unenv: 2.0.0-rc.24 unstorage: 2.0.0-alpha.7(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(db0@0.3.4)(lru-cache@11.5.0)(ofetch@2.0.0-alpha.3) optionalDependencies: + "@vercel/queue": 0.2.0 jiti: 2.7.0 rollup: 4.60.4 vite: 8.0.14(@types/node@25.9.1)(esbuild@0.27.7)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0) @@ -17016,12 +17030,12 @@ snapshots: - sqlite3 - uploadthing - nitro@3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)): + nitro@3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(@vercel/queue@0.2.0)(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)): dependencies: consola: 3.4.2 crossws: 0.4.5(srvx@0.11.16) db0: 0.3.4 - env-runner: 0.1.9 + env-runner: 0.1.9(@vercel/queue@0.2.0) h3: 2.0.1-rc.22(crossws@0.4.5(srvx@0.11.16)) hookable: 6.1.1 nf3: 0.3.17 @@ -17033,6 +17047,7 @@ snapshots: unenv: 2.0.0-rc.24 unstorage: 2.0.0-alpha.7(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(db0@0.3.4)(lru-cache@11.5.0)(ofetch@2.0.0-alpha.3) optionalDependencies: + "@vercel/queue": 0.2.0 jiti: 2.7.0 rollup: 4.60.4 vite: 8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0) diff --git a/specs/agent-session-resumability.md b/specs/agent-session-resumability.md index a098b0f7f..2c30bb171 100644 --- a/specs/agent-session-resumability.md +++ b/specs/agent-session-resumability.md @@ -15,7 +15,7 @@ Define the durable agent session log and how a single assistant turn is split in - Durable agent session history and its projection into Pi/runtime state. - Minimal session-log event schema at safe resume boundaries. - Pi replay/continue contract (`agent.state.messages = ...` + `continue`) across slices. -- Signed internal callback contract for timeout continuation. +- Continuation contract for queue-driven conversation workers. - Separation between canonical session logs, derived projections, and durable thread state. - Failure recovery and observability requirements. @@ -25,14 +25,15 @@ Define the durable agent session log and how a single assistant turn is split in - Backward compatibility with legacy `inflight_partial` state. - Replacing existing tool implementations or Slack transport UX. - Multi-turn planning policies (this spec covers one assistant turn/session at a time). -- A generic queue/lease/fencing workflow runtime. +- Conversation mailbox, queue wake-up, lease, and heartbeat mechanics owned by + `./task-execution.md`. - Reconciling or rewriting partially visible Slack assistant output after timeout. ## Contracts ### Spec Boundary -This spec owns how one assistant turn is persisted and resumed across execution slices. The full Slack-event-to-agent-to-Slack data flow belongs to `./chat-architecture.md`; user-visible Slack acknowledgements and final delivery belong to `./slack-agent-delivery.md`. +This spec owns how agent session state is persisted and resumed across execution slices. The durable mailbox, queue wake-up, lease, and heartbeat recovery flow belongs to `./task-execution.md`. The full Slack-event-to-agent-to-Slack data flow belongs to `./chat-architecture.md`; user-visible Slack progress and final delivery belong to `./slack-agent-delivery.md`. ### Identity Model @@ -40,9 +41,12 @@ This spec owns how one assistant turn is persisted and resumed across execution - `session_id`: Conversation-local session marker for the reduced session-log projection. It starts at `session_0`, advances when `projection_reset` creates a replacement projection, and is not the durable history key. -- `turn_id`: Internal identity for one resumable execution attempt inside the - conversation. This is only needed for pause/resume correlation and diagnostics. -- `slice_id`: Monotonic integer starting at `1` for each resumed execution chunk in the same turn. +- `turn_id`: Optional internal identity for one resumable execution attempt + inside the conversation. Queue-driven continuation does not need this value to + decide where to resume; the reduced conversation session log is the resume + source. +- `slice_id`: Diagnostic integer for one execution chunk in the same + conversation. The first-pass mailbox worker must not enforce a slice cap. - `event_id`: Stable identity for one durable session-log event. - `pause_event_id`: Event id carried by timeout/auth resume callbacks so stale callbacks can be dropped. @@ -53,15 +57,15 @@ events. Each pause event identifies one safe resume boundary inside that log. ### Runtime State Partition -- Chat SDK state is the ingress coordination layer. It owns webhook message - dedupe/cache, `concurrency: "queue"` storage, per-thread locks, thread - subscriptions, and `thread.setState()`/`channel.setState()` payloads. +- Task execution state is the ingress coordination layer. It owns durable + conversation mailboxes, queue wake-up nudges, conversation leases, and + heartbeat repair as specified in `./task-execution.md`. - Junior agent session state is separate application state. It owns the append-only model execution history and the minimal runtime transition facts needed to resume that history. -- Junior may reuse the same Redis connection as the Chat SDK adapter, but the - session log keyspace must be Junior-owned, not Chat SDK thread-state/cache - keys. +- Junior may reuse the same Redis connection as the mailbox and lease stores, + but the session log keyspace must remain Junior-owned and separate from + mailbox indexes. - The durable session log key is `junior:agent-session-log:` (prefixed by `JUNIOR_STATE_KEY_PREFIX` when configured). It stores an append-only chronological model-execution log with one deterministic @@ -88,20 +92,20 @@ events. Each pause event identifies one safe resume boundary inside that log. ### Ingress Queue Contract -Production Slack ingress uses Chat SDK `concurrency: "queue"` for each -normalized thread key. The SDK holds the per-thread lock while the active -handler runs. Messages that arrive during that handler are queued; after the -handler finishes, the SDK drains the queue, dispatches the latest queued -message, and passes earlier queued messages as `MessageContext.skipped`. +Production ingress appends normalized inbound messages to the durable +conversation mailbox and sends a queue wake-up nudge containing only the +`conversation_id`. Ingress does not decide whether a message starts a new turn +or steers an active one. -Junior must consume `MessageContext.skipped` as user-authored input for the -next dispatched turn. Ignoring it loses user messages even though the SDK queue -worked correctly. +The queue worker owns the conversation lease. Before each Pi `continue()`, and +again at each safe boundary before another model call, the worker drains pending +mailbox messages into the session log. Drained messages become part of the same +active conversation rather than a competing run. -Session-log writes for a live Slack turn must happen before the SDK handler -returns and releases the thread lock. Timeout and OAuth resume handlers must -acquire the same logical thread/conversation lock before reading or writing the -session log. +Session-log writes for drained inbound messages must happen before those +messages are marked injected in the mailbox. Queue delivery acknowledgement must +happen only after the worker has durably committed final completion, safe +cooperative yield, or a no-work result. ### Agent Session Log Contract @@ -153,8 +157,8 @@ happened: - `user_input_received`: records the user input that starts one assistant turn session when the first Pi user message does not already carry enough identity. -- `slice_started`: records that a serverless execution slice started when slice identity is - needed for timeout accounting, diagnostics, or callback validation. +- `slice_started`: records that a serverless execution chunk started when that + fact is needed for timeout accounting or diagnostics. - `pi_message`: records user, assistant, tool-call, tool-result, and host-authored Pi messages. - `projection_reset`: advances the current Pi projection to an earlier safe @@ -165,14 +169,13 @@ happened: authorization link for provider work that blocked the current session. - `authorization_completed`: records that the requester completed the authorization callback for the blocked provider work. -- `timeout_paused`: records a safe timeout boundary and carries the - `pause_event_id` used by the signed continuation callback. +- `timeout_paused`: records a safe timeout or cooperative-yield boundary when a + durable pause fact is needed beyond the Pi messages already in the log. - `auth_paused`: records a safe auth boundary and points at auth-owned callback state. - `pause_resumed`: records that a specific pause event was consumed when that - fact is needed to reject stale callbacks or explain slice continuity. Omit it - when a following `slice_started` event with - `reason=timeout_resume|auth_resume` is enough. + fact is needed to explain execution continuity. Omit it when a following + `slice_started` event with `reason=queue_resume|auth_resume` is enough. - `assistant_reply_delivered`: records that the final assistant reply for this session was accepted by Slack. - `session_abandoned`: records that this session must not resume because a @@ -347,8 +350,9 @@ Valid lifecycle transitions: 4. `delivered` is terminal 5. `abandoned` is terminal -The implementation should not persist a separate `running` lease state between -slices. Per-thread execution locks are owned by the Chat SDK state adapter. +The implementation should not persist a separate `running` lease state in the +session log. Conversation execution leases are mailbox-worker state owned by +`./task-execution.md`. ### Safe Resume Boundary Contract @@ -424,67 +428,59 @@ is resumable with `continue()`. If the previous slice timed out after producing uncommitted partial assistant text, that text may be regenerated in the next slice. User-visible output must only include committed transcript content. -### Slice Deadline And Timeout Pause Contract - -- Slice execution is bounded by: - - `AGENT_TURN_TIMEOUT_MS` inside `generateAssistantReply(...)` - - the platform/function max duration outside the agent loop -- On timeout: - 1. Abort the Pi agent and wait only a short bounded grace period for the in-flight prompt/continue call to settle before snapshotting Pi messages. If the run does not settle, use the best available in-memory Pi state and the last durable boundary rules below; timeout recovery must not wait until the platform/function max duration kills the request. - 2. If session context exists and a safe boundary can be materialized, append a `timeout_paused` event with: - - `pause_event_id` - - current `slice_id` - - `resumed_from_slice_id=` - - projected safe Pi boundary, directly or by deterministic projection rule - 3. Throw a retryable timeout error carrying `conversation_id`, `turn_id`, `slice_id`, and `pause_event_id`. - 4. If timeout pause persistence fails, fall back to normal non-resumable turn failure behavior. - -### Automatic Continuation Contract - -- Session continuation is the agent recovery model: Junior must be able to rebuild the runtime from durable thread state plus the latest safe session-log pause event and continue the same turn session. -- This spec covers session-log and resume mechanics. Transport retry/locking is defined in the chat architecture spec, and Slack-visible delivery behavior is defined in the Slack delivery spec. -- Automatic timeout continuation is the current proactive producer of session-continuation work for serverless/Vercel time limits. It is best-effort and currently uses a signed internal HTTP callback, not a generic queue/lease system. -- A timeout pause may be auto-scheduled only when no assistant text has been made visible to the user for the current turn. -- Once visible assistant output has started posting, the runtime must not auto-resume that turn or attempt to rewrite/reconcile the partial output. -- In the current Slack delivery contract, assistant text is not posted until the reply is finalized, so ordinary agent-generation timeouts still occur before visible output begins. -- If a later user message arrives while `activeTurnId` points at a session whose reduced state is awaiting automatic continuation, the live runtime must treat that message as a retry signal for the existing session: reschedule the pause callback, keep `activeTurnId` on the original session, and do not start a new agent turn. -- In that case, the last safe pause event may still exist for inspection or operator-driven recovery, but the user-visible turn is allowed to fail only after automatic continuation is impossible or exhausted. +### Cooperative Continuation Contract + +- Session continuation is the agent recovery model: Junior must be able to + rebuild runtime state from durable thread state plus the reduced session log + and call Pi `continue()`. +- The task execution spec owns when a serverless worker should yield, enqueue + another conversation wake-up, release the lease, and exit. +- This spec owns only the session-log requirement: every safe boundary that may + be resumed must already be durably represented in the session log before the + worker yields or before the next model call begins. +- Routine cooperative continuation must happen only at safe Pi boundaries. The + runtime must not create synthetic checkpoints midway through a model stream or + tool call. +- If a function dies during a model or tool call, recovery uses the last + previously persisted safe boundary. It does not need an emergency abort to + create a new boundary. +- Once visible assistant output has started posting, the runtime must not + auto-resume that turn or attempt to rewrite/reconcile the partial output. +- In the current Slack delivery contract, assistant text is not posted until the + reply is finalized, so ordinary generation and tool-loop continuation remains + eligible until final delivery starts. +- If a later user message arrives while the conversation is active, the mailbox + worker treats it as pending input for the same conversation and injects it at + the next safe boundary. It must not start a competing agent run for the same + conversation. ### In-Process Provider Retry Contract - Transient provider failures reported as terminal assistant messages with `stopReason=error` may be retried inside the same running slice before final Slack delivery. - Provider retry must not replay the original user prompt. It must remove only the trailing assistant error message(s), verify the remaining Pi history ends at a continuable boundary (`user` or `toolResult`), append a `projection_reset` event for that safe boundary, then call `continue()`. - Provider retry is bounded and uses short exponential backoff. If the retry limit is reached, if the error is not classified as transient, or if no safe boundary remains after trimming, the normal provider-failure reply path owns user-visible recovery. -- Provider retry does not create an awaiting pause and does not schedule a signed resume callback. If a retried slice later times out, the timeout continuation contract above applies. +- Provider retry does not create an awaiting pause. If a retried slice reaches a + cooperative yield boundary later, the conversation mailbox worker owns + re-enqueueing the conversation. - Provider retry is only allowed before final Slack reply delivery. The runtime must not retry by rewriting or reconciling text already posted to Slack. -### Internal Timeout-Resume Callback Contract +### Queue-Driven Resume Contract -The timeout-resume callback payload is: +A queue-driven resume payload contains only `conversation_id`. It must not carry +a checkpoint, slice id, or prompt text. -- `conversation_id` -- `turn_id` -- `pause_event_id` - -The callback must: - -1. Be authenticated with an HMAC signature over the request body plus timestamp. -2. Be rejected when the signature is invalid or too old. -3. Load and reduce the session log for `conversation_id`. -4. Exit without work when: - - no session log exists - - `state !== awaiting_resume` - - `resume_reason !== timeout` - - `latest_pause_event_id !== pause_event_id` -5. Acquire the same per-thread state-adapter lock used by live turn execution. Because callbacks are often scheduled before the scheduling live handler has released the lock, a busy lock must be retried for a short bounded window before the callback reschedules itself. -6. Rebuild turn runtime state from durable thread/configuration state: - - user message +The worker must: + +1. Acquire the conversation lease defined in `./task-execution.md`. +2. Load durable thread/configuration state: - conversation context + - pending mailbox messages - artifact state - sandbox identity - channel configuration -7. Restore Pi messages with `agent.state.messages = ...` and resume with `continue()`. -8. If the resumed slice times out again before visible output, schedule a new callback carrying the new `pause_event_id`. +3. Drain pending mailbox messages into the session log idempotently. +4. Restore Pi messages with `agent.state.messages = ...`. +5. Resume with `continue()`. ### Slice Lifecycle @@ -496,40 +492,32 @@ The callback must: context, runtime loads and reduces it, restores Pi from the projected messages, and appends the new user input without duplicating bootstrap context. -4. Slice `1` runs and eagerly persists sandbox/artifact state as those values change. +4. The queue worker runs and eagerly persists sandbox/artifact state as those values change. 5. If Slack accepts the final assistant reply, append `assistant_reply_delivered`. 6. If MCP auth pauses at a safe boundary, append `auth_paused`; the OAuth callback later consults auth-owned state before resuming. -7. If timeout is reached before any assistant text is visible, append `timeout_paused` and schedule the signed internal timeout-resume callback with that event id. -8. The timeout-resume handler validates `pause_event_id`, rebuilds durable runtime state, restores Pi messages, and calls `continue()`. -9. If the callback loses the per-thread lock because the scheduling live handler is still unwinding, it retries briefly and then reschedules the same pause-event callback without mutating state. -10. If the user pings the thread while the timeout pause is still awaiting resume, the runtime reschedules the existing callback. The ping must not create a second agent run for the same conversation. -11. If timeout happens after visible assistant output begins, keep the timeout pause event but do not auto-schedule continuation. +7. If the worker reaches a cooperative yield boundary, it ensures the latest safe boundary is durably represented in the session log, enqueues the conversation id, releases the lease, and exits. +8. The next queue worker rebuilds durable runtime state, restores Pi messages, drains newly pending mailbox input, and calls `continue()`. +9. If the worker disappears before a cooperative yield, heartbeat recovery requeues the conversation after the lease expires. The next worker resumes from the latest durable session-log boundary. +10. If timeout happens after visible assistant output begins, keep the last durable state but do not auto-reconcile partial visible output. ## Failure Model 1. Timeout or crash before a stable session-log append: no new boundary exists; the system can rely on the previous reduced state plus whatever thread state had already been eagerly persisted. -2. Timeout pause append succeeds but the timeout-resume callback is never sent or delivered: there is no sweeper today; continuation requires another explicit callback, a later user follow-up that reschedules the existing pause, or operator intervention. -3. Stale timeout-resume callbacks for an older `pause_event_id` are dropped without doing work. -4. Duplicate concurrent callbacks for the same thread are serialized by the shared per-thread state-adapter lock. Lock-busy callbacks retry for a short bounded window, then reschedule the same pause-event callback rather than abandoning the awaiting session. -5. Timeout after visible assistant output begins: automatic continuation is skipped to avoid duplicate/corrupt user-visible output. -6. Repeated resumed timeouts before visible output may produce further `timeout_paused` events with incremented slice ids. -7. A later user message after an ungraceful crash may build its prompt history from the active session's latest reduced Pi projection. If the prior session produced assistant text that was not committed to visible thread state, that trailing assistant text must be trimmed from the fresh-turn history view. +2. Queue nudge is never delivered after a safe boundary append: heartbeat finds pending mailbox or expired lease state and enqueues the conversation id. +3. Duplicate queue nudges for the same conversation are serialized by the conversation lease. +4. Timeout after visible assistant output begins: automatic continuation is skipped to avoid duplicate/corrupt user-visible output. +5. Repeated cooperative yields before visible output may produce further execution chunks, but first pass does not enforce a slice cap. +6. A later user message after an ungraceful crash may build its prompt history from the active session's latest reduced Pi projection. If the prior session produced assistant text that was not committed to visible thread state, that trailing assistant text must be trimmed from the fresh-turn history view. ## Observability Required log events/diagnostics: -- `agent_turn_timeout` -- `agent_turn_timeout_resume_log_append_failed` -- `agent_turn_timeout_resume_schedule_failed` -- `agent_turn_continuation_retry_schedule_failed` -- `agent_turn_timeout_resume_skipped_after_visible_output` +- `conversation_work_cooperative_yield` +- `conversation_work_lease_expired_requeued` +- `agent_turn_session_log_append_failed` +- `agent_turn_resume_skipped_after_visible_output` - `agent_turn_provider_retry` -- `timeout_resume_failed` -- `timeout_resume_handler_failed` -- `timeout_resume_lock_busy` -- `timeout_resume_lock_busy_retrying` -- `slack_turn_continuation_notice_post_failed` Required attributes when available: @@ -537,33 +525,27 @@ Required attributes when available: - `gen_ai.operation.name` - `gen_ai.request.model` - `app.ai.turn_timeout_ms` -- `app.ai.resume_conversation_id` -- `app.ai.resume_session_id` -- `app.ai.resume_from_slice_id` -- `app.ai.resume_next_slice_id` -- `app.ai.resume_pause_event_id` - `app.ai.conversation_id` - `app.ai.session_id` +- `app.conversation.id` - `messaging.message.id` ## Verification -1. Unit: timeout pause events trim trailing assistant-only messages and carry a unique `pause_event_id`. -2. Unit: signed timeout-resume callbacks verify successfully and tampered payloads are rejected. -3. Unit/integration: a timed-out turn resumes with restored `agent.state.messages` + `continue` and reaches a successful terminal reply when no assistant text had been made visible. -4. Unit/integration: a resumed timeout slice can time out again and schedule the next callback with the new `pause_event_id`. -5. Unit/integration: a lock-busy timeout callback retries before rescheduling the same pause event. -6. Integration: a user follow-up or duplicate delivery during an awaiting automatic continuation pause reschedules the existing session instead of starting a new turn. -7. Unit/integration: auth-driven resume restores the same active skill/MCP tool universe before `continue()`. -8. Unit/integration: eager sandbox/artifact persistence preserves resumed tool context across slices. -9. Unit/integration: fresh follow-up turns can recover Pi history from the active/last agent session log without depending on conversation-state Pi transcript mirroring. -10. Manual/eval: once assistant text is already visible, timeout does not auto-resume or attempt to reconcile partial thread output. -11. Unit/integration: transient provider failures retry with `continue()` from a safe boundary and do not duplicate prior tool execution. -12. Unit/integration: successful provider activation appends one `mcp_provider_connected` event, and resume restores providers from those events. Legacy Pi-message inference is allowed only while pre-event session logs still exist. +1. Unit: resumable boundaries trim trailing assistant-only messages when needed. +2. Unit/integration: queue-driven resume restores `agent.state.messages` and calls `continue()`. +3. Integration: a cooperative yield resumes in a later worker and reaches a successful terminal reply. +4. Integration: a user follow-up during active execution is appended to the mailbox and injected into the same conversation at the next safe boundary. +5. Unit/integration: auth-driven resume restores the same active skill/MCP tool universe before `continue()`. +6. Unit/integration: eager sandbox/artifact persistence preserves resumed tool context across execution chunks. +7. Unit/integration: fresh follow-up turns can recover Pi history from the active/last agent session log without depending on conversation-state Pi transcript mirroring. +8. Manual/eval: once assistant text is already visible, recovery does not auto-reconcile partial thread output. +9. Unit/integration: transient provider failures retry with `continue()` from a safe boundary and do not duplicate prior tool execution. +10. Unit/integration: successful provider activation appends one `mcp_provider_connected` event, and resume restores providers from those events. Legacy Pi-message inference is allowed only while pre-event session logs still exist. ## Related Specs - [Harness Agent Spec](./harness-agent.md) -- [Durable Slack Thread Workflows Spec](./archive/durable-workflows.md) (archived — unimplemented design) +- [Task Execution Spec](./task-execution.md) - [Agent Execution Spec](./agent-execution.md) - [Instrumentation Spec](./instrumentation.md) diff --git a/specs/agent-turn-handling.md b/specs/agent-turn-handling.md index 1de814ce2..6a3d29dac 100644 --- a/specs/agent-turn-handling.md +++ b/specs/agent-turn-handling.md @@ -135,7 +135,7 @@ Scenarios: 2. Authorization pause resumes: - When a turn resumes after an authorization pause, Junior must continue the pending user request from durable session history and answer with the final requested content only. 3. Timeout continuation resumes: - - When a turn resumes after a timeout continuation notice, Junior must continue the same pending turn and not apologize for or repeat the runtime continuation notice unless the final answer needs to explain an actual blocker. + - When a turn resumes after a timeout continuation, Junior must continue the same pending turn and not apologize for or mention routine runtime continuation unless the final answer needs to explain an actual blocker. ### 9. Attachments And Unavailable Vision diff --git a/specs/chat-architecture.md b/specs/chat-architecture.md index c7216f430..b1a8589cc 100644 --- a/specs/chat-architecture.md +++ b/specs/chat-architecture.md @@ -3,7 +3,7 @@ ## Metadata - Created: 2026-03-21 -- Last Edited: 2026-05-30 +- Last Edited: 2026-06-01 ## Purpose @@ -37,77 +37,77 @@ The core architecture is the flow of one Slack event through a durable agent tur ```mermaid flowchart TD - A[Slack webhook event] --> B[ingress/ JuniorChat normalization] - B --> C[Chat SDK dedupe, queue, per-thread lock] - C --> D[runtime/slack-runtime route: mention, DM, subscribed, lifecycle] - D --> E[runtime/turn-preparation] + A[Slack webhook event] --> B[Slack-specific ingress normalization] + B --> C[Append to durable conversation mailbox] + C --> D[Send Vercel Queue nudge: conversationId] + D --> E[Queue worker acquires conversation lease] E --> E1[Load persisted thread state] - E --> E2[Seed/backfill conversation if needed] - E --> E3[Upsert inbound user message and attachment metadata] + E --> E2[Drain pending mailbox messages] + E --> E3[Append drained input and attachment metadata to session log] E --> F[runtime/reply-executor] - F --> G{Existing active turn has continuable pause?} - G -->|yes| H[Reschedule existing continuation and post acknowledgement] - G -->|no| I[Start active turn and persist conversation] - I --> J[respond.ts generateAssistantReply] + F --> G[Restore Pi state from reduced session log] + G --> J[respond.ts generateAssistantReply / continue] J --> K[Build prompt context from durable conversation, config, artifacts, sandbox, attachments] - K --> L[Pi agent prompt/continue loop] + K --> L[Pi agent continue loop] L --> M[Tools, skills, MCP, sandbox] M --> N[Eager state updates: sandbox id, artifacts, pending auth] N --> L + L --> Y{Safe boundary and soft yield due?} + Y -->|yes| YA[Enqueue conversationId, release lease, ack delivery] + Y -->|no| L + L --> Z{Safe boundary and mailbox has new input?} + Z -->|yes| E2 + Z -->|no| O L --> O{Turn outcome} O -->|success/final diagnostics| P[Plan finalized Slack reply] O -->|auth pause| Q[Append auth pause event and pending auth pointer] - O -->|timeout at safe boundary| R[Append timeout pause event] + O -->|cooperative yield| R[Durable session-log boundary already committed] O -->|terminal failure| S[Capture failure and build fallback reply] - R --> T[Schedule signed internal continuation callback] - T --> U[Post durable continuation acknowledgement] + R --> YA Q --> V[Private auth link plus visible URL-free auth acknowledgement; live turn ends] - V --> AB[OAuth/MCP callback resumes session] + V --> AB[OAuth/MCP callback appends mailbox work and enqueues conversation] P --> W[Deliver finalized Slack reply/files] S --> W - W --> X[Persist assistant message and mark session delivered] - T --> Y[handlers/turn-resume callback] - Y --> Z[Validate callback state, pause event/session, and per-thread lock] - AB --> Z - Z --> AA[Rebuild state from durable thread/configuration stores] - AA --> J + W --> X[Persist assistant message, mark session delivered, release lease] + AB --> D ``` Normative rules: -1. Ingress normalizes events and hands them to queue/runtime. It must not decide agent behavior. -2. The Chat SDK queue/lock layer handles transport-level concerns such as duplicate inbound delivery, ordering, and lock serialization. It is not the source of truth for agent recovery. -3. Turn preparation is the only point that converts an inbound Slack message into persisted conversation context for an agent turn. +1. Ingress normalizes events, appends them to the durable mailbox, and sends a queue nudge. It must not decide agent behavior. +2. The task execution layer owns queue wake-ups, conversation leases, worker check-ins, and heartbeat recovery. Chat SDK queue/lock semantics are not canonical. +3. The mailbox worker is the only point that drains inbound messages into persisted agent session context. 4. `respond.ts` is the only owner of Pi agent execution, prompt/continue selection, timeout detection, and safe-boundary session-log event creation. 5. Tool calls and tool failures are internal agent-loop data until the assistant produces final turn diagnostics. Tool execution errors must be captured, but they are not automatically terminal user replies. Model-repairable failures must use the tool-error semantics from [Agent Execution Discipline Spec](./agent-execution.md). 6. User-visible assistant text is posted only after the reply is finalized and planned for Slack delivery. 7. Final turn success is defined by Slack accepting the visible final reply, not by model generation completing. 8. Agent recovery is session continuation: reload durable thread state plus the reduced session log, then continue the same session. It must not create a second active turn or re-run from transient process memory. -9. Serverless timeout handling is one producer of continuation pause events. It is a proactive accommodation for Vercel execution limits, not the general recovery architecture. -10. Acknowledgement messages, assistant status, and logs are observability/UX surfaces. They never substitute for persisted session recovery or final reply delivery. +9. Cooperative yield at safe agent-loop boundaries is the normal accommodation for Vercel execution limits. Platform death is recovered by queue redelivery and heartbeat repair from the latest durable session boundary. +10. Assistant status and logs are observability/UX surfaces. They never substitute for persisted session recovery or final reply delivery. Data authority by stage: -| Data | Authority | Notes | -| ---------------------------------------- | ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | -| Slack event shape | `ingress/` and Slack adapter payloads | Normalize IDs and attachments before runtime; do not infer behavior here. | -| Queue ordering and duplicate suppression | Chat SDK state adapter lock/queue | Prevent concurrent handler execution, but do not rely on queue memory for turn recovery. | -| Conversation transcript | Persisted thread state | Source for visible user/assistant thread history; assistant messages are added only after final Slack delivery. | -| Active pause routing | Thread state | Points callbacks at the paused session/event when a live run has parked for auth or timeout. | -| Pi execution transcript | Junior agent session log keyed by conversation id | Append-only model-execution log with a deterministic Pi-message projection; not a replacement for visible Slack transcript. | -| Sandbox/artifact state | Persisted thread state | Persist eagerly as it changes so a resumed slice can rebuild the same runtime world. | -| Pending auth | Auth-owned callback state plus session-log pause event | Auth pauses end the live turn after private link delivery and are resumed by callback. | -| Final Slack reply | Slack thread post acceptance | Completion is persisted only after Slack accepts the final visible reply. | -| Continuation acknowledgement | Slack thread post | User-facing status only; does not complete the turn. | +| Data | Authority | Notes | +| --------------------------------------- | ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | +| Slack event shape | `ingress/` and Slack adapter payloads | Normalize IDs and attachments before runtime; do not infer behavior here. | +| Queue wake-up and duplicate suppression | Conversation mailbox worker and lease | Queue messages are nudges; mailbox state and leases decide whether work exists and who may run it. | +| Conversation transcript | Persisted thread state | Source for visible user/assistant thread history; assistant messages are added only after final Slack delivery. | +| Active work routing | Conversation mailbox and lease | Pending messages are drained into the active conversation at safe boundaries. | +| Pi execution transcript | Junior agent session log keyed by conversation id | Append-only model-execution log with a deterministic Pi-message projection; not a replacement for visible Slack transcript. | +| Sandbox/artifact state | Persisted thread state | Persist eagerly as it changes so a resumed slice can rebuild the same runtime world. | +| Pending auth | Auth-owned callback state plus session-log pause event | Auth pauses end the live turn after private link delivery and are resumed by callback. | +| Final Slack reply | Slack thread post acceptance | Completion is persisted only after Slack accepts the final visible reply. | +| Routine continuation progress | Assistant status and `reportProgress` | Cooperative yields do not post filler thread messages. | Related contract ownership: -| Spec | Owns | -| --------------------------------- | ---------------------------------------------------------------------------------------------- | -| `./chat-architecture.md` | End-to-end turn data flow, data authority, and module boundaries. | -| `./agent-session-resumability.md` | Session-log schema, Pi `continue()` semantics, timeout callback safety, and session lifecycle. | -| `./slack-agent-delivery.md` | Slack entry surfaces, progress UX, continuation acknowledgements, and final reply delivery. | -| `./slack-outbound-contract.md` | Slack API write boundary, formatting, file upload, reactions, and Slack error mapping. | +| Spec | Owns | +| --------------------------------- | -------------------------------------------------------------------------------------- | +| `./chat-architecture.md` | End-to-end turn data flow, data authority, and module boundaries. | +| `./task-execution.md` | Conversation mailbox, queue wake-up, lease, cooperative yield, and heartbeat repair. | +| `./agent-session-resumability.md` | Session-log schema, Pi `continue()` semantics, and session lifecycle. | +| `./slack-agent-delivery.md` | Slack entry surfaces, progress UX, pause acknowledgements, and final reply delivery. | +| `./slack-outbound-contract.md` | Slack API write boundary, formatting, file upload, reactions, and Slack error mapping. | ### File Tree Responsibilities @@ -180,8 +180,8 @@ interface ChatTurnRuntime { #### Ingress Router -- Ingress owns event normalization, dedupe, thread-kind classification, and queue handoff. -- Chat SDK compatibility should be expressed through an explicit `Chat` subclass or wrapper, not prototype mutation or import-side effects. +- Ingress owns event normalization, dedupe, thread-kind classification, mailbox append, and queue handoff. +- Chat SDK must not own canonical ingress queueing, conversation locks, or long-running handler lifetime. - Ingress must not become a second authority for turn behavior that already belongs in runtime. - Canonical ingress code lives under `chat/ingress/*`. Do not reintroduce legacy patch modules for core ingress behavior. @@ -195,7 +195,7 @@ interface MessageIngressRouter { #### Queue Dispatcher -- The queue worker owns serialization, locking, ordering, and retry interaction. +- The queue worker owns conversation lease acquisition, mailbox draining, cooperative yield, and retry interaction. - Queue dispatch into the turn runtime must be an injected boundary. - Queue worker code must not import the production bot singleton. - Provider transport should stay behind a local queue transport module so retry policy and handler semantics remain project-owned. @@ -249,19 +249,19 @@ interface EvalScenarioRunner { - Adapter/backend selection belongs in the adapter layer. - Key-building, TTL policy, and persistence rules belong in store modules. - Consumers should import the narrowest store they need rather than routing all access through a broad facade. -- Per-thread Chat SDK locks use a short renewable state-adapter lease. Live workers heartbeat the lease while processing; if a worker disappears, the lease must expire quickly enough for a retry or continuation callback to acquire ownership. +- Conversation execution leases use a short renewable state-adapter lease. Live workers check in while processing; if a worker disappears, the lease must expire quickly enough for heartbeat or queue redelivery to re-enqueue the conversation. ### Turn Continuation Recovery -Turn continuation recovery covers any case where Junior has a durable safe resume boundary but does not know whether the active turn reached final Slack delivery: serverless timeout, lost or duplicate callback, transport retry, or user follow-up while thread state still points at an awaiting pause. +Turn continuation recovery covers any case where Junior has a durable safe resume boundary but does not know whether the active turn reached final Slack delivery: serverless timeout, lost or duplicate queue nudge, worker death, transport retry, or user follow-up while the conversation is active. Rules: -1. Recovery continues the existing conversation from durable thread state plus the reduced agent-session log keyed by the predictable conversation id. It must not start a second active run for a thread with an awaiting continuation pause. -2. Chat SDK retry, dedupe, queueing, and per-thread locking protect inbound delivery. They do not replace session continuation because they do not carry Junior's canonical agent session log, sandbox/artifact state, pending auth, or final Slack delivery state. -3. `respond.ts` appends safe pause events; `runtime/reply-executor.ts` schedules or reschedules continuation; `handlers/turn-resume.ts` validates pause event identity and lock ownership; `runtime/slack-resume.ts` reuses the normal final-delivery path. -4. Continuation acknowledgements are Slack UX only. They do not complete the turn and are not a recovery mechanism. -5. Lock-busy callback retry is bounded. There is no durable sweeper today, so retry exhaustion reschedules the same signed callback for the current pause event rather than completing or abandoning the turn. +1. Recovery continues the existing conversation from durable thread state plus the reduced agent-session log keyed by the predictable conversation id. It must not start a second active run for the same conversation. +2. Conversation mailbox state, queue wake-ups, and leases protect inbound delivery. They do not replace session continuation because they do not carry Junior's canonical agent session log, sandbox/artifact state, pending auth, or final Slack delivery state. +3. `respond.ts` appends safe session-log boundaries; the mailbox worker enqueues cooperative continuations; heartbeat re-enqueues expired leases and stranded pending mailbox work. +4. Routine cooperative continuation does not create a Slack acknowledgement message. Assistant status and `reportProgress` own progress UX. +5. Queue duplicate and active-lease cases are harmless because queue messages are conversation wake-up nudges, not canonical work records. ### Test And Eval Rules @@ -306,6 +306,7 @@ Current names that should be treated as transitional: ## Related Specs +- `./task-execution.md` - `./agent-session-resumability.md` - `./oauth-flows.md` - `./plugin.md` diff --git a/specs/index.md b/specs/index.md index 33a13a207..00fb28cd7 100644 --- a/specs/index.md +++ b/specs/index.md @@ -3,7 +3,7 @@ ## Metadata - Created: 2026-03-03 -- Last Edited: 2026-05-30 +- Last Edited: 2026-06-01 ## Purpose @@ -31,6 +31,7 @@ Define spec taxonomy, naming conventions, and canonical source-of-truth document - `specs/security-policy.md` - `specs/data-redaction-policy.md` - `specs/chat-architecture.md` +- `specs/task-execution.md` - `specs/agent-turn-handling.md` - `specs/slack-agent-delivery.md` - `specs/slack-outbound-contract.md` @@ -66,6 +67,7 @@ Define spec taxonomy, naming conventions, and canonical source-of-truth document For chat/agent/Slack turn behavior: - `specs/chat-architecture.md` owns the end-to-end turn data flow, data authority map, and module boundaries. +- `specs/task-execution.md` owns durable conversation mailbox execution, queue wake-up semantics, conversation leases, cooperative yield, and heartbeat repair. - `specs/agent-turn-handling.md` owns user-message response policy: when Junior answers, stays silent, asks, uses tools, satisfies Slack side effects, handles resumed turns, and considers a turn complete. - `specs/agent-execution.md` owns coding-agent execution discipline and the repository-wide model-repairable tool failure contract. - `specs/harness-agent.md` owns the Pi agent turn runtime contract, final output resolution, and turn diagnostics. diff --git a/specs/scheduler.md b/specs/scheduler.md index 0dbafb495..91e9afc3d 100644 --- a/specs/scheduler.md +++ b/specs/scheduler.md @@ -3,7 +3,7 @@ ## Metadata - Created: 2026-05-18 -- Last Edited: 2026-05-31 +- Last Edited: 2026-06-01 ## Purpose @@ -310,6 +310,7 @@ Use unit tests only for small deterministic helpers when integration or eval cov ## Related Specs - `./chat-architecture.md` +- `./task-execution.md` - `./agent-prompt.md` - `./agent-session-resumability.md` - `./trusted-plugin-heartbeat.md` diff --git a/specs/slack-agent-delivery.md b/specs/slack-agent-delivery.md index 640de2017..ba915706d 100644 --- a/specs/slack-agent-delivery.md +++ b/specs/slack-agent-delivery.md @@ -3,7 +3,7 @@ ## Metadata - Created: 2026-04-15 -- Last Edited: 2026-05-28 +- Last Edited: 2026-06-01 ## Purpose @@ -217,10 +217,9 @@ Current rules: 6. When a turn blocks on OAuth/MCP auth, Junior must privately deliver the auth link, post a brief visible thread acknowledgement that authorization is needed, clear `activeTurnId`, and persist thread-local pending-auth state. The visible acknowledgement must not include the auth URL or other secret-bearing state. 7. Automatic auth resumes must not post a separate public "account connected, continuing..." banner before the real resumed answer. The resumed answer itself is the visible continuation. 8. If auth completes after a newer thread message already replaced the blocked request, Junior stores the credentials but does not post a stale resumed answer. -9. When a turn session record is scheduled for automatic continuation, Junior must post a durable thread acknowledgement that the turn is continuing in the background. Assistant status alone is not sufficient because it is best effort and expires independently of thread history. -10. If a user follow-up or duplicate delivery hits the same awaiting continuation, Junior should acknowledge the existing continuation instead of creating a second visible turn. Session Record rescheduling mechanics belong to `./agent-session-resumability.md`. -11. Turn-continuation acknowledgements are not final assistant replies. They do not mark the original turn completed, and the final resumed answer must still be delivered through the normal finalized-reply path. -12. Turn-continuation acknowledgements may include a correlation-only footer with the conversation ID or trace link so operators can connect the durable notice to diagnostics. They must not include final-turn duration, token usage, or thinking-level metadata because those belong to the finalized reply. +9. Routine cooperative continuation must not post a visible "continuing in the background" thread acknowledgement. User-visible progress belongs to assistant status and `reportProgress`; final answers still use the finalized reply path. +10. If a user follow-up or duplicate delivery arrives while a turn is active, Junior should fold it into the active conversation at the next safe execution boundary instead of creating a second visible turn. Mailbox and worker mechanics belong to `./task-execution.md`. +11. Any explicit pause acknowledgement that remains for auth or exceptional failure handling is not a final assistant reply. It does not mark the original turn completed, and the final resumed answer must still be delivered through the normal finalized-reply path. ### 12. Testing Contract @@ -274,6 +273,7 @@ Required verification coverage for this contract: ## Related Specs - `./chat-architecture.md` +- `./task-execution.md` - `./slack-outbound-contract.md` - `./oauth-flows.md` - `./agent-session-resumability.md` diff --git a/specs/task-execution.md b/specs/task-execution.md new file mode 100644 index 000000000..129c01118 --- /dev/null +++ b/specs/task-execution.md @@ -0,0 +1,469 @@ +# Task Execution Spec + +## Metadata + +- Created: 2026-06-01 +- Last Edited: 2026-06-01 + +## Purpose + +Define Junior's durable execution contract for serverless runtimes where any +function invocation may disappear, time out around 300 seconds, or receive +duplicate queue deliveries. + +The system exists so inbound work is not lost, active conversations recover +without user pings, and long agent loops can continue across fresh serverless +invocations without turning every tool call into a queue round trip. + +## Scope + +- Durable conversation mailboxes for inbound work. +- Vercel Queue wake-up messages. +- Conversation-scoped leases and worker check-ins. +- Cooperative agent-loop yielding at safe Pi continuation boundaries. +- Heartbeat repair for expired leases and stranded mailbox work. +- The boundary between Slack-specific ingress and generic agent execution. +- First-pass migration away from Chat SDK queue, lock, and long-running handler + ownership. + +## Non-Goals + +- A generic workflow engine. +- A durable task database with task records, checkpoint references, child task + state machines, or slice counters. +- Queueing every model call or every tool call as a separate asynchronous task. +- Exactly-once external side-effect delivery. +- Mid-model-stream or mid-tool-call checkpointing. +- First-pass max turn age, max slice count, or poison-work policy. These are + TODO guardrails after the mailbox worker is in production. +- Using Slack thread messages as progress filler for routine continuation. + +## Contracts + +### Architecture Summary + +Junior uses a durable conversation mailbox plus a queue wake-up nudge. + +```mermaid +flowchart TD + A[Inbound source event] --> B[Source-specific normalization] + B --> C[Append inbound message to conversation mailbox] + C --> D[Send Vercel Queue nudge: conversationId] + D --> E[Queue consumer] + E --> F{Conversation lease active?} + F -->|yes| G[Send delayed nudge and ack delivery] + F -->|no or expired| H[Acquire conversation lease] + H --> I[Drain mailbox into agent session log] + I --> J[Restore Pi state and call continue] + J --> K[Agent loop: model and tool batches] + K --> L{Safe boundary} + L --> M[Drain newly pending mailbox messages] + M --> N{Soft yield due?} + N -->|yes| O[Send nudge, release lease, ack delivery] + N -->|no| K + L --> P{Agent final and inbox empty?} + P -->|yes| Q[Deliver final reply and release lease] + P -->|no| K +``` + +Normative rules: + +1. Durable mailbox state is the source of truth for pending inbound work. +2. Vercel Queue messages are wake-up nudges only. Their payload is + `{ conversationId }`. +3. Queue delivery is at-least-once. Duplicate nudges must be cheap and safe. +4. A worker may execute a conversation only while holding the conversation + lease. +5. The agent session log is the checkpoint. No task payload should name a + checkpoint or resume position. +6. Continuation means loading the latest durable conversation/session state and + calling Pi `continue()`. +7. Routine continuation must be silent in Slack. The agent-owned + `reportProgress` path and assistant status own user-visible progress. + +### Identity Model + +`conversationId` is the execution key. For Slack, it must be a stable normalized +thread identity such as `slack::`. + +`inboundMessageId` is the idempotency key for one normalized inbound message. +For Slack, it should be derived from the Slack team, channel, message timestamp, +event subtype/edit identity when relevant, and source event id when available. + +`leaseToken` is a random value proving that the current worker owns the +conversation lease. + +There is no durable task id in the first-pass design. + +### Ingress Contract + +Inbound source handlers are source-specific. Slack parsing, signature +verification, event subtype handling, assistant lifecycle event handling, and +attachment normalization may be Slack-specific. + +Ingress must not decide whether an inbound message is a new turn, steering for +an active turn, or a stale follow-up. It always performs the same durable handoff: + +1. Verify the source request. +2. Normalize a stable `conversationId` and `inboundMessageId`. +3. Persist the inbound message into the conversation mailbox idempotently. +4. Enqueue `{ conversationId }`. +5. Return the source HTTP acknowledgement quickly. + +The source HTTP acknowledgement is not the late acknowledgement. Late +acknowledgement applies to the queue delivery consumed by the worker. + +If enqueue fails after mailbox append, the heartbeat repair path must later +find the stranded pending mailbox work and enqueue another nudge. + +### Mailbox Contract + +The mailbox is conversation-owned durable state. Implementations may store it as +one record, indexed inbound message records, or both. The contract is: + +- inbound messages are deduped by `inboundMessageId` +- pending messages are ordered by source creation time and stable tie-breakers +- a pending message is not removed or marked injected until the corresponding + session-log append succeeds +- reinjecting the same `inboundMessageId` into the session log must be + idempotent +- messages that arrive while a worker is active remain pending until the worker + drains them at a safe boundary or a later worker resumes the conversation + +Conceptual shape: + +```ts +interface InboundMessageRecord { + inboundMessageId: string; + conversationId: string; + source: "slack" | "scheduler" | "plugin"; + createdAtMs: number; + receivedAtMs: number; + input: AgentInputMessage; + injectedAtMs?: number; +} + +interface ConversationWorkState { + conversationId: string; + lease?: ConversationLease; + lastEnqueuedAtMs?: number; + updatedAtMs: number; +} +``` + +The exact storage shape should stay simple. Do not add a separate task record +only to represent data already present in the mailbox or session log. + +### Queue Contract + +The first implementation should use Vercel Queues push consumers if Vercel +Queues satisfies these requirements: + +- at-least-once delivery +- consumer-controlled acknowledgement after handler completion +- redelivery when the consumer dies before acknowledgement +- visibility timeout or auto-extension suitable for serverless handlers +- idempotent send using a stable key when available + +The queue message payload is: + +```ts +interface ConversationQueueMessage { + conversationId: string; +} +``` + +Queue consumer rules: + +1. Load durable conversation work state before doing agent work. +2. If there is no pending or resumable work, acknowledge the queue delivery and + exit. +3. If another worker holds an unexpired lease, enqueue a delayed nudge for the + same `conversationId`, acknowledge the current delivery, and exit. +4. If the lease is absent or expired, acquire a new lease and process. +5. Acknowledge the queue delivery only after durable state is safe: final + delivery recorded, lease released after cooperative yield, no work found, or + unrecoverable failure recorded. + +The queue is not the state authority. A successful queue acknowledgement only +means that one wake-up delivery has been handled. + +The Vercel push consumer boundary is a thin adapter around the generic worker: +it validates the `{ conversationId }` payload, uses `handleCallback`, and keeps +the Vercel visibility timeout at 300 seconds. The internal push endpoint is +`/api/internal/agent/continue`, because each queue delivery asks Junior to +continue the latest durable agent state for that conversation. The app must wire +the concrete conversation runner before registering the queue trigger; otherwise +queue messages could be acknowledged without advancing agent state. + +### Lease And Check-In Contract + +The conversation lease serializes execution for one `conversationId`. + +Lease acquisition requires: + +- no current lease, or +- current `leaseExpiresAtMs <= now` + +Lease writes must include a fresh `leaseToken`. Any leased mutation must verify +that the stored token still matches the worker token. + +Initial timing defaults: + +```text +worker check-in interval: 15s +lease ttl: 90s +heartbeat scan interval: 30s +recovery trigger: leaseExpiresAtMs <= now +``` + +Check-ins are owned by the generic worker, not by agent progress events. While a +worker is leased, it periodically extends `leaseExpiresAtMs` and updates +`lastCheckInAtMs`. Agent progress events may update status or diagnostics, but +they are not required for lease liveness. + +There is one liveness rule: expired lease. `lastCheckInAtMs` is diagnostic +metadata. + +### Worker Contract + +A worker that owns the lease advances the conversation: + +1. Start the lease check-in timer. +2. Drain pending mailbox messages into the agent session log. +3. Restore Pi state from the reduced session log. +4. Call `continue()`. +5. At each safe boundary, drain newly pending mailbox messages into the same + active conversation before another model call starts. +6. If cooperative yield is due, enqueue `{ conversationId }`, release the lease, + acknowledge the queue delivery, and exit. +7. If the agent is final, drain the mailbox one last time before delivery. If new + messages were pending, continue instead of posting a stale answer. +8. Deliver the finalized reply through the destination delivery port. +9. Persist completion state, release the lease, and acknowledge the queue + delivery. + +Inbound messages that arrive during an active run are part of the active +conversation. They are injected at the next safe boundary, not treated as a +separate concurrent turn. + +### Cooperative Yield Contract + +Cooperative yielding prevents long agent loops from running into serverless +timeouts without queueing every tool call. + +Target timing for a 300 second function cap: + +```text +soft yield deadline: 240s from worker start +minimum budget before starting another model-loop iteration: 120s +checkpoint/requeue buffer: 60s +``` + +The worker checks yield eligibility only at safe boundaries: + +- before the first model call, if setup somehow consumed too much budget +- after a complete model response and its requested tool batch have finished +- after tool results have been durably appended to the session log +- after provider retry cleanup from a safe Pi boundary +- after auth pause state has been durably recorded +- before final reply delivery, after the final inbox drain + +Unsafe yield points: + +- midway through a model stream +- after an assistant tool request but before tool execution has produced durable + results +- midway through a tool call +- after final Slack delivery has started but before completion state is + persisted + +If the soft deadline passes during a model or tool call, the worker does not +invent a checkpoint or force an emergency abort. Correctness relies on the +latest durable session-log boundary, queue redelivery, and heartbeat recovery. + +When yielding, the worker: + +1. Ensures all safe-boundary session-log writes are complete. +2. Enqueues `{ conversationId }`. +3. Releases the lease. +4. Acknowledges the queue delivery. +5. Exits without posting a routine continuation message to Slack. + +### Agent Runtime Boundary + +The agent runtime should remain transport-agnostic. Slack-specific ingress may +normalize Slack events, but after mailbox injection the runtime consumes generic +agent input messages and generic delivery ports. + +Required ports are intentionally small: + +- load/drain inbound messages for a conversation +- append injected messages to the agent session log +- restore Pi state and call `continue()` +- update best-effort progress/status +- deliver the finalized reply + +The new implementation must not rely on Chat SDK for queueing, concurrency +locks, long-running handler lifetime, or conversation work recovery. Any +transitional compatibility wrapper must be treated as non-canonical and must not +own execution semantics. + +### Slack Delivery Contract + +Slack remains one delivery implementation. + +Rules specific to the mailbox worker: + +1. Slack HTTP ingress returns quickly after durable mailbox append and enqueue. +2. Assistant status should continue across cooperative yields by persisting the + latest progress/status state and re-establishing it when a later worker + resumes. +3. Routine cooperative yields must not post automatic "continuing in the + background" thread messages. +4. `reportProgress` and assistant status are the progress surface for long work. +5. Final visible replies still use the finalized Slack reply planner and are + delivered only after the agent has stopped and the inbox has been drained. +6. Slack delivery remains best effort around process death. First pass does not + add a generalized receipt or reconciliation system beyond persisted + conversation completion state. + +### Heartbeat Contract + +Heartbeat is a repair loop, not a worker. + +On each bounded scan, heartbeat must: + +1. Find conversations with expired leases. +2. Clear or replace the expired lease state. +3. Enqueue `{ conversationId }`. +4. Find conversations with pending mailbox messages, no unexpired lease, and no + recent enqueue marker. +5. Enqueue `{ conversationId }` for those stranded conversations. + +Heartbeat must not run the agent inline. It only repairs durable state and sends +queue wake-up nudges. + +Heartbeat scans must be bounded by limits so one large backlog does not exhaust +the cron invocation. Remaining work is left for later heartbeats. + +### Scheduler And Plugin Dispatch + +Scheduler and trusted plugin work should enter the same execution system by +creating or selecting a conversation identity, appending a normalized agent input +message to the mailbox, and enqueueing `{ conversationId }`. + +Source-specific scheduling, due-run claims, plugin idempotency, and destination +selection remain owned by their domain specs. Once claimed, execution should use +the same mailbox, lease, session-log, and delivery contracts as interactive +work. + +### TODO Guardrails + +The first pass intentionally avoids extra looping controls. After the mailbox +worker is proven in production, add policy for: + +- maximum wall-clock age for one active conversation run +- maximum consecutive recoveries without a new session-log boundary +- explicit cancel/stop semantics for user messages that should abandon active + work +- duplicate final-delivery suppression if duplicate replies are observed + +These guardrails must not complicate the first-pass mailbox/lease design. + +## Failure Model + +1. Source request dies before mailbox append: no Junior work exists. The source + platform may retry according to its own delivery contract. +2. Mailbox append succeeds but queue send fails: heartbeat finds pending mailbox + work and enqueues a nudge. +3. Queue sends duplicate nudges: only one worker can hold the lease; duplicates + acknowledge after observing no work or an active lease. +4. Queue delivery observes an active lease: it sends a delayed nudge and + acknowledges so the message that arrived during active work is not stranded + if the active worker misses the final drain. +5. Worker dies while leased: check-ins stop, `leaseExpiresAtMs` passes, + heartbeat clears/requeues, and the next worker resumes from the latest + durable session-log state. +6. Worker dies after appending inbound messages to the session log but before + marking them injected: reinjection must be idempotent by `inboundMessageId`. +7. Worker dies during a model call or tool call: recovery resumes from the + latest safe session-log boundary; no mid-call state is assumed durable. +8. Worker yields cooperatively and dies after enqueue but before queue + acknowledgement: redelivery observes released lease or no unsafe work and + remains harmless. +9. Worker dies after final delivery starts: Slack post and durable completion + are not atomic. First pass accepts best-effort delivery semantics and does + not add special reconciliation beyond persisted completion state. +10. Heartbeat misses one scan: Vercel Queue redelivery or the next heartbeat can + still recover because leases and mailbox messages are durable. + +## Observability + +Required event names should distinguish normal progress from repair: + +- `conversation_work_enqueued` +- `conversation_work_lease_acquired` +- `conversation_work_check_in_failed` +- `conversation_work_nudge_deferred_for_active_lease` +- `conversation_work_mailbox_drained` +- `conversation_work_cooperative_yield` +- `conversation_work_completed` +- `conversation_work_lease_expired_requeued` +- `conversation_work_pending_requeued` +- `conversation_work_failed` + +Required attributes when available: + +- `app.conversation.id` +- `app.conversation.source` +- `app.inbound.message_id` +- `app.inbound.pending_count` +- `app.queue.message_id` +- `app.queue.delivery_id` +- `app.lease.token_hash` +- `app.lease.expires_at_ms` +- `app.worker.elapsed_ms` +- `app.worker.soft_yield_deadline_ms` +- `app.worker.remaining_budget_ms` +- `gen_ai.request.model` +- `gen_ai.provider.name` + +Logs and spans must not include raw Slack tokens, OAuth credentials, raw +authorization URLs, or unredacted private message bodies. + +## Verification + +Required tests: + +1. Unit: mailbox append is idempotent by `inboundMessageId`. +2. Unit/integration: enqueue failure after mailbox append is repaired by + heartbeat. +3. Integration: duplicate queue nudges do not run a conversation concurrently. +4. Integration: active-lease queue delivery defers a nudge and acknowledges. +5. Integration: worker check-in extends the lease while a long model/tool call + is in progress. +6. Integration: expired lease is cleared/requeued by heartbeat. +7. Integration: pending mailbox messages with no recent enqueue marker are + requeued by heartbeat. +8. Integration: message arriving during active execution is injected at the next + safe boundary and answered as part of the same conversation run. +9. Integration: final inbox drain prevents posting a stale answer when a new + message arrived before delivery. +10. Integration: cooperative yield near the 240 second soft deadline releases + the lease, enqueues another nudge, and does not post a continuation message. +11. Integration: recovery after death during model/tool work resumes from the + latest durable session-log boundary. +12. Integration: Slack ingress returns after durable mailbox append and enqueue, + not after agent execution. +13. Evals: realistic multi-message Slack follow-ups during long work are folded + into the active answer without losing user intent. + +## Related Specs + +- [Chat Architecture Spec](./chat-architecture.md) +- [Agent Session Resumability Spec](./agent-session-resumability.md) +- [Slack Agent Delivery Spec](./slack-agent-delivery.md) +- [Scheduler Spec](./scheduler.md) +- [Trusted Plugin Dispatch Spec](./trusted-plugin-dispatch.md) +- [Testing Spec](./testing.md) diff --git a/specs/trusted-plugin-dispatch.md b/specs/trusted-plugin-dispatch.md index 9c6be8f78..09e12fc1f 100644 --- a/specs/trusted-plugin-dispatch.md +++ b/specs/trusted-plugin-dispatch.md @@ -298,6 +298,7 @@ Use unit tests for: ## Related Specs - `./trusted-plugin-heartbeat.md` +- `./task-execution.md` - `./scheduler.md` - `./agent-session-resumability.md` - `./chat-architecture.md` From 748fc05492e62b792f9176476fe160bc2efe08c3 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 09:34:32 +0200 Subject: [PATCH 02/45] fix(chat): Preserve Slack mailbox completion state Keep queued Slack mailbox records pending until the Slack runtime handoff succeeds. Mark only processed records as injected after the handler returns. Complete successful Slack handlers even after the soft deadline has elapsed. This avoids duplicate queue nudges that can replay injected work. Refs GH-470 Co-Authored-By: GPT-5 Codex --- .../src/chat/task-execution/slack-work.ts | 30 +++- .../junior/src/chat/task-execution/store.ts | 43 ++++++ .../integration/conversation-work.test.ts | 133 ++++++++++++++++++ 3 files changed, 202 insertions(+), 4 deletions(-) diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index ad5860ca1..404ba830c 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -24,6 +24,7 @@ import type { import { getConversationWorkState, countPendingConversationMessages, + markConversationMessagesInjected, } from "@/chat/task-execution/store"; import type { ConversationWorkerContext, @@ -187,6 +188,17 @@ function getInstallation( return {}; } +function getPendingRecords( + work: { messages: InboundMessageRecord[] } | undefined, +): InboundMessageRecord[] { + if (!work) { + return []; + } + return work.messages + .filter((message) => message.injectedAtMs === undefined) + .sort(compareInboundMessages); +} + /** Build the worker run function for queued Slack conversation work. */ export function createSlackConversationWorker( options: CreateSlackConversationWorkerOptions, @@ -202,7 +214,12 @@ export function createSlackConversationWorker( : { status: "completed" }; } - let records = await context.drainMailbox(async () => {}); + let records = getPendingRecords( + await getConversationWorkState({ + conversationId: context.conversationId, + state, + }), + ); if (records.length === 0) { records = await getCrashRecoveryRecords({ conversationId: context.conversationId, @@ -264,9 +281,14 @@ export function createSlackConversationWorker( }, }); - return context.shouldYield() - ? { status: "yielded" } - : { status: "completed" }; + await markConversationMessagesInjected({ + conversationId: context.conversationId, + inboundMessageIds: records.map((record) => record.inboundMessageId), + leaseToken: context.leaseToken, + state, + }); + + return { status: "completed" }; }; } diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index bdb576223..6ea2aae4a 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -657,6 +657,49 @@ export async function drainConversationMailbox(args: { }); } +/** Mark selected leased mailbox entries after their session-log injection succeeds. */ +export async function markConversationMessagesInjected(args: { + conversationId: string; + inboundMessageIds: string[]; + leaseToken: string; + nowMs?: number; + state?: StateAdapter; +}): Promise { + const nowMs = args.nowMs ?? now(); + const inboundMessageIds = new Set(args.inboundMessageIds); + return await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current || current.lease?.leaseToken !== args.leaseToken) { + return false; + } + if (inboundMessageIds.size === 0) { + return true; + } + + let changed = false; + const messages = current.messages.map((message) => { + if ( + !inboundMessageIds.has(message.inboundMessageId) || + message.injectedAtMs !== undefined + ) { + return message; + } + changed = true; + return { ...message, injectedAtMs: nowMs }; + }); + if (!changed) { + return true; + } + + await writeWorkState(state, { + ...current, + messages, + updatedAtMs: nowMs, + }); + return true; + }); +} + /** Mark the leased conversation as needing another queue-delivered slice. */ export async function requestConversationContinuation(args: { conversationId: string; diff --git a/packages/junior/tests/integration/conversation-work.test.ts b/packages/junior/tests/integration/conversation-work.test.ts index 9c1720dc8..19ac7d89e 100644 --- a/packages/junior/tests/integration/conversation-work.test.ts +++ b/packages/junior/tests/integration/conversation-work.test.ts @@ -13,6 +13,7 @@ import { countPendingConversationMessages, drainConversationMailbox, getConversationWorkState, + markConversationMessagesInjected, startConversationWork, type InboundMessageRecord, } from "@/chat/task-execution/store"; @@ -479,6 +480,14 @@ describe("conversation work execution", () => { nowMs: 3_000, }), ).resolves.toBe("lost_lease"); + await expect( + markConversationMessagesInjected({ + conversationId: CONVERSATION_ID, + inboundMessageIds: ["m1"], + leaseToken: "wrong-token", + nowMs: 3_000, + }), + ).resolves.toBe(false); }); it("maps the generic queue port to Vercel Queue send options", async () => { @@ -684,6 +693,130 @@ describe("conversation work execution", () => { expect(calls[0]?.skipped.map((message) => message.id)).toEqual([ "1712345.0001", ]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + }); + + it("keeps Slack mailbox records pending when the runtime handoff fails", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + + await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + waitUntil: () => {}, + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }, + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + throw new Error("runtime failed before durable handoff"); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).rejects.toThrow("runtime failed before durable handoff"); + + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.lease).toBeUndefined(); + expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); + }); + + it("completes Slack mailbox work when the handler finishes after the soft deadline", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + let currentNowMs = 1_000; + + await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + waitUntil: () => {}, + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }, + }); + queue.sent = []; + + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + currentNowMs = 242_000; + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(queue.sent).toEqual([]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.needsRun).toBe(false); + expect(work ? countPendingConversationMessages(work) : 0).toBe(0); }); it("rejects malformed Vercel Queue payloads", async () => { From bbacfed3d107186bd96368063db9d4a24b603479 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 10:09:43 +0200 Subject: [PATCH 03/45] fix(chat): Harden durable continuation recovery Preserve runnable state across leased queue work, process pending Slack mailbox records before timeout resumes, and avoid replaying already-injected Slack work after recovery. Add replay protection for already-delivered Slack replies and a high-water timeout slice cap so pathological continuations fail instead of scheduling forever. Refs GH-470 Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/runtime/reply-executor.ts | 6 + .../src/chat/runtime/turn-preparation.ts | 5 + .../src/chat/services/turn-session-record.ts | 57 ++++- .../src/chat/task-execution/slack-work.ts | 29 +-- .../junior/src/chat/task-execution/store.ts | 29 ++- .../junior/src/chat/task-execution/worker.ts | 11 +- .../integration/conversation-work.test.ts | 215 ++++++++++++++++++ .../integration/slack/bot-handlers.test.ts | 79 +++++++ .../unit/services/turn-session-record.test.ts | 51 +++++ specs/agent-session-resumability.md | 5 +- specs/task-execution.md | 5 +- 11 files changed, 438 insertions(+), 54 deletions(-) diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index b79e673f9..3afcd49f7 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -425,6 +425,12 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { return; } } + if (preparedState.userMessageAlreadyReplied) { + await persistThreadState(thread, { + conversation: preparedState.conversation, + }); + return; + } const configReply = await maybeApplyProviderDefaultConfigRequest({ channelConfiguration: preparedState.channelConfiguration, requesterId: message.author.userId, diff --git a/packages/junior/src/chat/runtime/turn-preparation.ts b/packages/junior/src/chat/runtime/turn-preparation.ts index d0aa180f8..741231a42 100644 --- a/packages/junior/src/chat/runtime/turn-preparation.ts +++ b/packages/junior/src/chat/runtime/turn-preparation.ts @@ -49,6 +49,7 @@ export interface PreparedTurnState { conversationContext?: string; sandboxId?: string; sandboxDependencyProfileHash?: string; + userMessageAlreadyReplied?: boolean; userMessageId?: string; } @@ -205,6 +206,9 @@ export function createPrepareTurnState(deps: PrepareTurnStateDeps) { explicitMention: args.explicitMention, text: args.text.userText, }); + const userMessageAlreadyReplied = conversation.messages.some( + (entry) => entry.id === incomingUserMessage.id && entry.meta?.replied, + ); const userMessageId = upsertConversationMessage( conversation, @@ -256,6 +260,7 @@ export function createPrepareTurnState(deps: PrepareTurnStateDeps) { sandboxId: existingSandboxId, sandboxDependencyProfileHash: existingSandboxDependencyProfileHash, conversationContext, + userMessageAlreadyReplied, userMessageId, }; }; diff --git a/packages/junior/src/chat/services/turn-session-record.ts b/packages/junior/src/chat/services/turn-session-record.ts index 8ed0296ff..e9e37f550 100644 --- a/packages/junior/src/chat/services/turn-session-record.ts +++ b/packages/junior/src/chat/services/turn-session-record.ts @@ -12,6 +12,8 @@ import { } from "@/chat/respond-helpers"; import { addAgentTurnUsage, type AgentTurnUsage } from "@/chat/usage"; +export const AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES = 48; + export interface TurnSessionContext { conversationId?: string; sessionId?: string; @@ -159,7 +161,7 @@ export async function persistRunningSessionRecord(args: { ...((args.requester ?? latestSessionRecord?.requester) ? { requester: args.requester ?? latestSessionRecord?.requester } : {}), - ...(getActiveTraceId() ?? latestSessionRecord?.traceId + ...((getActiveTraceId() ?? latestSessionRecord?.traceId) ? { traceId: getActiveTraceId() ?? latestSessionRecord?.traceId } : {}), }); @@ -217,7 +219,7 @@ export async function persistCompletedSessionRecord(args: { ...((args.requester ?? latestSessionRecord?.requester) ? { requester: args.requester ?? latestSessionRecord?.requester } : {}), - ...(getActiveTraceId() ?? latestSessionRecord?.traceId + ...((getActiveTraceId() ?? latestSessionRecord?.traceId) ? { traceId: getActiveTraceId() ?? latestSessionRecord?.traceId } : {}), }); @@ -290,7 +292,7 @@ export async function persistAuthPauseSessionRecord(args: { ...((args.requester ?? latestSessionRecord?.requester) ? { requester: args.requester ?? latestSessionRecord?.requester } : {}), - ...(getActiveTraceId() ?? latestSessionRecord?.traceId + ...((getActiveTraceId() ?? latestSessionRecord?.traceId) ? { traceId: getActiveTraceId() ?? latestSessionRecord?.traceId } : {}), }); @@ -340,19 +342,50 @@ export async function persistTimeoutSessionRecord(args: { if (piMessages.length === 0 || !isContinuableBoundary(piMessages)) { return undefined; } + const cumulativeDurationMs = addDurationMs( + latestSessionRecord?.cumulativeDurationMs, + args.currentDurationMs, + ); + const cumulativeUsage = addAgentTurnUsage( + latestSessionRecord?.cumulativeUsage, + args.currentUsage, + ); + if (nextSliceId > AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES) { + await upsertAgentTurnSessionRecord({ + ...((args.channelName ?? latestSessionRecord?.channelName) + ? { + channelName: args.channelName ?? latestSessionRecord?.channelName, + } + : {}), + conversationId: args.conversationId, + cumulativeDurationMs, + cumulativeUsage, + sessionId: args.sessionId, + sliceId: args.currentSliceId, + state: "failed", + piMessages, + ...(args.loadedSkillNames + ? { loadedSkillNames: args.loadedSkillNames } + : {}), + resumeReason: "timeout", + resumedFromSliceId: latestSessionRecord?.resumedFromSliceId, + errorMessage: `Turn exceeded timeout resume slice limit (${AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES})`, + ...((args.requester ?? latestSessionRecord?.requester) + ? { requester: args.requester ?? latestSessionRecord?.requester } + : {}), + ...((getActiveTraceId() ?? latestSessionRecord?.traceId) + ? { traceId: getActiveTraceId() ?? latestSessionRecord?.traceId } + : {}), + }); + return undefined; + } return await upsertAgentTurnSessionRecord({ ...((args.channelName ?? latestSessionRecord?.channelName) ? { channelName: args.channelName ?? latestSessionRecord?.channelName } : {}), conversationId: args.conversationId, - cumulativeDurationMs: addDurationMs( - latestSessionRecord?.cumulativeDurationMs, - args.currentDurationMs, - ), - cumulativeUsage: addAgentTurnUsage( - latestSessionRecord?.cumulativeUsage, - args.currentUsage, - ), + cumulativeDurationMs, + cumulativeUsage, sessionId: args.sessionId, sliceId: nextSliceId, state: "awaiting_resume", @@ -366,7 +399,7 @@ export async function persistTimeoutSessionRecord(args: { ...((args.requester ?? latestSessionRecord?.requester) ? { requester: args.requester ?? latestSessionRecord?.requester } : {}), - ...(getActiveTraceId() ?? latestSessionRecord?.traceId + ...((getActiveTraceId() ?? latestSessionRecord?.traceId) ? { traceId: getActiveTraceId() ?? latestSessionRecord?.traceId } : {}), }); diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 404ba830c..679a91c72 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -23,7 +23,6 @@ import type { } from "@/chat/task-execution/store"; import { getConversationWorkState, - countPendingConversationMessages, markConversationMessagesInjected, } from "@/chat/task-execution/store"; import type { @@ -163,19 +162,6 @@ async function resumeAwaitingTimeout(conversationId: string): Promise { return true; } -async function getCrashRecoveryRecords(args: { - conversationId: string; - state?: StateAdapter; -}): Promise { - const work = await getConversationWorkState(args); - if (!work || countPendingConversationMessages(work) > 0) { - return []; - } - return work.messages - .filter((message) => message.source === "slack") - .sort(compareInboundMessages); -} - function getInstallation( records: InboundMessageRecord[], ): SlackInstallationContext { @@ -208,12 +194,6 @@ export function createSlackConversationWorker( const state = getConnectedState(options.state); await state.connect(); - if (await resumeAwaitingTimeout(context.conversationId)) { - return context.shouldYield() - ? { status: "yielded" } - : { status: "completed" }; - } - let records = getPendingRecords( await getConversationWorkState({ conversationId: context.conversationId, @@ -221,12 +201,9 @@ export function createSlackConversationWorker( }), ); if (records.length === 0) { - records = await getCrashRecoveryRecords({ - conversationId: context.conversationId, - state, - }); - } - if (records.length === 0) { + if (await resumeAwaitingTimeout(context.conversationId)) { + return { status: "completed" }; + } return { status: "completed" }; } diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index 6ea2aae4a..7b3d3395d 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -519,8 +519,9 @@ export async function requestConversationWork(args: { }): Promise { const nowMs = args.nowMs ?? now(); return await withConversationMutation(args, async (state) => { + const existing = await readWorkState(state, args.conversationId); const current = - (await readWorkState(state, args.conversationId)) ?? + existing ?? emptyWorkState({ conversationId: args.conversationId, nowMs, @@ -530,7 +531,7 @@ export async function requestConversationWork(args: { needsRun: true, updatedAtMs: nowMs, }); - return { status: current === undefined ? "created" : "updated" }; + return { status: existing === undefined ? "created" : "updated" }; }); } @@ -563,7 +564,7 @@ export async function startConversationWork(args: { const nowMs = args.nowMs ?? now(); return await withConversationMutation(args, async (state) => { const current = await readWorkState(state, args.conversationId); - if (!current || !hasRunnableWork(current)) { + if (!current) { return { status: "no_work" }; } if (isLeaseActive(current.lease, nowMs)) { @@ -572,6 +573,9 @@ export async function startConversationWork(args: { leaseExpiresAtMs: current.lease!.leaseExpiresAtMs, }; } + if (!hasRunnableWork(current)) { + return { status: "no_work" }; + } const lease: ConversationLease = { leaseToken: randomUUID(), @@ -582,7 +586,7 @@ export async function startConversationWork(args: { await writeWorkState(state, { ...current, lease, - needsRun: true, + needsRun: false, updatedAtMs: nowMs, }); return { @@ -644,13 +648,18 @@ export async function drainConversationMailbox(args: { const drainedIds = new Set( pending.map((message) => message.inboundMessageId), ); + const messages = current.messages.map((message) => + drainedIds.has(message.inboundMessageId) + ? { ...message, injectedAtMs: nowMs } + : message, + ); + const hasPending = messages.some( + (message) => message.injectedAtMs === undefined, + ); await writeWorkState(state, { ...current, - messages: current.messages.map((message) => - drainedIds.has(message.inboundMessageId) - ? { ...message, injectedAtMs: nowMs } - : message, - ), + messages, + needsRun: hasPending, updatedAtMs: nowMs, }); return pending; @@ -761,7 +770,7 @@ export async function completeConversationWork(args: { await writeWorkState(state, { ...current, lease: undefined, - needsRun: hasPending, + needsRun: current.needsRun || hasPending, updatedAtMs: nowMs, }); return hasPending ? "pending" : "completed"; diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index 8d0eea486..c6e9887dd 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -17,7 +17,6 @@ import { export const CONVERSATION_WORK_DEFER_DELAY_MS = 15_000; export const CONVERSATION_WORK_SOFT_YIELD_AFTER_MS = 240_000; -export const CONVERSATION_WORK_MIN_NEXT_ITERATION_BUDGET_MS = 120_000; export interface ConversationWorkerContext { checkIn(): Promise; @@ -130,7 +129,9 @@ export async function processConversationWork( }); if ( !initial || - (countPendingConversationMessages(initial) === 0 && !initial.needsRun) + (countPendingConversationMessages(initial) === 0 && + !initial.needsRun && + !initial.lease) ) { return { status: "no_work" }; } @@ -268,6 +269,12 @@ export async function processConversationWork( return { status: "completed" }; } catch (error) { try { + await requestConversationContinuation({ + conversationId, + leaseToken: lease.leaseToken, + nowMs: now(options), + state: options.state, + }); await releaseConversationWork({ conversationId, leaseToken: lease.leaseToken, diff --git a/packages/junior/tests/integration/conversation-work.test.ts b/packages/junior/tests/integration/conversation-work.test.ts index 19ac7d89e..e49e59023 100644 --- a/packages/junior/tests/integration/conversation-work.test.ts +++ b/packages/junior/tests/integration/conversation-work.test.ts @@ -14,6 +14,7 @@ import { drainConversationMailbox, getConversationWorkState, markConversationMessagesInjected, + requestConversationWork, startConversationWork, type InboundMessageRecord, } from "@/chat/task-execution/store"; @@ -27,6 +28,7 @@ import { processConversationQueueMessage } from "@/chat/task-execution/vercel-ca import { createVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; vi.hoisted(() => { process.env.JUNIOR_STATE_ADAPTER = "memory"; @@ -241,6 +243,32 @@ describe("conversation work execution", () => { await expect(first).resolves.toEqual({ status: "completed" }); }); + it("preserves work requested while a lease is running", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + run: async (context) => { + await context.drainMailbox(async () => {}); + await requestConversationWork({ + conversationId: context.conversationId, + nowMs: 2_000, + }); + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "completed" }); + + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(state?.lease).toBeUndefined(); + expect(state?.needsRun).toBe(true); + expect(state ? countPendingConversationMessages(state) : 0).toBe(0); + }); + it("drains pending messages and completes the leased conversation", async () => { const queue = new FakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -388,6 +416,35 @@ describe("conversation work execution", () => { expect(injected).toEqual([["m1"], ["m2"]]); }); + it("clears the run marker after draining messages that arrived during active execution", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + run: async (context) => { + await context.drainMailbox(async () => {}); + await appendInboundMessage({ + message: inboundMessage("m2", { + createdAtMs: 2_000, + receivedAtMs: 2_100, + }), + nowMs: 2_100, + }); + await context.drainMailbox(async () => {}); + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "completed" }); + + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(state?.needsRun).toBe(false); + expect(state ? countPendingConversationMessages(state) : 0).toBe(0); + }); + it("requeues instead of completing when final mailbox work remains", async () => { const queue = new FakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -700,6 +757,164 @@ describe("conversation work execution", () => { expect(work ? countPendingConversationMessages(work) : 0).toBe(0); }); + it("processes pending Slack follow-ups before timeout continuation", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + await upsertAgentTurnSessionRecord({ + conversationId: CONVERSATION_ID, + sessionId: "turn-timeout", + sliceId: 2, + state: "awaiting_resume", + resumeReason: "timeout", + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "original request" }], + timestamp: 1_000, + }, + ], + }); + + await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> follow-up`, + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + waitUntil: () => {}, + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }, + }); + + const calls: string[] = []; + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (_thread, message) => { + calls.push(message.text); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(calls).toEqual([expect.stringContaining("follow-up")]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + }); + + it("does not replay injected Slack mailbox records after lease recovery", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + + await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + waitUntil: () => {}, + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }, + }); + const lease = await startConversationWork({ + conversationId: CONVERSATION_ID, + nowMs: 2_000, + state, + }); + expect(lease.status).toBe("acquired"); + if (lease.status !== "acquired") { + return; + } + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + const inboundMessageIds = + work?.messages.map((message) => message.inboundMessageId) ?? []; + await markConversationMessagesInjected({ + conversationId: CONVERSATION_ID, + inboundMessageIds, + leaseToken: lease.leaseToken, + nowMs: 3_000, + state, + }); + await recoverConversationWork({ + nowMs: 2_000 + CONVERSATION_WORK_LEASE_TTL_MS, + queue, + state, + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + throw new Error("injected messages should not replay"); + }, + handleSubscribedMessage: async () => { + throw new Error("injected messages should not replay"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "no_work" }); + + const recovered = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(recovered?.needsRun).toBe(false); + expect(recovered ? countPendingConversationMessages(recovered) : 0).toBe(0); + }); + it("keeps Slack mailbox records pending when the runtime handoff fails", async () => { const queue = new FakeQueue(); const state = getStateAdapter(); diff --git a/packages/junior/tests/integration/slack/bot-handlers.test.ts b/packages/junior/tests/integration/slack/bot-handlers.test.ts index 9e0267352..1231103b2 100644 --- a/packages/junior/tests/integration/slack/bot-handlers.test.ts +++ b/packages/junior/tests/integration/slack/bot-handlers.test.ts @@ -152,6 +152,85 @@ describe("bot handlers (integration)", () => { expect(hasReply).toBe(true); }); + it("does not replay a message that already has a delivered reply", async () => { + const conversationId = "slack:C_REPLAY:1700000000.000"; + const generateAssistantReply = vi.fn(); + const { slackRuntime } = createRuntime({ + services: { + replyExecutor: { + generateAssistantReply, + }, + }, + }); + const thread = createTestThread({ + id: conversationId, + state: { + conversation: { + schemaVersion: 1, + backfill: { + completedAtMs: 1, + source: "recent_messages", + }, + compactions: [], + piMessages: [], + messages: [ + { + id: "msg-replayed", + role: "user", + text: "please answer once", + createdAtMs: 1, + author: { + userId: "U-test", + }, + meta: { + replied: true, + slackTs: "1700000000.000", + }, + }, + { + id: "assistant-reply", + role: "assistant", + text: "Already answered.", + createdAtMs: 2, + author: { + isBot: true, + userName: "Junior", + }, + meta: { + replied: true, + }, + }, + ], + processing: {}, + stats: { + compactedMessageCount: 0, + estimatedContextTokens: 0, + totalMessageCount: 2, + updatedAtMs: 2, + }, + vision: { + byFileId: {}, + }, + }, + }, + }); + + await expect( + slackRuntime.handleNewMention( + thread, + createTestMessage({ + id: "msg-replayed", + threadId: conversationId, + text: "please answer once", + isMention: true, + }), + ), + ).resolves.toBeUndefined(); + + expect(generateAssistantReply).not.toHaveBeenCalled(); + expect(thread.posts).toEqual([]); + }); + it("handleSubscribedMessage with explicit mention: replies when should_reply is true", async () => { const { slackRuntime } = createTestChatRuntime({ services: { diff --git a/packages/junior/tests/unit/services/turn-session-record.test.ts b/packages/junior/tests/unit/services/turn-session-record.test.ts index 08bab5bbe..cc5f28651 100644 --- a/packages/junior/tests/unit/services/turn-session-record.test.ts +++ b/packages/junior/tests/unit/services/turn-session-record.test.ts @@ -205,6 +205,57 @@ describe("persistAuthPauseSessionRecord", () => { }); }); + it("fails timeout sessions instead of scheduling beyond the slice cap", async () => { + const { + AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES, + persistTimeoutSessionRecord, + } = await import("@/chat/services/turn-session-record"); + const { getAgentTurnSessionRecord, upsertAgentTurnSessionRecord } = + await import("@/chat/state/turn-session"); + + const piMessages: PiMessage[] = [ + { + role: "user", + content: [{ type: "text", text: "keep trying" }], + timestamp: 1, + }, + ]; + + await upsertAgentTurnSessionRecord({ + conversationId: "conversation-timeout-cap", + sessionId: "turn-timeout-cap", + sliceId: AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES, + state: "awaiting_resume", + piMessages, + resumeReason: "timeout", + cumulativeDurationMs: 12_000, + }); + + await expect( + persistTimeoutSessionRecord({ + conversationId: "conversation-timeout-cap", + sessionId: "turn-timeout-cap", + currentSliceId: AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES, + currentDurationMs: 3_000, + messages: piMessages, + errorMessage: "timed out again", + logContext: { + modelId: "test-model", + }, + }), + ).resolves.toBeUndefined(); + + await expect( + getAgentTurnSessionRecord("conversation-timeout-cap", "turn-timeout-cap"), + ).resolves.toMatchObject({ + state: "failed", + sliceId: AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES, + cumulativeDurationMs: 15_000, + errorMessage: expect.stringContaining("slice limit"), + piMessages, + }); + }); + it("falls back to the last stored safe boundary when auth pause captures a non-continuable tail", async () => { const { persistAuthPauseSessionRecord } = await import("@/chat/services/turn-session-record"); diff --git a/specs/agent-session-resumability.md b/specs/agent-session-resumability.md index 2c30bb171..830d081e7 100644 --- a/specs/agent-session-resumability.md +++ b/specs/agent-session-resumability.md @@ -46,7 +46,8 @@ This spec owns how agent session state is persisted and resumed across execution decide where to resume; the reduced conversation session log is the resume source. - `slice_id`: Diagnostic integer for one execution chunk in the same - conversation. The first-pass mailbox worker must not enforce a slice cap. + conversation. The mailbox worker must not enforce a slice cap; timeout + poison-work guards live in turn-session persistence. - `event_id`: Stable identity for one durable session-log event. - `pause_event_id`: Event id carried by timeout/auth resume callbacks so stale callbacks can be dropped. @@ -506,7 +507,7 @@ The worker must: 2. Queue nudge is never delivered after a safe boundary append: heartbeat finds pending mailbox or expired lease state and enqueues the conversation id. 3. Duplicate queue nudges for the same conversation are serialized by the conversation lease. 4. Timeout after visible assistant output begins: automatic continuation is skipped to avoid duplicate/corrupt user-visible output. -5. Repeated cooperative yields before visible output may produce further execution chunks, but first pass does not enforce a slice cap. +5. Repeated cooperative yields before visible output may produce further execution chunks, but timeout continuation must stop at the configured high-water slice cap and mark the session failed instead of scheduling another queue nudge. 6. A later user message after an ungraceful crash may build its prompt history from the active session's latest reduced Pi projection. If the prior session produced assistant text that was not committed to visible thread state, that trailing assistant text must be trimmed from the fresh-turn history view. ## Observability diff --git a/specs/task-execution.md b/specs/task-execution.md index 129c01118..31e1eab04 100644 --- a/specs/task-execution.md +++ b/specs/task-execution.md @@ -34,8 +34,9 @@ invocations without turning every tool call into a queue round trip. - Queueing every model call or every tool call as a separate asynchronous task. - Exactly-once external side-effect delivery. - Mid-model-stream or mid-tool-call checkpointing. -- First-pass max turn age, max slice count, or poison-work policy. These are - TODO guardrails after the mailbox worker is in production. +- Owning model-execution poison-work policy. Timeout slice caps belong to + `./agent-session-resumability.md`; this layer only requeues or releases + conversation work based on durable runnable state. - Using Slack thread messages as progress filler for routine continuation. ## Contracts From 4b4b4d0dc844c86b65eab405e3b39ada453037fb Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 11:14:04 +0200 Subject: [PATCH 04/45] fix(chat): Requeue runnable conversation work Preserve runnable conversation state when a leased worker completes after needsRun was marked during execution. This keeps continuation and late work recovery from waiting on heartbeat repair. Scope worker and heartbeat queue idempotency keys to a specific wake-up attempt so provider dedupe cannot suppress later legitimate recovery nudges. Move deterministic worker, lease, mailbox, and timeout-resume coverage into component tests and document the layer boundary. Refs GH-470 Co-Authored-By: Codex GPT-5 --- packages/junior/package.json | 2 +- .../src/chat/task-execution/heartbeat.ts | 15 +++- .../junior/src/chat/task-execution/store.ts | 7 +- .../junior/src/chat/task-execution/worker.ts | 35 +++++--- .../runtime/timeout-resume.test.ts | 31 +++++-- .../task-execution}/conversation-work.test.ts | 87 +++++++++++++++++-- packages/junior/vitest.config.ts | 1 + specs/agent-session-resumability.md | 12 +-- specs/component-testing.md | 69 +++++++++++++++ specs/index.md | 1 + specs/integration-testing.md | 9 +- specs/task-execution.md | 54 +++++++----- specs/testing.md | 42 +++++---- specs/unit-testing.md | 5 +- 14 files changed, 283 insertions(+), 87 deletions(-) rename packages/junior/tests/{unit => component}/runtime/timeout-resume.test.ts (85%) rename packages/junior/tests/{integration => component/task-execution}/conversation-work.test.ts (92%) create mode 100644 specs/component-testing.md diff --git a/packages/junior/package.json b/packages/junior/package.json index 2fa627afe..8c7e7c2ea 100644 --- a/packages/junior/package.json +++ b/packages/junior/package.json @@ -48,7 +48,7 @@ "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", "lint": "oxlint --config .oxlintrc.json --deny-warnings src tests scripts bin tsup.config.ts", "lint:fix": "oxlint --config .oxlintrc.json --deny-warnings --fix src tests scripts bin tsup.config.ts", - "test": "pnpm run test:slack-boundary && pnpm run test:arch-boundary && vitest run", + "test": "pnpm run test:slack-boundary && pnpm run test:arch-boundary && vitest run --maxWorkers=4", "test:watch": "vitest", "test:slack-boundary": "node scripts/check-slack-test-boundary.mjs", "test:arch-boundary": "depcruise --config .dependency-cruiser.mjs src/chat", diff --git a/packages/junior/src/chat/task-execution/heartbeat.ts b/packages/junior/src/chat/task-execution/heartbeat.ts index 15aa1ab0b..8038c78ac 100644 --- a/packages/junior/src/chat/task-execution/heartbeat.ts +++ b/packages/junior/src/chat/task-execution/heartbeat.ts @@ -20,8 +20,9 @@ export interface ConversationWorkRecoveryResult { function heartbeatIdempotencyKey( reason: string, conversationId: string, + nowMs: number, ): string { - return `heartbeat:${reason}:${conversationId}`; + return `heartbeat:${reason}:${conversationId}:${nowMs}`; } async function sendRecoveryNudge(args: { @@ -79,7 +80,11 @@ export async function recoverConversationWork(args: { } await sendRecoveryNudge({ conversationId, - idempotencyKey: heartbeatIdempotencyKey("lease", conversationId), + idempotencyKey: heartbeatIdempotencyKey( + "lease", + conversationId, + args.nowMs, + ), nowMs: args.nowMs, queue: args.queue, state: args.state, @@ -106,7 +111,11 @@ export async function recoverConversationWork(args: { await sendRecoveryNudge({ conversationId, - idempotencyKey: heartbeatIdempotencyKey("pending", conversationId), + idempotencyKey: heartbeatIdempotencyKey( + "pending", + conversationId, + args.nowMs, + ), nowMs: args.nowMs, queue: args.queue, state: args.state, diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index 7b3d3395d..692ae7741 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -753,7 +753,7 @@ export async function releaseConversationWork(args: { }); } -/** Finish a leased conversation when no pending mailbox work remains. */ +/** Finish a leased conversation and report whether runnable work remains. */ export async function completeConversationWork(args: { conversationId: string; leaseToken: string; @@ -767,13 +767,14 @@ export async function completeConversationWork(args: { return "lost_lease"; } const hasPending = pendingMessages(current).length > 0; + const hasRunnableWork = current.needsRun || hasPending; await writeWorkState(state, { ...current, lease: undefined, - needsRun: current.needsRun || hasPending, + needsRun: hasRunnableWork, updatedAtMs: nowMs, }); - return hasPending ? "pending" : "completed"; + return hasRunnableWork ? "pending" : "completed"; }); } diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index c6e9887dd..03e473b2e 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -55,8 +55,12 @@ function now(options: ProcessConversationWorkOptions): number { return options.nowMs?.() ?? Date.now(); } -function nudgeIdempotencyKey(reason: string, conversationId: string): string { - return `${reason}:${conversationId}`; +function nudgeIdempotencyKey( + reason: string, + conversationId: string, + nowMs: number, +): string { + return `${reason}:${conversationId}:${nowMs}`; } async function sendWakeNudge(args: { @@ -145,11 +149,12 @@ export async function processConversationWork( return { status: "no_work" }; } if (lease.status === "active") { + const nudgeNowMs = now(options); await sendWakeNudge({ conversationId, delayMs: CONVERSATION_WORK_DEFER_DELAY_MS, - idempotencyKey: nudgeIdempotencyKey("active", conversationId), - nowMs: now(options), + idempotencyKey: nudgeIdempotencyKey("active", conversationId, nudgeNowMs), + nowMs: nudgeNowMs, options, }); logInfo( @@ -206,10 +211,11 @@ export async function processConversationWork( try { const result = await options.run(workerContext); if (result.status === "yielded") { + const yieldNowMs = now(options); const continuationMarked = await requestConversationContinuation({ conversationId, leaseToken: lease.leaseToken, - nowMs: now(options), + nowMs: yieldNowMs, state: options.state, }); if (!continuationMarked) { @@ -217,14 +223,18 @@ export async function processConversationWork( } await sendWakeNudge({ conversationId, - idempotencyKey: nudgeIdempotencyKey("yield", conversationId), - nowMs: now(options), + idempotencyKey: nudgeIdempotencyKey( + "yield", + conversationId, + yieldNowMs, + ), + nowMs: yieldNowMs, options, }); await releaseConversationWork({ conversationId, leaseToken: lease.leaseToken, - nowMs: now(options), + nowMs: yieldNowMs, state: options.state, }); logInfo( @@ -249,10 +259,15 @@ export async function processConversationWork( return { status: "lost_lease" }; } if (completion === "pending") { + const nudgeNowMs = now(options); await sendWakeNudge({ conversationId, - idempotencyKey: nudgeIdempotencyKey("pending", conversationId), - nowMs: now(options), + idempotencyKey: nudgeIdempotencyKey( + "pending", + conversationId, + nudgeNowMs, + ), + nowMs: nudgeNowMs, options, }); return { status: "pending_requeued" }; diff --git a/packages/junior/tests/unit/runtime/timeout-resume.test.ts b/packages/junior/tests/component/runtime/timeout-resume.test.ts similarity index 85% rename from packages/junior/tests/unit/runtime/timeout-resume.test.ts rename to packages/junior/tests/component/runtime/timeout-resume.test.ts index 6b540baf3..04c94f4c4 100644 --- a/packages/junior/tests/unit/runtime/timeout-resume.test.ts +++ b/packages/junior/tests/component/runtime/timeout-resume.test.ts @@ -8,6 +8,17 @@ import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { getConversationWorkState } from "@/chat/task-execution/store"; import { disconnectStateAdapter } from "@/chat/state/adapter"; +const ORIGINAL_ENV = vi.hoisted(() => { + const original = { + JUNIOR_SECRET: process.env.JUNIOR_SECRET, + JUNIOR_STATE_ADAPTER: process.env.JUNIOR_STATE_ADAPTER, + SLACK_SIGNING_SECRET: process.env.SLACK_SIGNING_SECRET, + }; + process.env.JUNIOR_STATE_ADAPTER = "memory"; + process.env.JUNIOR_SECRET = "resume-secret"; + return original; +}); + class FakeQueue implements ConversationWorkQueue { sent: Array<{ conversationId: string; @@ -45,9 +56,15 @@ function makeSignedResumeRequest(body: Record): Request { }); } -describe("timeout resume callback signing", () => { - const originalSlackSigningSecret = process.env.SLACK_SIGNING_SECRET; +function restoreEnv(name: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[name]; + return; + } + process.env[name] = value; +} +describe("timeout resume callback signing", () => { beforeEach(async () => { process.env.JUNIOR_STATE_ADAPTER = "memory"; process.env.JUNIOR_SECRET = "resume-secret"; @@ -56,13 +73,9 @@ describe("timeout resume callback signing", () => { afterEach(async () => { await disconnectStateAdapter(); - delete process.env.JUNIOR_STATE_ADAPTER; - delete process.env.JUNIOR_SECRET; - if (originalSlackSigningSecret === undefined) { - delete process.env.SLACK_SIGNING_SECRET; - } else { - process.env.SLACK_SIGNING_SECRET = originalSlackSigningSecret; - } + restoreEnv("JUNIOR_STATE_ADAPTER", ORIGINAL_ENV.JUNIOR_STATE_ADAPTER); + restoreEnv("JUNIOR_SECRET", ORIGINAL_ENV.JUNIOR_SECRET); + restoreEnv("SLACK_SIGNING_SECRET", ORIGINAL_ENV.SLACK_SIGNING_SECRET); vi.restoreAllMocks(); }); diff --git a/packages/junior/tests/integration/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts similarity index 92% rename from packages/junior/tests/integration/conversation-work.test.ts rename to packages/junior/tests/component/task-execution/conversation-work.test.ts index e49e59023..9f56dfc88 100644 --- a/packages/junior/tests/integration/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -198,7 +198,7 @@ describe("conversation work execution", () => { expect(queue.sent).toEqual([ { conversationId: CONVERSATION_ID, - idempotencyKey: `heartbeat:pending:${CONVERSATION_ID}`, + idempotencyKey: `heartbeat:pending:${CONVERSATION_ID}:62000`, }, ]); }); @@ -243,23 +243,26 @@ describe("conversation work execution", () => { await expect(first).resolves.toEqual({ status: "completed" }); }); - it("preserves work requested while a lease is running", async () => { + it("requeues work requested while a lease is running", async () => { const queue = new FakeQueue(); + let currentNowMs = 1_000; await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, queue, run: async (context) => { await context.drainMailbox(async () => {}); + currentNowMs = 2_000; await requestConversationWork({ conversationId: context.conversationId, - nowMs: 2_000, + nowMs: currentNowMs, }); return { status: "completed" }; }, }), - ).resolves.toEqual({ status: "completed" }); + ).resolves.toEqual({ status: "pending_requeued" }); const state = await getConversationWorkState({ conversationId: CONVERSATION_ID, @@ -267,6 +270,46 @@ describe("conversation work execution", () => { expect(state?.lease).toBeUndefined(); expect(state?.needsRun).toBe(true); expect(state ? countPendingConversationMessages(state) : 0).toBe(0); + expect(queue.sent).toMatchObject([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `pending:${CONVERSATION_ID}:2000`, + }, + ]); + }); + + it("uses fresh queue idempotency keys for repeated worker requeues", async () => { + const queue = new FakeQueue(); + let currentNowMs = 1_000; + await requestConversationWork({ + conversationId: CONVERSATION_ID, + nowMs: currentNowMs, + }); + + async function runSlice(nowMs: number): Promise { + currentNowMs = nowMs; + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, + queue, + run: async (context) => { + await requestConversationWork({ + conversationId: context.conversationId, + nowMs: currentNowMs, + }); + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "pending_requeued" }); + } + + await runSlice(2_000); + await runSlice(63_000); + + expect(queue.sent.map((send) => send.idempotencyKey)).toEqual([ + `pending:${CONVERSATION_ID}:2000`, + `pending:${CONVERSATION_ID}:63000`, + ]); }); it("drains pending messages and completes the leased conversation", async () => { @@ -353,7 +396,7 @@ describe("conversation work execution", () => { expect(queue.sent).toMatchObject([ { conversationId: CONVERSATION_ID, - idempotencyKey: `heartbeat:lease:${CONVERSATION_ID}`, + idempotencyKey: `heartbeat:lease:${CONVERSATION_ID}:92000`, }, ]); }); @@ -371,6 +414,29 @@ describe("conversation work execution", () => { expect(queue.sent).toHaveLength(1); }); + it("uses fresh queue idempotency keys for repeated heartbeat recovery", async () => { + const queue = new FakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + recoverConversationWork({ + nowMs: 62_000, + queue, + }), + ).resolves.toEqual({ expiredLeaseCount: 0, pendingCount: 1 }); + await expect( + recoverConversationWork({ + nowMs: 122_001, + queue, + }), + ).resolves.toEqual({ expiredLeaseCount: 0, pendingCount: 1 }); + + expect(queue.sent.map((send) => send.idempotencyKey)).toEqual([ + `heartbeat:pending:${CONVERSATION_ID}:62000`, + `heartbeat:pending:${CONVERSATION_ID}:122001`, + ]); + }); + it("runs conversation work recovery from the core heartbeat", async () => { const queue = new FakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -383,7 +449,7 @@ describe("conversation work execution", () => { expect(queue.sent).toEqual([ { conversationId: CONVERSATION_ID, - idempotencyKey: `heartbeat:pending:${CONVERSATION_ID}`, + idempotencyKey: `heartbeat:pending:${CONVERSATION_ID}:62000`, }, ]); }); @@ -447,19 +513,22 @@ describe("conversation work execution", () => { it("requeues instead of completing when final mailbox work remains", async () => { const queue = new FakeQueue(); + let currentNowMs = 1_000; await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, queue, run: async (context) => { await context.drainMailbox(async () => {}); + currentNowMs = 2_100; await appendInboundMessage({ message: inboundMessage("m2", { createdAtMs: 2_000, receivedAtMs: 2_100, }), - nowMs: 2_100, + nowMs: currentNowMs, }); return { status: "completed" }; }, @@ -468,7 +537,7 @@ describe("conversation work execution", () => { expect(queue.sent).toMatchObject([ { conversationId: CONVERSATION_ID, - idempotencyKey: `pending:${CONVERSATION_ID}`, + idempotencyKey: `pending:${CONVERSATION_ID}:2100`, }, ]); }); @@ -499,7 +568,7 @@ describe("conversation work execution", () => { expect(queue.sent).toMatchObject([ { conversationId: CONVERSATION_ID, - idempotencyKey: `yield:${CONVERSATION_ID}`, + idempotencyKey: `yield:${CONVERSATION_ID}:242000`, }, ]); }); diff --git a/packages/junior/vitest.config.ts b/packages/junior/vitest.config.ts index 83f34d6b4..13a0c0cb3 100644 --- a/packages/junior/vitest.config.ts +++ b/packages/junior/vitest.config.ts @@ -23,6 +23,7 @@ for (const envRoot of [workspaceRoot, packageRoot]) { } process.env.JUNIOR_SECRET = "junior-test-secret"; +process.env.JUNIOR_STATE_ADAPTER = "memory"; process.env.JUNIOR_STATE_KEY_PREFIX ??= `junior:test:${process.pid}`; export default defineConfig({ diff --git a/specs/agent-session-resumability.md b/specs/agent-session-resumability.md index 830d081e7..d3f149521 100644 --- a/specs/agent-session-resumability.md +++ b/specs/agent-session-resumability.md @@ -534,15 +534,15 @@ Required attributes when available: ## Verification 1. Unit: resumable boundaries trim trailing assistant-only messages when needed. -2. Unit/integration: queue-driven resume restores `agent.state.messages` and calls `continue()`. +2. Component/integration: queue-driven resume restores `agent.state.messages` and calls `continue()`. 3. Integration: a cooperative yield resumes in a later worker and reaches a successful terminal reply. 4. Integration: a user follow-up during active execution is appended to the mailbox and injected into the same conversation at the next safe boundary. -5. Unit/integration: auth-driven resume restores the same active skill/MCP tool universe before `continue()`. -6. Unit/integration: eager sandbox/artifact persistence preserves resumed tool context across execution chunks. -7. Unit/integration: fresh follow-up turns can recover Pi history from the active/last agent session log without depending on conversation-state Pi transcript mirroring. +5. Component/integration: auth-driven resume restores the same active skill/MCP tool universe before `continue()`. +6. Component/integration: eager sandbox/artifact persistence preserves resumed tool context across execution chunks. +7. Component/integration: fresh follow-up turns can recover Pi history from the active/last agent session log without depending on conversation-state Pi transcript mirroring. 8. Manual/eval: once assistant text is already visible, recovery does not auto-reconcile partial thread output. -9. Unit/integration: transient provider failures retry with `continue()` from a safe boundary and do not duplicate prior tool execution. -10. Unit/integration: successful provider activation appends one `mcp_provider_connected` event, and resume restores providers from those events. Legacy Pi-message inference is allowed only while pre-event session logs still exist. +9. Component/integration: transient provider failures retry with `continue()` from a safe boundary and do not duplicate prior tool execution. +10. Component/integration: successful provider activation appends one `mcp_provider_connected` event, and resume restores providers from those events. Legacy Pi-message inference is allowed only while pre-event session logs still exist. ## Related Specs diff --git a/specs/component-testing.md b/specs/component-testing.md new file mode 100644 index 000000000..088ef1445 --- /dev/null +++ b/specs/component-testing.md @@ -0,0 +1,69 @@ +# Component Testing Spec + +## Metadata + +- Created: 2026-06-02 +- Last Edited: 2026-06-02 + +## Intent + +Component tests validate deterministic service/runtime contracts that span more +than one module but do not need full product wiring to prove the invariant. + +Use this layer when real domain code, memory-backed state, and explicit local +ports give a clearer contract test than either a narrow unit test or a broad +integration test. + +## Scope + +In scope: + +- Durable store and state-machine contracts. +- Queue wake-up, lease, heartbeat, and worker coordination. +- Service orchestration where the dependency boundary is a small injected port. +- Adapter contracts that can be proven with a fake client or MSW without running + the full Slack/runtime path. +- Persistence behavior using the shared memory state adapter. + +## Non-Goals + +- Slack-visible behavior, final reply delivery, and Slack HTTP request contracts + that require real runtime wiring. +- Model-dependent behavior or conversational quality. +- Tests that patch production singletons or module imports to steer a user-visible + workflow. +- Exhaustive branch coverage for implementation details. + +## Substitution Policy + +Allowed: + +- Fake queue, clock, random-id, callback, and agent-runner ports. +- Shared memory-backed state adapters. +- MSW handlers when the adapter boundary itself is the contract. +- Local spies on explicit injected ports. + +Disallowed: + +- Broad dependency bags or service locators created only for tests. +- `vi.mock` of runtime modules to force unrelated branches. +- Fake Slack delivery and fake reply execution together to prove a single + user-visible outcome. Use integration or eval for that. + +## Naming and Placement + +- Preferred path: `packages/junior/tests/component/**`. +- Test titles should name the durable/service outcome, not the implementation + branch that happens to produce it. +- Keep component files focused by feature or service boundary, for example + `tests/component/task-execution/*`. + +## Required Characteristics + +1. Use real domain modules for the contract under test. +2. Keep fake ports explicit, small, and role-named. +3. Assert the externally relevant service result first, then durable state when + the durable state is part of the contract. +4. Prefer one representative failure or race case per invariant. +5. Do not promote a component test to integration solely to satisfy a coverage + checklist. diff --git a/specs/index.md b/specs/index.md index 00fb28cd7..0d820eafd 100644 --- a/specs/index.md +++ b/specs/index.md @@ -58,6 +58,7 @@ Define spec taxonomy, naming conventions, and canonical source-of-truth document - `specs/otel-semantics.md` - `specs/testing.md` - `specs/unit-testing.md` +- `specs/component-testing.md` - `specs/integration-testing.md` - `specs/eval-testing.md` - `specs/slack-http-mocking.md` diff --git a/specs/integration-testing.md b/specs/integration-testing.md index 0a0d93982..0986e9ff7 100644 --- a/specs/integration-testing.md +++ b/specs/integration-testing.md @@ -3,11 +3,11 @@ ## Metadata - Created: 2026-03-03 -- Last Edited: 2026-05-28 +- Last Edited: 2026-06-02 ## Intent -Integration tests validate real runtime wiring and Slack-facing behavior, with deterministic control only at the agent boundary. This is the default test layer for most product/runtime changes. Evals take this role only when the contract is agent-facing behavior that depends on model interpretation. +Integration tests validate real runtime wiring and Slack-facing behavior, with deterministic control only at the agent boundary. Use this layer when the contract depends on production composition, handler routing, external transport behavior, or user-visible runtime outcomes. Evals take this role only when the contract is agent-facing behavior that depends on model interpretation. ## Scope @@ -22,6 +22,7 @@ In scope: ## Non-Goals - Pure algorithmic invariants better covered by unit tests. +- Deterministic service/runtime contracts better covered by component tests. - Judge-scored conversational quality (belongs to evals). ## Required Runtime Shape @@ -65,11 +66,11 @@ Do not let low-level stream ordering or request-shape assertions dominate genera ## Classification Guidance -If a test relies on runtime module mocks to drive control-flow branches, classify it as unit (not integration). +If a test relies on runtime module mocks to drive control-flow branches, classify it as unit or component instead of integration. If the behavior under test depends on natural-language interpretation, continuity, or model choice, classify it as eval instead of integration. -If a product/runtime change can be proven with real wiring plus a deterministic fake agent, integration is the default answer. +If a product/runtime change can be proven only by real wiring plus a deterministic fake agent, integration is the right answer. If the contract is a deterministic store, worker, queue-port, lease, or service-coordination invariant, prefer a component test. Do not keep a scenario in integration solely because a fake classifier fixture is easier than writing the corresponding eval. When the real contract is ambiguous natural-language behavior or reply quality, promote it to eval. diff --git a/specs/task-execution.md b/specs/task-execution.md index 31e1eab04..1855fe02e 100644 --- a/specs/task-execution.md +++ b/specs/task-execution.md @@ -3,7 +3,7 @@ ## Metadata - Created: 2026-06-01 -- Last Edited: 2026-06-01 +- Last Edited: 2026-06-02 ## Purpose @@ -189,6 +189,12 @@ Queue consumer rules: The queue is not the state authority. A successful queue acknowledgement only means that one wake-up delivery has been handled. +Queue idempotency keys must be scoped to the source of one wake-up attempt: +the inbound message id, worker nudge timestamp, or heartbeat scan timestamp. +They must not be stable only by `conversationId` and reason, because that can +suppress a later legitimate recovery or continuation nudge inside the queue +provider's idempotency window. + The Vercel push consumer boundary is a thin adapter around the generic worker: it validates the `{ conversationId }` payload, uses `handleCallback`, and keeps the Vercel visibility timeout at 300 seconds. The internal push endpoint is @@ -435,29 +441,33 @@ authorization URLs, or unredacted private message bodies. ## Verification -Required tests: - -1. Unit: mailbox append is idempotent by `inboundMessageId`. -2. Unit/integration: enqueue failure after mailbox append is repaired by - heartbeat. -3. Integration: duplicate queue nudges do not run a conversation concurrently. -4. Integration: active-lease queue delivery defers a nudge and acknowledges. -5. Integration: worker check-in extends the lease while a long model/tool call - is in progress. -6. Integration: expired lease is cleared/requeued by heartbeat. -7. Integration: pending mailbox messages with no recent enqueue marker are - requeued by heartbeat. -8. Integration: message arriving during active execution is injected at the next - safe boundary and answered as part of the same conversation run. -9. Integration: final inbox drain prevents posting a stale answer when a new - message arrived before delivery. -10. Integration: cooperative yield near the 240 second soft deadline releases - the lease, enqueues another nudge, and does not post a continuation message. -11. Integration: recovery after death during model/tool work resumes from the - latest durable session-log boundary. +Required invariants, using the lowest layer that proves the contract: + +1. Component: mailbox append is idempotent by `inboundMessageId`. +2. Component: enqueue failure after mailbox append is repaired by heartbeat. +3. Component: duplicate queue nudges do not run a conversation concurrently. +4. Component: active-lease queue delivery defers a nudge and acknowledges. +5. Component: worker check-in extends the lease while a long model/tool call is + in progress. +6. Component: expired leases and stranded pending mailbox messages are + cleared/requeued by heartbeat. +7. Component: work requested while a lease is running is requeued immediately + when the lease completes, even if no mailbox messages are pending. +8. Component: repeated worker and heartbeat requeues use fresh queue + idempotency keys so provider dedupe cannot suppress later runnable work. +9. Component: messages that arrive during active execution are injected at the + next safe boundary or requeued instead of being lost. +10. Component: final inbox drain prevents completing a stale answer when new work + arrived before delivery. +11. Component: cooperative yield near the soft deadline releases the lease and + enqueues another nudge. 12. Integration: Slack ingress returns after durable mailbox append and enqueue, not after agent execution. -13. Evals: realistic multi-message Slack follow-ups during long work are folded +13. Integration: a queue-driven Slack worker path reaches the real Slack runtime + and finalized delivery with deterministic fake-agent output. +14. Component/integration: recovery after death during model/tool work resumes + from the latest durable session-log boundary. +15. Evals: realistic multi-message Slack follow-ups during long work are folded into the active answer without losing user intent. ## Related Specs diff --git a/specs/testing.md b/specs/testing.md index 242ab8a7a..345fe3900 100644 --- a/specs/testing.md +++ b/specs/testing.md @@ -3,7 +3,7 @@ ## Metadata - Created: 2026-03-03 -- Last Edited: 2026-05-28 +- Last Edited: 2026-06-02 ## Purpose @@ -14,23 +14,26 @@ Use this file as the source of truth for where a test belongs and what it is all Start from the real product contract, not the easiest seam to mock. -1. Use integration tests for most product/runtime changes: real wiring, handler behavior, Slack-facing contracts, persistence, routing, auth resumes, and other user-visible behavior that does not depend on the model interpreting language correctly. -2. Use evals when the contract is agent-facing behavior. Treat evals as the integration-style layer for prompt behavior, natural-language routing, continuity, reply quality, and other outcomes where model interpretation is the contract. -3. Use unit tests only for tightly local deterministic logic: parsing, scoring, routing heuristics, retry math, pure transforms, normalization, and similar algorithmic invariants. +1. Use evals when the contract is agent-facing behavior. Treat evals as the integration-style layer for prompt behavior, natural-language routing, continuity, reply quality, and other outcomes where model interpretation is the contract. +2. Use integration tests when the contract is real product wiring or an external/user-visible boundary: handler behavior, Slack-facing contracts, persistence/routing through production composition, auth resumes, final delivery, and other behavior that does not depend on the model interpreting language correctly. +3. Use component tests for deterministic service/runtime contracts that cross modules but are still best proven through explicit local ports: durable stores, queue wake-up ports, worker state machines, lease/recovery coordination, persistence adapters, and similar orchestration invariants. +4. Use unit tests only for tightly local deterministic logic: parsing, scoring, routing heuristics, retry math, pure transforms, normalization, and similar algorithmic invariants. -Do not default to unit tests for runtime behavior just because they are easier to write. +Do not default to unit tests for runtime behavior just because they are easier to write. Do not force deterministic service contracts into broad integration tests when a component test would prove the invariant more directly and with fewer fake layers. ## Test Layers -| Layer | Primary Goal | Scope | Allowed Substitutions | Disallowed | -| --------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | -| Unit | Validate local deterministic invariants | Single module/function and tight collaborators | Local stubs/mocks (`vi.mock`, fakes) | Baseline product/runtime behavior, Slack HTTP contract assertions, and conversational quality scoring | -| Integration | Validate runtime/product behavior and external contracts | Real app wiring + Slack-facing behavior + persistence/routing boundaries | Deterministic fake agent at the agent boundary only | Runtime module/function mocks for behavior paths | -| Eval (Agent Behavior) | Validate agent-facing conversational outcomes end-to-end | End-to-end harnessed conversation flows scored by judge criteria | Case-level behavior fixtures and controlled environment flags | Low-level HTTP payload-shape assertions and internals-only checks | +| Layer | Primary Goal | Scope | Allowed Substitutions | Disallowed | +| --------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| Unit | Validate local deterministic invariants | Single module/function and tight collaborators | Local stubs/mocks (`vi.mock`, fakes) | Baseline product/runtime behavior, Slack HTTP contract assertions, and conversational quality scoring | +| Component | Validate deterministic service/runtime contracts | Real domain modules plus memory state and explicit local ports | Fake queue/clock/agent-runner ports, memory adapters, MSW for adapter contracts | User-visible Slack delivery flows, model interpretation, broad runtime module mocks | +| Integration | Validate runtime/product behavior and external contracts | Real app wiring + Slack-facing behavior + persistence/routing boundaries | Deterministic fake agent at the agent boundary only | Runtime module/function mocks for behavior paths | +| Eval (Agent Behavior) | Validate agent-facing conversational outcomes end-to-end | End-to-end harnessed conversation flows scored by judge criteria | Case-level behavior fixtures and controlled environment flags | Low-level HTTP payload-shape assertions and internals-only checks | ## Canonical Specs - Unit rules: `./unit-testing.md` +- Component rules: `./component-testing.md` - Integration rules: `./integration-testing.md` - Evals rules: `./eval-testing.md` - Slack HTTP fixture/MSW details: `./slack-http-mocking.md` @@ -38,7 +41,7 @@ Do not default to unit tests for runtime behavior just because they are easier t ## Shared Rules Across All Layers -Layer selection is mandatory: classify the test contract first and choose `unit` vs `integration` vs `eval` before writing assertions. +Layer selection is mandatory: classify the test contract first and choose `unit`, `component`, `integration`, or `eval` before writing assertions. 1. Tests must be deterministic and isolated. 2. External HTTP is blocked by default in tests and evals; use MSW or the shared HTTP interceptor fixtures. Local URLs, model endpoints, and Vercel sandbox/OIDC control-plane traffic are the only live exceptions. @@ -77,10 +80,12 @@ Ask these questions in order: 1. Does the contract depend on the model choosing the right behavior or producing the right reply quality? Use `eval`. Examples: natural-language routing, passive participation, multi-turn continuity, prompt/skill behavior, research-answer shape. -2. Otherwise, is this a product/runtime change with real wiring, Slack-visible behavior, persistence, auth resume, or API contract effects? +2. Otherwise, is this a product/runtime change whose contract is real wiring, Slack-visible behavior, auth resume, final delivery, or API contract effects? Use `integration`. - This is the default answer for most non-trivial product changes. -3. Otherwise, is the contract tightly local deterministic logic or an algorithmic invariant? +3. Otherwise, is this a deterministic service/runtime contract that crosses modules but can be proven through real domain code plus explicit local ports? + Use `component`. + Examples: mailbox/lease state machines, queue wake-up coordination, heartbeat repair, persistence adapter contracts, workflow runner decisions, and service orchestration with fake queue/clock/agent-runner ports. +4. Otherwise, is the contract tightly local deterministic logic or an algorithmic invariant? Use `unit`. Examples: retry math, pure transforms, normalization, local scoring, parser behavior, deterministic state transitions. @@ -92,13 +97,14 @@ These rules are mandatory whenever mocks or fakes appear in a test. 1. Mock one boundary, not a whole workflow. 2. The mocked boundary must be the thing the layer is explicitly allowed to replace. -3. If a test needs to fake persisted state, Slack delivery, and reply execution together to prove one user-visible outcome, move it to integration or eval. -4. If the same user-visible contract is already covered by a higher-fidelity integration or eval test, narrow the mocked test to a local invariant or delete it. -5. Prefer real memory-backed state and the shared Slack/MSW harness over ad-hoc `Map` stores when the behavior crosses handler/runtime boundaries. +3. If a component test needs fake ports, keep them explicit and role-named. Do not use module-level mocks to steer unrelated runtime branches. +4. If a test needs to fake persisted state, Slack delivery, and reply execution together to prove one user-visible outcome, move it to integration or eval. +5. If the same user-visible contract is already covered by a higher-fidelity integration or eval test, narrow the mocked test to a local invariant or delete it. +6. Prefer real memory-backed state and the shared Slack/MSW harness over ad-hoc `Map` stores when the behavior crosses handler/runtime boundaries. ## Enforcement -`pnpm --filter @sentry/junior run test:slack-boundary` enforces major boundary rules: +`pnpm --filter @sentry/junior run test:slack-boundary` enforces major Slack boundary rules for designated integration behavior tests: - Eval files cannot import Slack contract internals. - Integration behavior tests cannot use runtime module mocks. diff --git a/specs/unit-testing.md b/specs/unit-testing.md index 2e53efc89..0100cb9fe 100644 --- a/specs/unit-testing.md +++ b/specs/unit-testing.md @@ -3,7 +3,7 @@ ## Metadata - Created: 2026-03-03 -- Last Edited: 2026-05-28 +- Last Edited: 2026-06-02 ## Intent @@ -20,6 +20,7 @@ In scope: ## Non-Goals - Real handler/runtime flows that rebuild thread state, call Slack APIs, or exercise multi-module orchestration. +- Deterministic multi-module service contracts better covered by component tests. - Slack HTTP request/response contract validation. - Full runtime Slack event handling behavior. - Conversational quality and multi-turn judge-scored outcomes. @@ -38,7 +39,7 @@ Recommended: - Assert behavior at module outputs rather than internal calls where practical. - Do not treat logger or tracer calls as required behavior unless the test is explicitly validating instrumentation. - Do not unit test prompt builders by asserting exact or substring prompt prose. If prompt wording matters, cover the resulting user-visible behavior with evals or integration tests. -- If a test has to mock large parts of the runtime or Slack client to prove a user-visible flow, reclassify it as integration or eval instead of growing the unit seam. +- If a test has to mock large parts of the runtime or Slack client to prove a user-visible flow, reclassify it as component, integration, or eval instead of growing the unit seam. ## Data and Fixtures From 2a945013d2d3a95e099e34430ba3ae6854160b43 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 11:29:23 +0200 Subject: [PATCH 05/45] fix(slack): Ack App Home publish failures Treat app_home_opened view publishing as a best-effort side effect so transient Slack API failures do not make the Events API webhook return 500 and trigger repeated Slack retries. Add a Slack webhook integration test that drives a signed app_home_opened event through the real ingress path while views.publish fails. Refs GH-470 Co-Authored-By: Codex GPT-5 --- .../junior/src/chat/ingress/slack-webhook.ts | 20 +++- .../slack/app-home-webhook.test.ts | 104 ++++++++++++++++++ 2 files changed, 119 insertions(+), 5 deletions(-) create mode 100644 packages/junior/tests/integration/slack/app-home-webhook.test.ts diff --git a/packages/junior/src/chat/ingress/slack-webhook.ts b/packages/junior/src/chat/ingress/slack-webhook.ts index 1f076fd6d..2eac85269 100644 --- a/packages/junior/src/chat/ingress/slack-webhook.ts +++ b/packages/junior/src/chat/ingress/slack-webhook.ts @@ -269,6 +269,20 @@ async function handleSlackEvent(args: { const installation = installationFromEnvelope(args.body); const receivedAtMs = Date.now(); + async function publishAppHomeViewBestEffort(userId: string): Promise { + try { + await publishAppHomeView( + getSlackClient(), + userId, + createUserTokenStore(), + ); + } catch (error) { + logException(error, "slack_app_home_publish_failed", { + slackUserId: userId, + }); + } + } + await runWithWorkspaceTeamId(installation.teamId, () => runWithSlackInstallation({ adapter, @@ -339,11 +353,7 @@ async function handleSlackEvent(args: { } if (event.type === "app_home_opened" && event.user) { - await publishAppHomeView( - getSlackClient(), - event.user, - createUserTokenStore(), - ); + await publishAppHomeViewBestEffort(event.user); return; } diff --git a/packages/junior/tests/integration/slack/app-home-webhook.test.ts b/packages/junior/tests/integration/slack/app-home-webhook.test.ts new file mode 100644 index 000000000..9543aecb8 --- /dev/null +++ b/packages/junior/tests/integration/slack/app-home-webhook.test.ts @@ -0,0 +1,104 @@ +import { createHmac } from "node:crypto"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { createMemoryState } from "@chat-adapter/state-memory"; +import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; +import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; +import { + getCapturedSlackApiCalls, + queueSlackApiError, + resetSlackApiMockState, +} from "../../msw/handlers/slack-api"; + +const SIGNING_SECRET = "test-signing-secret"; +const BOT_USER_ID = "U_BOT"; +const ORIGINAL_ENV = { ...process.env }; + +function signSlackBody(body: string, timestamp: string): string { + return `v0=${createHmac("sha256", SIGNING_SECRET) + .update(`v0:${timestamp}:${body}`) + .digest("hex")}`; +} + +function slackWebhookRequest(body: unknown): Request { + const serialized = JSON.stringify(body); + const timestamp = String(Math.floor(Date.now() / 1000)); + return new Request("https://example.test/api/webhooks/slack", { + method: "POST", + headers: { + "content-type": "application/json", + "x-slack-request-timestamp": timestamp, + "x-slack-signature": signSlackBody(serialized, timestamp), + }, + body: serialized, + }); +} + +describe("Slack webhook: App Home events", () => { + beforeEach(() => { + process.env = { + ...ORIGINAL_ENV, + SLACK_BOT_TOKEN: "xoxb-test-token", + }; + resetSlackApiMockState(); + }); + + afterEach(async () => { + process.env = { ...ORIGINAL_ENV }; + resetSlackApiMockState(); + await disconnectStateAdapter(); + }); + + it("acknowledges app_home_opened when publishing the view fails", async () => { + queueSlackApiError("views.publish", { + error: "internal_error", + status: 200, + }); + + const state = createMemoryState(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test-token", + botUserId: BOT_USER_ID, + signingSecret: SIGNING_SECRET, + }); + + const response = await handleSlackWebhook({ + request: slackWebhookRequest({ + team_id: "T123", + type: "event_callback", + event: { + type: "app_home_opened", + user: "U123", + event_ts: "1712345.0001", + }, + }), + waitUntil: () => {}, + services: { + getSlackAdapter: () => slackAdapter, + queue: { + send: async () => { + throw new Error("app_home_opened should not enqueue work"); + }, + }, + runtime: { + handleAssistantContextChanged: async () => { + throw new Error("unexpected assistant context callback"); + }, + handleAssistantThreadStarted: async () => { + throw new Error("unexpected assistant thread callback"); + }, + handleNewMention: async () => { + throw new Error("unexpected mention callback"); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed message callback"); + }, + }, + state, + }, + }); + + expect(response.status).toBe(200); + expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(1); + }); +}); From 607d52539490960511fab3be2d96ee917ea9277e Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 13:16:01 +0200 Subject: [PATCH 06/45] fix(slack): Align restored thread route context Derive restored Slack thread subscription context from the promoted batch route used for dispatch. Mixed queued batches now pass a mention-context thread into the mention handler instead of inheriting subscribed context from the latest message metadata. Add a component regression that persists a mention plus a subscribed follow-up, verifies the mixed mailbox routes, and checks the restored thread context observed by the runtime. Refs GH-470 Co-Authored-By: Codex GPT-5 --- .../src/chat/task-execution/slack-work.ts | 5 +- .../task-execution/conversation-work.test.ts | 96 +++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 679a91c72..ddbbc52ac 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -232,9 +232,10 @@ export function createSlackConversationWorker( if (!latestMessage) { return; } + const route = routeForRecords(records); const thread = restoreThread({ adapter, - isSubscribedContext: latestMetadata.route === "subscribed", + isSubscribedContext: route === "subscribed", message: latestMessage, state, threadJson: latestMetadata.thread, @@ -245,7 +246,7 @@ export function createSlackConversationWorker( totalSinceLastHandler: messages.length, }; - if (routeForRecords(records) === "mention") { + if (route === "mention") { await options.runtime.handleNewMention(thread, latestMessage, { messageContext, }); diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index 9f56dfc88..1e8f4768c 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -826,6 +826,102 @@ describe("conversation work execution", () => { expect(work ? countPendingConversationMessages(work) : 0).toBe(0); }); + it("keeps restored thread context aligned with promoted mention routing", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + const calls: Array<{ + message: Message; + skipped: Message[]; + thread: Thread; + }> = []; + const subscribedValues: boolean[] = []; + const ingressServices = { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }; + + await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + ts: "1712345.0001", + }), + ), + waitUntil: () => {}, + services: ingressServices, + }); + await state.subscribe(CONVERSATION_ID); + await handleSlackWebhook({ + request: slackWebhookRequest( + slackEnvelope({ + eventType: "message", + text: "follow-up without an explicit mention", + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + waitUntil: () => {}, + services: ingressServices, + }); + const workBeforeProcessing = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect( + workBeforeProcessing?.messages.map((record) => record.input.metadata), + ).toEqual([ + expect.objectContaining({ route: "mention" }), + expect.objectContaining({ route: "subscribed" }), + ]); + await state.unsubscribe(CONVERSATION_ID); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (thread, message, hooks) => { + subscribedValues.push(await thread.isSubscribed()); + calls.push({ + thread, + message, + skipped: hooks?.messageContext?.skipped ?? [], + }); + }, + handleSubscribedMessage: async () => { + throw new Error( + "mixed mention batches should promote to mention", + ); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.message.id).toBe("1712345.0002"); + expect(calls[0]?.skipped.map((message) => message.id)).toEqual([ + "1712345.0001", + ]); + expect(subscribedValues).toEqual([false]); + }); + it("processes pending Slack follow-ups before timeout continuation", async () => { const queue = new FakeQueue(); const state = getStateAdapter(); From 7254b2923d3623f58fe701eaf24ecb32be5b4f45 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 16:24:45 +0200 Subject: [PATCH 07/45] build: Repair rebased lockfile Update the rebased lockfile so Vercel Queue peer resolution covers the dashboard and example workspace entries now present on main. Refs GH-470 Co-Authored-By: Codex GPT-5 --- pnpm-lock.yaml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b1e966471..c0df2fa76 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -52,7 +52,7 @@ importers: version: link:../../packages/junior-agent-browser "@sentry/junior-dashboard": specifier: workspace:* - version: file:packages/junior-dashboard(@aws-sdk/credential-provider-web-identity@3.972.43)(@opentelemetry/api@1.9.1)(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(react-is@19.2.6)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0))(vitest@4.1.7(@edge-runtime/vm@5.0.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(msw@2.14.6(@types/node@25.9.1)(typescript@6.0.3))(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0))) + version: file:packages/junior-dashboard(@aws-sdk/credential-provider-web-identity@3.972.43)(@opentelemetry/api@1.9.1)(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(@vercel/queue@0.2.0)(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(react-is@19.2.6)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0))(vitest@4.1.7(@edge-runtime/vm@5.0.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(msw@2.14.6(@types/node@25.9.1)(typescript@6.0.3))(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0))) "@sentry/junior-datadog": specifier: workspace:* version: link:../../packages/junior-datadog @@ -86,7 +86,7 @@ importers: version: 2.7.0 nitro: specifier: 3.0.260522-beta - version: 3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)) + version: 3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(@vercel/queue@0.2.0)(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)) typescript: specifier: ^6.0.3 version: 6.0.3 @@ -8279,6 +8279,13 @@ packages: } engines: { node: 18 || 20 || >=22 } + mixpart@0.0.6: + resolution: + { + integrity: sha512-CRdXtgfQH2jARmtNmPR0Q7jL20fiESbaYk1b0KvLD0jCdUuemepREtsbd8nbiY6BHV9OGGddAZITNXklupUPUQ==, + } + engines: { node: ">=20.0.0" } + minimatch@3.1.5: resolution: { @@ -13484,14 +13491,14 @@ snapshots: "@sentry/core@10.53.1": {} - "@sentry/junior-dashboard@file:packages/junior-dashboard(@aws-sdk/credential-provider-web-identity@3.972.43)(@opentelemetry/api@1.9.1)(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(react-is@19.2.6)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0))(vitest@4.1.7(@edge-runtime/vm@5.0.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(msw@2.14.6(@types/node@25.9.1)(typescript@6.0.3))(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)))": + "@sentry/junior-dashboard@file:packages/junior-dashboard(@aws-sdk/credential-provider-web-identity@3.972.43)(@opentelemetry/api@1.9.1)(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(@vercel/queue@0.2.0)(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(react-is@19.2.6)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0))(vitest@4.1.7(@edge-runtime/vm@5.0.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(msw@2.14.6(@types/node@25.9.1)(typescript@6.0.3))(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)))": dependencies: "@sentry/junior": file:packages/junior(@aws-sdk/credential-provider-web-identity@3.972.43)(@opentelemetry/api@1.9.1) "@tanstack/react-query": 5.100.14(react@19.2.6) better-auth: 1.6.11(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.7(@edge-runtime/vm@5.0.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(msw@2.14.6(@types/node@25.9.1)(typescript@6.0.3))(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0))) hono: 4.12.22 lucide-react: 1.17.0(react@19.2.6) - nitro: 3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)) + nitro: 3.0.260522-beta(@vercel/blob@2.4.0)(@vercel/functions@3.6.0(@aws-sdk/credential-provider-web-identity@3.972.43))(@vercel/queue@0.2.0)(chokidar@5.0.0)(jiti@2.7.0)(lru-cache@11.5.0)(rollup@4.60.4)(vite@8.0.14(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.3)(yaml@2.9.0)) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) react-router: 7.16.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6) @@ -14307,6 +14314,13 @@ snapshots: dependencies: "@vercel/python-analysis": 0.11.1 + "@vercel/queue@0.2.0": + dependencies: + "@vercel/oidc": 3.4.1 + minimatch: 10.2.5 + mixpart: 0.0.6 + picocolors: 1.1.1 + "@vercel/redwood@2.4.13(rollup@4.60.4)": dependencies: "@vercel/nft": 1.5.0(rollup@4.60.4) @@ -16890,6 +16904,8 @@ snapshots: dependencies: brace-expansion: 5.0.6 + mixpart@0.0.6: {} + minimatch@3.1.5: dependencies: brace-expansion: 1.1.14 From 405f5f1bd7e9930cc83ba5abb65e88692305e628 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 16:31:54 +0200 Subject: [PATCH 08/45] fix(slack): Ack events before durable handoff Move Slack event callback processing behind waitUntil so Slack receives a fast acknowledgement while durable mailbox work still runs through the existing handoff path. Give Vercel Queue visibility a buffer beyond the function timeout to avoid redelivery racing host teardown. Refs GH-470 Co-Authored-By: Codex GPT-5 --- .../junior/src/chat/ingress/slack-webhook.ts | 24 +++--- .../chat/task-execution/vercel-callback.ts | 15 +++- .../task-execution/conversation-work.test.ts | 54 ++++++++---- .../slack/app-home-webhook.test.ts | 83 ++++++++++++++++++- packages/junior/tests/unit/vercel.test.ts | 7 ++ specs/task-execution.md | 12 +-- 6 files changed, 159 insertions(+), 36 deletions(-) diff --git a/packages/junior/src/chat/ingress/slack-webhook.ts b/packages/junior/src/chat/ingress/slack-webhook.ts index 2eac85269..506df7d29 100644 --- a/packages/junior/src/chat/ingress/slack-webhook.ts +++ b/packages/junior/src/chat/ingress/slack-webhook.ts @@ -81,7 +81,10 @@ export interface SlackWebhookServices { state?: StateAdapter; } -function enqueue(waitUntil: WaitUntilFn, task: Promise): void { +function enqueue( + waitUntil: WaitUntilFn, + task: Promise | (() => Promise), +): void { waitUntil(task); } @@ -575,15 +578,16 @@ export async function handleSlackWebhook(args: { } if (parsed.type === "event_callback") { - try { - await handleSlackEvent({ - body: parsed, - services: args.services, - }); - } catch (error) { - logException(error, "slack_event_enqueue_failed"); - throw error; - } + enqueue(args.waitUntil, async () => { + try { + await handleSlackEvent({ + body: parsed, + services: args.services, + }); + } catch (error) { + logException(error, "slack_event_enqueue_failed"); + } + }); } return new Response("ok", { status: 200 }); diff --git a/packages/junior/src/chat/task-execution/vercel-callback.ts b/packages/junior/src/chat/task-execution/vercel-callback.ts index fd5dc746e..44d2c8dbe 100644 --- a/packages/junior/src/chat/task-execution/vercel-callback.ts +++ b/packages/junior/src/chat/task-execution/vercel-callback.ts @@ -1,5 +1,6 @@ import { handleCallback } from "@vercel/queue"; import type { StateAdapter } from "chat"; +import { getChatConfig } from "@/chat/config"; import { runWithTurnRequestDeadline } from "@/chat/runtime/request-deadline"; import type { ConversationQueueMessage, ConversationWorkQueue } from "./queue"; import { getVercelConversationWorkQueue } from "./vercel-queue"; @@ -10,7 +11,7 @@ import { type ConversationWorkerContext, } from "./worker"; -export const CONVERSATION_WORK_VISIBILITY_TIMEOUT_SECONDS = 300; +export const CONVERSATION_WORK_VISIBILITY_TIMEOUT_BUFFER_SECONDS = 30; export interface ProcessConversationQueueMessageOptions { checkInIntervalMs?: number; @@ -42,6 +43,16 @@ function parseConversationQueueMessage( }; } +/** Resolve queue visibility so redelivery waits past the host timeout boundary. */ +export function resolveConversationWorkVisibilityTimeoutSeconds( + functionMaxDurationSeconds = getChatConfig().functionMaxDurationSeconds, +): number { + return ( + functionMaxDurationSeconds + + CONVERSATION_WORK_VISIBILITY_TIMEOUT_BUFFER_SECONDS + ); +} + /** Process one Vercel Queue payload with the generic conversation worker. */ export async function processConversationQueueMessage( message: unknown, @@ -71,7 +82,7 @@ export function createVercelConversationWorkCallback( { visibilityTimeoutSeconds: options.visibilityTimeoutSeconds ?? - CONVERSATION_WORK_VISIBILITY_TIMEOUT_SECONDS, + resolveConversationWorkVisibilityTimeoutSeconds(), }, ); } diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index 1e8f4768c..cc7e5ff0a 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -29,6 +29,7 @@ import { createVercelConversationWorkQueue } from "@/chat/task-execution/vercel- import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import type { WaitUntilFn } from "@/handlers/types"; vi.hoisted(() => { process.env.JUNIOR_STATE_ADAPTER = "memory"; @@ -139,6 +140,32 @@ function deferred(): { return { promise, resolve, reject }; } +type WaitUntilTask = () => Promise; + +function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { + return (task) => { + tasks.push(typeof task === "function" ? task : () => task); + }; +} + +async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { + for (let index = 0; index < tasks.length; index += 1) { + await tasks[index]?.(); + } +} + +async function handleSlackWebhookAndFlush( + args: Omit[0], "waitUntil">, +): Promise { + const waitUntilTasks: WaitUntilTask[] = []; + const response = await handleSlackWebhook({ + ...args, + waitUntil: collectWaitUntil(waitUntilTasks), + }); + await flushWaitUntil(waitUntilTasks); + return response; +} + describe("conversation work execution", () => { beforeEach(async () => { await disconnectStateAdapter(); @@ -686,13 +713,12 @@ describe("conversation work execution", () => { signingSecret: SLACK_SIGNING_SECRET, }); - const response = await handleSlackWebhook({ + const response = await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> deploy status`, }), ), - waitUntil: () => {}, services: { getSlackAdapter: () => slackAdapter, queue, @@ -747,14 +773,13 @@ describe("conversation work execution", () => { thread: Thread; }> = []; - await handleSlackWebhook({ + await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> first`, ts: "1712345.0001", }), ), - waitUntil: () => {}, services: { getSlackAdapter: () => slackAdapter, queue, @@ -767,7 +792,7 @@ describe("conversation work execution", () => { state, }, }); - await handleSlackWebhook({ + await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> second`, @@ -775,7 +800,6 @@ describe("conversation work execution", () => { threadTs: "1712345.0001", }), ), - waitUntil: () => {}, services: { getSlackAdapter: () => slackAdapter, queue, @@ -853,18 +877,17 @@ describe("conversation work execution", () => { state, }; - await handleSlackWebhook({ + await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> first`, ts: "1712345.0001", }), ), - waitUntil: () => {}, services: ingressServices, }); await state.subscribe(CONVERSATION_ID); - await handleSlackWebhook({ + await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ eventType: "message", @@ -873,7 +896,6 @@ describe("conversation work execution", () => { threadTs: "1712345.0001", }), ), - waitUntil: () => {}, services: ingressServices, }); const workBeforeProcessing = await getConversationWorkState({ @@ -946,7 +968,7 @@ describe("conversation work execution", () => { ], }); - await handleSlackWebhook({ + await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> follow-up`, @@ -954,7 +976,6 @@ describe("conversation work execution", () => { threadTs: "1712345.0001", }), ), - waitUntil: () => {}, services: { getSlackAdapter: () => slackAdapter, queue, @@ -1006,13 +1027,12 @@ describe("conversation work execution", () => { signingSecret: SLACK_SIGNING_SECRET, }); - await handleSlackWebhook({ + await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> first`, }), ), - waitUntil: () => {}, services: { getSlackAdapter: () => slackAdapter, queue, @@ -1090,13 +1110,12 @@ describe("conversation work execution", () => { signingSecret: SLACK_SIGNING_SECRET, }); - await handleSlackWebhook({ + await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> first`, }), ), - waitUntil: () => {}, services: { getSlackAdapter: () => slackAdapter, queue, @@ -1149,13 +1168,12 @@ describe("conversation work execution", () => { }); let currentNowMs = 1_000; - await handleSlackWebhook({ + await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> first`, }), ), - waitUntil: () => {}, services: { getSlackAdapter: () => slackAdapter, queue, diff --git a/packages/junior/tests/integration/slack/app-home-webhook.test.ts b/packages/junior/tests/integration/slack/app-home-webhook.test.ts index 9543aecb8..da54a4abd 100644 --- a/packages/junior/tests/integration/slack/app-home-webhook.test.ts +++ b/packages/junior/tests/integration/slack/app-home-webhook.test.ts @@ -4,6 +4,7 @@ import { createMemoryState } from "@chat-adapter/state-memory"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; import { disconnectStateAdapter } from "@/chat/state/adapter"; +import type { WaitUntilFn } from "@/handlers/types"; import { getCapturedSlackApiCalls, queueSlackApiError, @@ -34,6 +35,20 @@ function slackWebhookRequest(body: unknown): Request { }); } +type WaitUntilTask = () => Promise; + +function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { + return (task) => { + tasks.push(typeof task === "function" ? task : () => task); + }; +} + +async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { + for (let index = 0; index < tasks.length; index += 1) { + await tasks[index]?.(); + } +} + describe("Slack webhook: App Home events", () => { beforeEach(() => { process.env = { @@ -56,6 +71,7 @@ describe("Slack webhook: App Home events", () => { }); const state = createMemoryState(); + const waitUntilTasks: WaitUntilTask[] = []; const slackAdapter = createJuniorSlackAdapter({ botToken: "xoxb-test-token", botUserId: BOT_USER_ID, @@ -72,7 +88,7 @@ describe("Slack webhook: App Home events", () => { event_ts: "1712345.0001", }, }), - waitUntil: () => {}, + waitUntil: collectWaitUntil(waitUntilTasks), services: { getSlackAdapter: () => slackAdapter, queue: { @@ -99,6 +115,71 @@ describe("Slack webhook: App Home events", () => { }); expect(response.status).toBe(200); + expect(waitUntilTasks).toHaveLength(1); + expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(0); + await flushWaitUntil(waitUntilTasks); expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(1); }); + + it("acknowledges message events before durable handoff work runs", async () => { + const state = createMemoryState(); + const waitUntilTasks: WaitUntilTask[] = []; + const queueMessages: Array<{ conversationId: string }> = []; + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test-token", + botUserId: BOT_USER_ID, + signingSecret: SIGNING_SECRET, + }); + + const response = await handleSlackWebhook({ + request: slackWebhookRequest({ + team_id: "T123", + type: "event_callback", + event: { + type: "app_mention", + user: "U123", + text: `<@${BOT_USER_ID}> hello`, + channel: "C123", + ts: "1712345.0001", + event_ts: "1712345.0001", + channel_type: "channel", + }, + }), + waitUntil: collectWaitUntil(waitUntilTasks), + services: { + getSlackAdapter: () => slackAdapter, + queue: { + send: async (message) => { + queueMessages.push(message); + return { messageId: "queue-1" }; + }, + }, + runtime: { + handleAssistantContextChanged: async () => { + throw new Error("unexpected assistant context callback"); + }, + handleAssistantThreadStarted: async () => { + throw new Error("unexpected assistant thread callback"); + }, + handleNewMention: async () => { + throw new Error("unexpected mention callback"); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed message callback"); + }, + }, + state, + }, + }); + + expect(response.status).toBe(200); + expect(waitUntilTasks).toHaveLength(1); + expect(queueMessages).toEqual([]); + + await flushWaitUntil(waitUntilTasks); + + expect(queueMessages).toEqual([ + { conversationId: "slack:C123:1712345.0001" }, + ]); + }); }); diff --git a/packages/junior/tests/unit/vercel.test.ts b/packages/junior/tests/unit/vercel.test.ts index 7f07d348f..f7906bfff 100644 --- a/packages/junior/tests/unit/vercel.test.ts +++ b/packages/junior/tests/unit/vercel.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { resolveConversationWorkVisibilityTimeoutSeconds } from "@/chat/task-execution/vercel-callback"; import { DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC } from "@/chat/task-execution/vercel-queue"; import { juniorVercelConfig } from "@/vercel"; @@ -33,3 +34,9 @@ describe("juniorVercelConfig", () => { expect(config.buildCommand).toBeUndefined(); }); }); + +describe("resolveConversationWorkVisibilityTimeoutSeconds", () => { + it("keeps queue redelivery past the function timeout boundary", () => { + expect(resolveConversationWorkVisibilityTimeoutSeconds(300)).toBe(330); + }); +}); diff --git a/specs/task-execution.md b/specs/task-execution.md index 1855fe02e..be363d071 100644 --- a/specs/task-execution.md +++ b/specs/task-execution.md @@ -197,11 +197,13 @@ provider's idempotency window. The Vercel push consumer boundary is a thin adapter around the generic worker: it validates the `{ conversationId }` payload, uses `handleCallback`, and keeps -the Vercel visibility timeout at 300 seconds. The internal push endpoint is -`/api/internal/agent/continue`, because each queue delivery asks Junior to -continue the latest durable agent state for that conversation. The app must wire -the concrete conversation runner before registering the queue trigger; otherwise -queue messages could be acknowledged without advancing agent state. +the Vercel visibility timeout slightly beyond the configured function timeout +so redelivery does not race host teardown at the exact timeout boundary. The +internal push endpoint is `/api/internal/agent/continue`, because each queue +delivery asks Junior to continue the latest durable agent state for that +conversation. The app must wire the concrete conversation runner before +registering the queue trigger; otherwise queue messages could be acknowledged +without advancing agent state. ### Lease And Check-In Contract From 24c093285cfd7a817ca109bedcfc14fa3d754ab3 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 16:36:39 +0200 Subject: [PATCH 09/45] fix(chat): Report lost Slack worker leases Return a lost-lease worker result when Slack mailbox injection marking no longer owns the lease, and check lease ownership before invoking Slack runtime side effects. Remove a redundant pending-record sort so the worker ordering contract stays clear. Refs GH-470 Co-Authored-By: Codex GPT-5 --- .../src/chat/task-execution/slack-work.ts | 12 +++- .../junior/src/chat/task-execution/worker.ts | 5 +- .../task-execution/conversation-work.test.ts | 61 +++++++++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index ddbbc52ac..9945dac7c 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -194,7 +194,7 @@ export function createSlackConversationWorker( const state = getConnectedState(options.state); await state.connect(); - let records = getPendingRecords( + const records = getPendingRecords( await getConversationWorkState({ conversationId: context.conversationId, state, @@ -207,7 +207,6 @@ export function createSlackConversationWorker( return { status: "completed" }; } - records = records.sort(compareInboundMessages); const latestRecord = records[records.length - 1]; if (!latestRecord) { return { status: "completed" }; @@ -220,6 +219,10 @@ export function createSlackConversationWorker( ); } + if (!(await context.checkIn())) { + return { status: "lost_lease" }; + } + await runWithSlackInstallation({ adapter, installation: getInstallation(records), @@ -259,12 +262,15 @@ export function createSlackConversationWorker( }, }); - await markConversationMessagesInjected({ + const messagesMarked = await markConversationMessagesInjected({ conversationId: context.conversationId, inboundMessageIds: records.map((record) => record.inboundMessageId), leaseToken: context.leaseToken, state, }); + if (!messagesMarked) { + return { status: "lost_lease" }; + } return { status: "completed" }; }; diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index 03e473b2e..d8746e7d9 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -29,7 +29,7 @@ export interface ConversationWorkerContext { } export interface ConversationWorkerResult { - status: "completed" | "yielded"; + status: "completed" | "lost_lease" | "yielded"; } export interface ConversationWorkProcessResult { @@ -210,6 +210,9 @@ export async function processConversationWork( try { const result = await options.run(workerContext); + if (result.status === "lost_lease") { + return { status: "lost_lease" }; + } if (result.status === "yielded") { const yieldNowMs = now(options); const continuationMarked = await requestConversationContinuation({ diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index cc7e5ff0a..df58a9da5 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -1157,6 +1157,67 @@ describe("conversation work execution", () => { expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); }); + it("reports lost lease when Slack injection marking loses ownership", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }, + }); + + let handled = 0; + const worker = createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + handled += 1; + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }); + + await expect( + worker({ + checkIn: async () => true, + conversationId: CONVERSATION_ID, + drainMailbox: async () => [], + leaseToken: "stale-lease", + shouldYield: () => false, + }), + ).resolves.toEqual({ status: "lost_lease" }); + + expect(handled).toBe(1); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + }); + it("completes Slack mailbox work when the handler finishes after the soft deadline", async () => { const queue = new FakeQueue(); const state = getStateAdapter(); From 37171a7fe57a617f9d76c14422d8087a4c57c8a1 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 16:41:15 +0200 Subject: [PATCH 10/45] fix(slack): Pass event waitUntil promises Pass an actual Promise to waitUntil for Slack event callbacks so Vercel extends the invocation while durable ingress work finishes. Keep the fast ACK behavior covered by blocking queue send in the integration test and asserting the response returns before that work completes. Refs GH-470 Co-Authored-By: Codex GPT-5 --- .../junior/src/chat/ingress/slack-webhook.ts | 22 ++++++------- .../slack/app-home-webhook.test.ts | 32 +++++++++++++------ 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/packages/junior/src/chat/ingress/slack-webhook.ts b/packages/junior/src/chat/ingress/slack-webhook.ts index 506df7d29..de25ea17c 100644 --- a/packages/junior/src/chat/ingress/slack-webhook.ts +++ b/packages/junior/src/chat/ingress/slack-webhook.ts @@ -81,10 +81,7 @@ export interface SlackWebhookServices { state?: StateAdapter; } -function enqueue( - waitUntil: WaitUntilFn, - task: Promise | (() => Promise), -): void { +function enqueue(waitUntil: WaitUntilFn, task: Promise): void { waitUntil(task); } @@ -578,16 +575,15 @@ export async function handleSlackWebhook(args: { } if (parsed.type === "event_callback") { - enqueue(args.waitUntil, async () => { - try { - await handleSlackEvent({ - body: parsed, - services: args.services, - }); - } catch (error) { + enqueue( + args.waitUntil, + handleSlackEvent({ + body: parsed, + services: args.services, + }).catch((error) => { logException(error, "slack_event_enqueue_failed"); - } - }); + }), + ); } return new Response("ok", { status: 200 }); diff --git a/packages/junior/tests/integration/slack/app-home-webhook.test.ts b/packages/junior/tests/integration/slack/app-home-webhook.test.ts index da54a4abd..41ec5e899 100644 --- a/packages/junior/tests/integration/slack/app-home-webhook.test.ts +++ b/packages/junior/tests/integration/slack/app-home-webhook.test.ts @@ -35,20 +35,31 @@ function slackWebhookRequest(body: unknown): Request { }); } -type WaitUntilTask = () => Promise; +type WaitUntilTask = Promise; function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { return (task) => { - tasks.push(typeof task === "function" ? task : () => task); + tasks.push(typeof task === "function" ? task() : task); }; } async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]?.(); + await tasks[index]; } } +function deferred(): { + promise: Promise; + resolve(value: T): void; +} { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + describe("Slack webhook: App Home events", () => { beforeEach(() => { process.env = { @@ -116,15 +127,16 @@ describe("Slack webhook: App Home events", () => { expect(response.status).toBe(200); expect(waitUntilTasks).toHaveLength(1); - expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(0); await flushWaitUntil(waitUntilTasks); expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(1); }); - it("acknowledges message events before durable handoff work runs", async () => { + it("acknowledges message events before durable handoff work finishes", async () => { const state = createMemoryState(); const waitUntilTasks: WaitUntilTask[] = []; const queueMessages: Array<{ conversationId: string }> = []; + const queueSendEntered = deferred(); + const finishQueueSend = deferred(); const slackAdapter = createJuniorSlackAdapter({ botToken: "xoxb-test-token", botUserId: BOT_USER_ID, @@ -151,6 +163,8 @@ describe("Slack webhook: App Home events", () => { queue: { send: async (message) => { queueMessages.push(message); + queueSendEntered.resolve(); + await finishQueueSend.promise; return { messageId: "queue-1" }; }, }, @@ -174,12 +188,12 @@ describe("Slack webhook: App Home events", () => { expect(response.status).toBe(200); expect(waitUntilTasks).toHaveLength(1); - expect(queueMessages).toEqual([]); - - await flushWaitUntil(waitUntilTasks); - + await queueSendEntered.promise; expect(queueMessages).toEqual([ { conversationId: "slack:C123:1712345.0001" }, ]); + + finishQueueSend.resolve(); + await flushWaitUntil(waitUntilTasks); }); }); From f8670691fae0b2ba3a43ba6d4992452f4f3a3d31 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 16:53:12 +0200 Subject: [PATCH 11/45] fix(slack): Preserve bot mentions and disconnect refreshes Allow Slack bot-authored messages that explicitly mention Junior to reach durable ingress while retaining the self-message and noisy subtype guards. Keep App Home disconnect unlink and refresh failures isolated so a failed unlink does not prevent the view refresh from being attempted. Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/app/production.ts | 2 + .../junior/src/chat/ingress/slack-webhook.ts | 74 ++++++-- .../slack/app-home-webhook.test.ts | 159 +++++++++++++++++- 3 files changed, 219 insertions(+), 16 deletions(-) diff --git a/packages/junior/src/chat/app/production.ts b/packages/junior/src/chat/app/production.ts index 4d5317541..008ca4442 100644 --- a/packages/junior/src/chat/app/production.ts +++ b/packages/junior/src/chat/app/production.ts @@ -1,5 +1,6 @@ import type { SlackAdapter } from "@chat-adapter/slack"; import { createSlackRuntime } from "@/chat/app/factory"; +import { createUserTokenStore } from "@/chat/capabilities/factory"; import { getSlackBotToken, getSlackClientId, @@ -57,6 +58,7 @@ export function getProductionSlackWebhookServices(): SlackWebhookServices { getSlackAdapter: getProductionSlackAdapter, queue: getVercelConversationWorkQueue(), runtime: getProductionSlackRuntime(), + userTokenStore: createUserTokenStore(), }; } diff --git a/packages/junior/src/chat/ingress/slack-webhook.ts b/packages/junior/src/chat/ingress/slack-webhook.ts index de25ea17c..22c8d2b81 100644 --- a/packages/junior/src/chat/ingress/slack-webhook.ts +++ b/packages/junior/src/chat/ingress/slack-webhook.ts @@ -31,6 +31,7 @@ import { getStateAdapter } from "@/chat/state/adapter"; import { handleSlashCommand } from "@/chat/ingress/slash-command"; import { createUserTokenStore } from "@/chat/capabilities/factory"; import { unlinkProvider } from "@/chat/credentials/unlink-provider"; +import type { UserTokenStore } from "@/chat/credentials/user-token-store"; import { publishAppHomeView } from "@/chat/slack/app-home"; import { getSlackClient } from "@/chat/slack/client"; import { logException, withSpan } from "@/chat/logging"; @@ -57,6 +58,28 @@ type SlackEventEnvelope = { type?: string; }; +const IGNORED_MESSAGE_SUBTYPES = new Set([ + "message_changed", + "message_deleted", + "message_replied", + "channel_join", + "channel_leave", + "channel_topic", + "channel_purpose", + "channel_name", + "channel_archive", + "channel_unarchive", + "group_join", + "group_leave", + "group_topic", + "group_purpose", + "group_name", + "group_archive", + "group_unarchive", + "ekm_access_denied", + "tombstone", +]); + interface SlackInteractivePayload { actions?: Array<{ action_id?: string; @@ -79,6 +102,7 @@ export interface SlackWebhookServices { | "handleSubscribedMessage" >; state?: StateAdapter; + userTokenStore?: UserTokenStore; } function enqueue(waitUntil: WaitUntilFn, task: Promise): void { @@ -114,6 +138,10 @@ function textMentionsBot( return Boolean(botUserId && event.text?.includes(`<@${botUserId}>`)); } +function shouldIgnoreMessageSubtype(event: SlackMessageEvent): boolean { + return Boolean(event.subtype && IGNORED_MESSAGE_SUBTYPES.has(event.subtype)); +} + function normalizeMessageThreadId(message: Message): string { const normalized = normalizeIncomingSlackThreadId(message.threadId, message); if (normalized !== message.threadId) { @@ -145,7 +173,6 @@ async function buildThread(args: { function shouldIgnoreMessage(message: Message): boolean { return ( message.author.isMe === true || - message.author.isBot === true || isExternalSlackUser(message.raw as Record | undefined) ); } @@ -265,17 +292,14 @@ async function handleSlackEvent(args: { const adapter = args.services.getSlackAdapter(); const state = args.services.state ?? getStateAdapter(); + const userTokenStore = args.services.userTokenStore ?? createUserTokenStore(); await state.connect(); const installation = installationFromEnvelope(args.body); const receivedAtMs = Date.now(); async function publishAppHomeViewBestEffort(userId: string): Promise { try { - await publishAppHomeView( - getSlackClient(), - userId, - createUserTokenStore(), - ); + await publishAppHomeView(getSlackClient(), userId, userTokenStore); } catch (error) { logException(error, "slack_app_home_publish_failed", { slackUserId: userId, @@ -359,7 +383,7 @@ async function handleSlackEvent(args: { if ( (event.type === "message" || event.type === "app_mention") && - !event.subtype && + !shouldIgnoreMessageSubtype(event) && event.channel && event.ts ) { @@ -431,8 +455,8 @@ async function handleSlashCommandForm(args: { } async function handleInteractivePayload(args: { - adapter: SlackAdapter; payload: SlackInteractivePayload; + userTokenStore: UserTokenStore; }): Promise { if (args.payload.type !== "block_actions") { return; @@ -451,12 +475,27 @@ async function handleInteractivePayload(args: { "chat.app_home_disconnect", { slackUserId: userId }, async () => { - await unlinkProvider(userId, provider, createUserTokenStore()); - await publishAppHomeView( - getSlackClient(), - userId, - createUserTokenStore(), - ); + try { + await unlinkProvider(userId, provider, args.userTokenStore); + } catch (error) { + logException( + error, + "app_home_disconnect_unlink_failed", + { slackUserId: userId }, + { "app.credential.provider": provider }, + ); + } + + try { + await publishAppHomeView(getSlackClient(), userId, args.userTokenStore); + } catch (error) { + logException( + error, + "app_home_disconnect_publish_failed", + { slackUserId: userId }, + { "app.credential.provider": provider }, + ); + } }, ); } @@ -528,7 +567,12 @@ async function handleSlackForm(args: { adapter, installation: installationFromInteractive(payload), state, - task: () => handleInteractivePayload({ adapter, payload }), + task: () => + handleInteractivePayload({ + payload, + userTokenStore: + args.services.userTokenStore ?? createUserTokenStore(), + }), }).catch((error) => { logException(error, "slack_interactive_payload_failed", { slackUserId: buildAuthorFromInteractive(payload.user).userId, diff --git a/packages/junior/tests/integration/slack/app-home-webhook.test.ts b/packages/junior/tests/integration/slack/app-home-webhook.test.ts index 41ec5e899..851b750eb 100644 --- a/packages/junior/tests/integration/slack/app-home-webhook.test.ts +++ b/packages/junior/tests/integration/slack/app-home-webhook.test.ts @@ -1,7 +1,8 @@ import { createHmac } from "node:crypto"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createMemoryState } from "@chat-adapter/state-memory"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; +import type { UserTokenStore } from "@/chat/credentials/user-token-store"; import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; import { disconnectStateAdapter } from "@/chat/state/adapter"; import type { WaitUntilFn } from "@/handlers/types"; @@ -35,6 +36,49 @@ function slackWebhookRequest(body: unknown): Request { }); } +function slackFormRequest(params: URLSearchParams): Request { + const serialized = params.toString(); + const timestamp = String(Math.floor(Date.now() / 1000)); + return new Request("https://example.test/api/webhooks/slack", { + method: "POST", + headers: { + "content-type": "application/x-www-form-urlencoded", + "x-slack-request-timestamp": timestamp, + "x-slack-signature": signSlackBody(serialized, timestamp), + }, + body: serialized, + }); +} + +function interactiveDisconnectPayload(): Record { + return { + type: "block_actions", + team: { id: "T123" }, + user: { + id: "U123", + team_id: "T123", + username: "alice", + }, + actions: [ + { + action_id: "app_home_disconnect", + value: "notion", + }, + ], + }; +} + +function createTokenStore( + overrides: Partial = {}, +): UserTokenStore { + return { + get: vi.fn(async () => undefined), + set: vi.fn(async () => undefined), + delete: vi.fn(async () => undefined), + ...overrides, + }; +} + type WaitUntilTask = Promise; function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { @@ -196,4 +240,117 @@ describe("Slack webhook: App Home events", () => { finishQueueSend.resolve(); await flushWaitUntil(waitUntilTasks); }); + + it("routes explicit mentions from other Slack bots", async () => { + const state = createMemoryState(); + const waitUntilTasks: WaitUntilTask[] = []; + const queueMessages: Array<{ conversationId: string }> = []; + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test-token", + botUserId: BOT_USER_ID, + signingSecret: SIGNING_SECRET, + }); + + const response = await handleSlackWebhook({ + request: slackWebhookRequest({ + team_id: "T123", + type: "event_callback", + event: { + type: "message", + subtype: "bot_message", + bot_id: "B_DEPLOY", + username: "Deploy Bot", + text: `<@${BOT_USER_ID}> production deploy failed`, + channel: "C123", + ts: "1712345.0002", + event_ts: "1712345.0002", + channel_type: "channel", + }, + }), + waitUntil: collectWaitUntil(waitUntilTasks), + services: { + getSlackAdapter: () => slackAdapter, + queue: { + send: async (message) => { + queueMessages.push(message); + return { messageId: "queue-1" }; + }, + }, + runtime: { + handleAssistantContextChanged: async () => { + throw new Error("unexpected assistant context callback"); + }, + handleAssistantThreadStarted: async () => { + throw new Error("unexpected assistant thread callback"); + }, + handleNewMention: async () => { + throw new Error("unexpected mention callback"); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed message callback"); + }, + }, + state, + }, + }); + + expect(response.status).toBe(200); + expect(waitUntilTasks).toHaveLength(1); + await flushWaitUntil(waitUntilTasks); + expect(queueMessages).toEqual([ + { conversationId: "slack:C123:1712345.0002" }, + ]); + }); + + it("refreshes App Home after disconnect unlink failure", async () => { + const state = createMemoryState(); + const waitUntilTasks: WaitUntilTask[] = []; + const deleteToken = vi.fn(async () => { + throw new Error("token store unavailable"); + }); + const userTokenStore = createTokenStore({ delete: deleteToken }); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test-token", + botUserId: BOT_USER_ID, + signingSecret: SIGNING_SECRET, + }); + const params = new URLSearchParams({ + payload: JSON.stringify(interactiveDisconnectPayload()), + }); + + const response = await handleSlackWebhook({ + request: slackFormRequest(params), + waitUntil: collectWaitUntil(waitUntilTasks), + services: { + getSlackAdapter: () => slackAdapter, + queue: { + send: async () => { + throw new Error("interactive disconnect should not enqueue work"); + }, + }, + runtime: { + handleAssistantContextChanged: async () => { + throw new Error("unexpected assistant context callback"); + }, + handleAssistantThreadStarted: async () => { + throw new Error("unexpected assistant thread callback"); + }, + handleNewMention: async () => { + throw new Error("unexpected mention callback"); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed message callback"); + }, + }, + state, + userTokenStore, + }, + }); + + expect(response.status).toBe(200); + expect(waitUntilTasks).toHaveLength(1); + await flushWaitUntil(waitUntilTasks); + expect(deleteToken).toHaveBeenCalledWith("U123", "notion"); + expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(1); + }); }); From 13fbdccf6bd293f2ce148ac96f87ecaa825f1ba0 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 18:28:14 +0200 Subject: [PATCH 12/45] fix(slack): Steer rapid queued messages during active turns Drain leased Slack mailbox messages into Pi steering at safe turn boundaries so rapid follow-ups influence the active agent run instead of waiting for a stale serial turn. Persist the steered user messages before marking them injected so recovery remains durable. Tighten burst reliability by retrying conversation work locks longer, validating Slack signatures before initializing stateful webhook services, and aligning the example app's Vercel Queue trigger with the real callback route. Co-Authored-By: GPT-5 Codex --- apps/example/README.md | 2 +- apps/example/server.ts | 2 + apps/example/vercel.json | 13 +- packages/junior/src/chat/app/production.ts | 2 +- .../junior/src/chat/ingress/slack-webhook.ts | 21 +-- packages/junior/src/chat/respond-helpers.ts | 11 +- packages/junior/src/chat/respond.ts | 107 +++++++++-- .../junior/src/chat/runtime/reply-executor.ts | 57 +++++- .../junior/src/chat/runtime/slack-runtime.ts | 56 ++++++ .../src/chat/task-execution/slack-work.ts | 44 +++++ .../junior/src/chat/task-execution/store.ts | 24 ++- packages/junior/src/vercel.ts | 2 +- .../task-execution/conversation-work.test.ts | 167 +++++++++++++++++- .../slack/app-home-webhook.test.ts | 2 +- .../slack/webhook-auth-boundary.test.ts | 57 ++++++ .../junior/tests/unit/cli/init-cli.test.ts | 2 +- .../runtime/respond-provider-retry.test.ts | 62 ++++++- .../junior/tests/unit/turn-result.test.ts | 37 ++++ packages/junior/tests/unit/vercel.test.ts | 19 +- 19 files changed, 640 insertions(+), 47 deletions(-) create mode 100644 packages/junior/tests/integration/slack/webhook-auth-boundary.test.ts diff --git a/apps/example/README.md b/apps/example/README.md index 78613be26..6fe613859 100644 --- a/apps/example/README.md +++ b/apps/example/README.md @@ -47,5 +47,5 @@ Copy `.env.example` and set: - `plugins.ts` is the single source of truth for installed plugin registrations and trusted runtime plugins in this app - `nitro.config.ts` points `juniorNitro()` at `./plugins` so plugin content is copied into the build output and exposed to runtime through the virtual config module -- `server.ts` calls `createApp()` without repeating the plugin list +- `server.ts` imports the same plugin set and passes it to `createApp({ plugins })` so local dev and built bundles load identical runtime plugins - root `pnpm dev` starts a local heartbeat loop that calls `/api/internal/heartbeat` every minute, matching the production cron pulse used for trusted plugin heartbeats and stale dispatch recovery; it also defaults `JUNIOR_BASE_URL` to the local server when unset so signed internal callbacks can recover dispatched runs diff --git a/apps/example/server.ts b/apps/example/server.ts index 68312fc90..37bef749f 100644 --- a/apps/example/server.ts +++ b/apps/example/server.ts @@ -1,9 +1,11 @@ import { createApp } from "@sentry/junior"; import { initSentry } from "@sentry/junior/instrumentation"; +import { plugins } from "./plugins.ts"; initSentry(); const app = await createApp({ + plugins, configDefaults: { "sentry.org": "sentry", }, diff --git a/apps/example/vercel.json b/apps/example/vercel.json index 41f6a580a..48fe1ab46 100644 --- a/apps/example/vercel.json +++ b/apps/example/vercel.json @@ -6,5 +6,16 @@ "path": "/api/internal/heartbeat", "schedule": "* * * * *" } - ] + ], + "functions": { + "api/internal/agent/continue.ts": { + "maxDuration": 300, + "experimentalTriggers": [ + { + "type": "queue/v2beta", + "topic": "junior_conversation_work" + } + ] + } + } } diff --git a/packages/junior/src/chat/app/production.ts b/packages/junior/src/chat/app/production.ts index 008ca4442..95a08085b 100644 --- a/packages/junior/src/chat/app/production.ts +++ b/packages/junior/src/chat/app/production.ts @@ -56,9 +56,9 @@ export function getProductionSlackRuntime(): ReturnType< export function getProductionSlackWebhookServices(): SlackWebhookServices { return { getSlackAdapter: getProductionSlackAdapter, + getUserTokenStore: createUserTokenStore, queue: getVercelConversationWorkQueue(), runtime: getProductionSlackRuntime(), - userTokenStore: createUserTokenStore(), }; } diff --git a/packages/junior/src/chat/ingress/slack-webhook.ts b/packages/junior/src/chat/ingress/slack-webhook.ts index 22c8d2b81..9b06880b6 100644 --- a/packages/junior/src/chat/ingress/slack-webhook.ts +++ b/packages/junior/src/chat/ingress/slack-webhook.ts @@ -15,7 +15,6 @@ import { type SlackConversationRoute, } from "@/chat/task-execution/slack-work"; import { - ensureSlackAdapterInitialized, runWithSlackInstallation, verifySlackSignature, type SlackInstallationContext, @@ -92,6 +91,7 @@ interface SlackInteractivePayload { } export interface SlackWebhookServices { + getUserTokenStore?: () => UserTokenStore; getSlackAdapter: () => SlackAdapter; queue: ConversationWorkQueue; runtime: Pick< @@ -102,13 +102,16 @@ export interface SlackWebhookServices { | "handleSubscribedMessage" >; state?: StateAdapter; - userTokenStore?: UserTokenStore; } function enqueue(waitUntil: WaitUntilFn, task: Promise): void { waitUntil(task); } +function getUserTokenStore(services: SlackWebhookServices): UserTokenStore { + return services.getUserTokenStore?.() ?? createUserTokenStore(); +} + function parseJson(body: string): unknown { try { return JSON.parse(body); @@ -292,14 +295,17 @@ async function handleSlackEvent(args: { const adapter = args.services.getSlackAdapter(); const state = args.services.state ?? getStateAdapter(); - const userTokenStore = args.services.userTokenStore ?? createUserTokenStore(); await state.connect(); const installation = installationFromEnvelope(args.body); const receivedAtMs = Date.now(); async function publishAppHomeViewBestEffort(userId: string): Promise { try { - await publishAppHomeView(getSlackClient(), userId, userTokenStore); + await publishAppHomeView( + getSlackClient(), + userId, + getUserTokenStore(args.services), + ); } catch (error) { logException(error, "slack_app_home_publish_failed", { slackUserId: userId, @@ -570,8 +576,7 @@ async function handleSlackForm(args: { task: () => handleInteractivePayload({ payload, - userTokenStore: - args.services.userTokenStore ?? createUserTokenStore(), + userTokenStore: getUserTokenStore(args.services), }), }).catch((error) => { logException(error, "slack_interactive_payload_failed", { @@ -590,10 +595,6 @@ export async function handleSlackWebhook(args: { }): Promise { const adapter = args.services.getSlackAdapter(); const body = await args.request.text(); - await ensureSlackAdapterInitialized({ - adapter, - state: args.services.state, - }); if (!verifySlackSignature({ adapter, body, request: args.request })) { return new Response("Invalid signature", { status: 401 }); diff --git a/packages/junior/src/chat/respond-helpers.ts b/packages/junior/src/chat/respond-helpers.ts index 246237ff3..a0b099917 100644 --- a/packages/junior/src/chat/respond-helpers.ts +++ b/packages/junior/src/chat/respond-helpers.ts @@ -379,15 +379,18 @@ export function extractAssistantText(message: AssistantMessage): string { export function getTerminalAssistantMessages( messages: readonly unknown[], ): AssistantMessage[] { - let lastToolResultIndex = -1; + let lastInputBoundaryIndex = -1; for (let index = messages.length - 1; index >= 0; index -= 1) { - if (isToolResultMessage(messages[index])) { - lastToolResultIndex = index; + if ( + isToolResultMessage(messages[index]) || + getPiMessageRole(messages[index]) === "user" + ) { + lastInputBoundaryIndex = index; break; } } - return messages.slice(lastToolResultIndex + 1).filter(isAssistantMessage); + return messages.slice(lastInputBoundaryIndex + 1).filter(isAssistantMessage); } /** Upsert a skill into the active skills list by name. */ diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index 6ae696856..59c284367 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -212,12 +212,7 @@ export interface ReplyRequestContext { /** Absolute wall-clock deadline for this host request, in milliseconds. */ turnDeadlineAtMs?: number; channelConfiguration?: ChannelConfigurationService; - userAttachments?: Array<{ - data?: Buffer; - mediaType: string; - filename?: string; - promptText?: string; - }>; + userAttachments?: ReplyRequestAttachment[]; inboundAttachmentCount?: number; omittedImageAttachmentCount?: number; sandbox?: { @@ -234,6 +229,9 @@ export interface ReplyRequestContext { webSearch?: WebSearchToolDeps; }; onStatus?: (status: AssistantStatusSpec) => void | Promise; + drainSteeringMessages?: ( + inject: (messages: ReplySteeringMessage[]) => Promise, + ) => Promise; onAuthPending?: ( pendingAuth: ConversationPendingAuthState, ) => void | Promise; @@ -245,6 +243,20 @@ export interface ReplyRequestContext { }) => void; } +export interface ReplyRequestAttachment { + data?: Buffer; + mediaType: string; + filename?: string; + promptText?: string; +} + +export interface ReplySteeringMessage { + omittedImageAttachmentCount?: number; + text: string; + timestampMs?: number; + userAttachments?: ReplyRequestAttachment[]; +} + let startupDiscoveryLogged = false; const MAX_ROUTER_ATTACHMENT_PREVIEW_CHARS = 2_000; @@ -405,6 +417,19 @@ function buildUserTurnInput(args: { return { routerBlocks, userContentParts }; } +function buildSteeringPiMessage(message: ReplySteeringMessage): PiMessage { + const { userContentParts } = buildUserTurnInput({ + userTurnText: message.text, + userAttachments: message.userAttachments, + omittedImageAttachmentCount: message.omittedImageAttachmentCount ?? 0, + }); + return { + role: "user", + content: userContentParts, + timestamp: message.timestampMs ?? Date.now(), + } as PiMessage; +} + /** Run a full agent turn: discover skills, execute tools, and return the assistant reply. */ export async function generateAssistantReply( messageText: string, @@ -1062,16 +1087,6 @@ export async function generateAssistantReply( // mid-run native tool-list mutation. // ── Agent execution ────────────────────────────────────────────── - agent = new Agent({ - getApiKey: () => getPiGatewayApiKeyOverride(), - streamFn: createTracedStreamFn({ conversationPrivacy }), - initialState: { - systemPrompt: baseInstructions, - model: resolveGatewayModel(botConfig.modelId), - thinkingLevel: toAgentThinkingLevel(thinkingSelection.thinkingLevel), - tools: agentTools, - }, - }); let hasEmittedText = false; let needsSeparator = false; const persistSafeBoundary = async ( @@ -1096,6 +1111,66 @@ export async function generateAssistantReply( requester, }); }; + const drainSteeringMessages = async (): Promise => { + if ( + !context.drainSteeringMessages || + !turnSessionState.canUseTurnSession || + !sessionConversationId || + !sessionId + ) { + return; + } + + try { + let piMessages: PiMessage[] = []; + await context.drainSteeringMessages(async (messages) => { + piMessages = messages.map(buildSteeringPiMessage); + if (piMessages.length === 0) { + return; + } + await persistSafeBoundary([...agent!.state.messages, ...piMessages]); + }); + for (const message of piMessages) { + agent!.steer(message); + } + if (piMessages.length > 0) { + logInfo( + "agent_turn_steering_messages_injected", + spanContext, + { + "app.ai.steering_message_count": piMessages.length, + }, + "Agent turn steering messages injected", + ); + } + } catch (error) { + logWarn( + "agent_turn_steering_messages_drain_failed", + spanContext, + { + "exception.message": + error instanceof Error ? error.message : String(error), + }, + "Agent turn steering message drain failed", + ); + } + }; + + agent = new Agent({ + getApiKey: () => getPiGatewayApiKeyOverride(), + streamFn: createTracedStreamFn({ conversationPrivacy }), + steeringMode: "all", + prepareNextTurn: async () => { + await drainSteeringMessages(); + return undefined; + }, + initialState: { + systemPrompt: baseInstructions, + model: resolveGatewayModel(botConfig.modelId), + thinkingLevel: toAgentThinkingLevel(thinkingSelection.thinkingLevel), + tools: agentTools, + }, + }); const unsubscribe = agent.subscribe((event) => { if (event.type === "turn_end" && event.toolResults.length > 0) { diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 3afcd49f7..7f016ec1c 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -27,7 +27,10 @@ import { } from "@/chat/slack/reply"; import { buildSlackOutputMessage } from "@/chat/slack/output"; import { getSlackErrorObservabilityAttributes } from "@/chat/slack/errors"; -import { generateAssistantReply as generateAssistantReplyImpl } from "@/chat/respond"; +import { + generateAssistantReply as generateAssistantReplyImpl, + type ReplySteeringMessage, +} from "@/chat/respond"; import { shouldEmitDevAgentTrace } from "@/chat/runtime/dev-agent-trace"; import { getAssistantThreadContext, @@ -265,8 +268,12 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { beforeFirstResponsePost?: () => Promise; explicitMention?: boolean; onToolInvocation?: (invocation: TurnToolInvocation) => void; + onTurnStatePersisted?: () => Promise; preparedState?: PreparedTurnState; queuedMessages?: QueuedTurnMessage[]; + drainSteeringMessages?: ( + inject: (messages: QueuedTurnMessage[]) => Promise, + ) => Promise; } = {}, ) { if (message.author.isMe) { @@ -511,6 +518,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { await persistThreadState(thread, { conversation: preparedState.conversation, }); + await options.onTurnStatePersisted?.(); const resolvedUserName = message.author.userName ?? fallbackIdentity?.userName; @@ -629,6 +637,52 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }); const toolChannelId = preparedState.artifacts.assistantContextChannelId ?? channelId; + const resolveSteeringMessages = async ( + queuedMessages: QueuedTurnMessage[], + ): Promise => { + return await Promise.all( + queuedMessages.map(async (queued) => { + const attachments = queued.message.attachments; + return { + text: queued.userText, + timestampMs: queued.message.metadata.dateSent.getTime(), + omittedImageAttachmentCount: + !isVisionEnabled() && + hasPotentialImageAttachment(attachments) + ? countPotentialImageAttachments(attachments) + : 0, + userAttachments: await deps.resolveUserAttachments( + attachments, + { + threadId, + requesterId: queued.message.author.userId, + channelId, + runId, + conversation: preparedState.conversation, + messageTs: getSlackMessageTs(queued.message), + }, + ), + }; + }), + ); + }; + const drainSteeringMessages = options.drainSteeringMessages + ? async ( + inject: (messages: ReplySteeringMessage[]) => Promise, + ): Promise => { + let injectedMessages: ReplySteeringMessage[] | undefined; + const drained = await options.drainSteeringMessages!( + async (queuedMessages) => { + injectedMessages = + await resolveSteeringMessages(queuedMessages); + await inject(injectedMessages); + }, + ); + return ( + injectedMessages ?? (await resolveSteeringMessages(drained)) + ); + } + : undefined; let reply = await deps.services.generateAssistantReply( effectiveUserText, { @@ -693,6 +747,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }, onStatus: (nextStatus) => status.update(nextStatus), onToolInvocation: options.onToolInvocation, + drainSteeringMessages, }, ); const diagnosticsContext = { diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index 5c77cdff9..99a44809e 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -45,8 +45,12 @@ export interface AssistantLifecycleEvent { export interface ReplyHooks { beforeFirstResponsePost?: () => Promise; + drainSteeringMessages?: ( + inject: (messages: Message[]) => Promise, + ) => Promise; messageContext?: MessageContext; onToolInvocation?: (invocation: TurnToolInvocation) => void; + onTurnStatePersisted?: () => Promise; } const THREAD_OPTOUT_ACK = @@ -139,8 +143,12 @@ export interface SlackTurnRuntimeDependencies { beforeFirstResponsePost?: () => Promise; explicitMention?: boolean; onToolInvocation?: (invocation: TurnToolInvocation) => void; + onTurnStatePersisted?: () => Promise; preparedState?: TPreparedState; queuedMessages?: QueuedTurnMessage[]; + drainSteeringMessages?: ( + inject: (messages: QueuedTurnMessage[]) => Promise, + ) => Promise; }, ) => Promise; decideSubscribedReply: SubscribedReplyPolicy; @@ -183,6 +191,42 @@ function getQueuedMessages( }); } +function getQueuedMessagesFromSlackMessages( + messages: Message[], + options: { + explicitMention: boolean; + stripLeadingBotMention: SlackTurnRuntimeDependencies["stripLeadingBotMention"]; + }, +): QueuedTurnMessage[] { + return getQueuedMessages( + { skipped: messages, totalSinceLastHandler: messages.length }, + options, + ); +} + +function createSteeringMessageDrain( + hooks: ReplyHooks | undefined, + options: { + explicitMention: boolean; + stripLeadingBotMention: SlackTurnRuntimeDependencies["stripLeadingBotMention"]; + }, +): + | (( + inject: (messages: QueuedTurnMessage[]) => Promise, + ) => Promise) + | undefined { + if (!hooks?.drainSteeringMessages) { + return undefined; + } + + return async (inject) => { + const drained = await hooks.drainSteeringMessages!(async (messages) => { + await inject(getQueuedMessagesFromSlackMessages(messages, options)); + }); + return getQueuedMessagesFromSlackMessages(drained, options); + }; +} + export interface SlackTurnRuntime< _TPreparedState, TAssistantEvent extends AssistantLifecycleEvent = AssistantLifecycleEvent, @@ -355,11 +399,17 @@ export function createSlackTurnRuntime< explicitMention: true, stripLeadingBotMention: deps.stripLeadingBotMention, }); + const drainSteeringMessages = createSteeringMessageDrain(hooks, { + explicitMention: true, + stripLeadingBotMention: deps.stripLeadingBotMention, + }); await deps.replyToThread(thread, message, { explicitMention: true, beforeFirstResponsePost: hooks?.beforeFirstResponsePost, queuedMessages, onToolInvocation: toolInvocationHook, + drainSteeringMessages, + onTurnStatePersisted: hooks?.onTurnStatePersisted, }); }); } catch (error) { @@ -452,6 +502,10 @@ export function createSlackTurnRuntime< explicitMention: Boolean(message.isMention), stripLeadingBotMention: deps.stripLeadingBotMention, }); + const drainSteeringMessages = createSteeringMessageDrain(hooks, { + explicitMention: Boolean(message.isMention), + stripLeadingBotMention: deps.stripLeadingBotMention, + }); const combinedText = combineTurnText(queuedMessages, currentText); const turnIsExplicitMention = Boolean(message.isMention) || @@ -554,6 +608,8 @@ export function createSlackTurnRuntime< beforeFirstResponsePost: hooks?.beforeFirstResponsePost, queuedMessages, onToolInvocation: toolInvocationHook, + drainSteeringMessages, + onTurnStatePersisted: hooks?.onTurnStatePersisted, }); }); } catch (error) { diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 9945dac7c..356d99a7e 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -248,16 +248,60 @@ export function createSlackConversationWorker( skipped, totalSinceLastHandler: messages.length, }; + const initialInboundMessageIds = records.map( + (record) => record.inboundMessageId, + ); + let initialMessagesPersisted = false; + const markInitialMessagesInjected = async (): Promise => { + if (initialMessagesPersisted) { + return true; + } + const marked = await markConversationMessagesInjected({ + conversationId: context.conversationId, + inboundMessageIds: initialInboundMessageIds, + leaseToken: context.leaseToken, + state, + }); + initialMessagesPersisted = marked; + return marked; + }; + const onTurnStatePersisted = async (): Promise => { + if (!(await markInitialMessagesInjected())) { + throw new Error( + `Conversation work lease lost before Slack turn handoff for ${context.conversationId}`, + ); + } + }; + const drainSteeringMessages = async ( + inject: (messages: Message[]) => Promise, + ): Promise => { + let restoredMessages: Message[] | undefined; + const drained = await context.drainMailbox(async (pendingRecords) => { + const messages = pendingRecords.map((record) => + restoreMessage({ adapter, record }), + ); + restoredMessages = messages; + await inject(messages); + }); + return ( + restoredMessages ?? + drained.map((record) => restoreMessage({ adapter, record })) + ); + }; if (route === "mention") { await options.runtime.handleNewMention(thread, latestMessage, { messageContext, + drainSteeringMessages, + onTurnStatePersisted, }); return; } await options.runtime.handleSubscribedMessage(thread, latestMessage, { messageContext, + drainSteeringMessages, + onTurnStatePersisted, }); }, }); diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index 692ae7741..6979c5bcf 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -9,8 +9,10 @@ const CONVERSATION_WORK_PREFIX = "junior:conversation-work"; const CONVERSATION_WORK_SCHEMA_VERSION = 1; const CONVERSATION_WORK_INDEX_MAX_LENGTH = 10_000; const CONVERSATION_WORK_INDEX_LOCK_TTL_MS = 10_000; +const CONVERSATION_WORK_INDEX_LOCK_WAIT_MS = 2_000; +const CONVERSATION_WORK_INDEX_LOCK_RETRY_MS = 25; const CONVERSATION_WORK_MUTATION_LOCK_TTL_MS = 10_000; -const CONVERSATION_WORK_MUTATION_WAIT_MS = 2_000; +const CONVERSATION_WORK_MUTATION_WAIT_MS = 10_000; const CONVERSATION_WORK_MUTATION_RETRY_MS = 25; export const CONVERSATION_WORK_LEASE_TTL_MS = 90_000; @@ -302,12 +304,20 @@ async function withIndexLock( state: StateAdapter, callback: () => Promise, ): Promise { - const lock = await state.acquireLock( - indexLockKey(), - CONVERSATION_WORK_INDEX_LOCK_TTL_MS, - ); - if (!lock) { - throw new Error("Could not acquire conversation work index lock"); + const startedAtMs = now(); + let lock: Lock | null; + while (true) { + lock = await state.acquireLock( + indexLockKey(), + CONVERSATION_WORK_INDEX_LOCK_TTL_MS, + ); + if (lock) { + break; + } + if (now() - startedAtMs >= CONVERSATION_WORK_INDEX_LOCK_WAIT_MS) { + throw new Error("Could not acquire conversation work index lock"); + } + await sleep(CONVERSATION_WORK_INDEX_LOCK_RETRY_MS); } try { return await callback(); diff --git a/packages/junior/src/vercel.ts b/packages/junior/src/vercel.ts index 1a2ebb9a0..adc51ed1d 100644 --- a/packages/junior/src/vercel.ts +++ b/packages/junior/src/vercel.ts @@ -18,7 +18,7 @@ export function juniorVercelConfig(options: JuniorVercelConfigOptions = {}) { }, ], functions: { - "server.ts": { + "api/internal/agent/continue.ts": { maxDuration: 300, experimentalTriggers: [ { diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index df58a9da5..cbc726f34 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -1,6 +1,6 @@ import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { Message, Thread } from "chat"; +import type { Message, StateAdapter, Thread } from "chat"; import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; import { runHeartbeat } from "@/chat/agent-dispatch/heartbeat"; @@ -81,6 +81,47 @@ function inboundMessage( }; } +function delayIndexLockOnce(state: StateAdapter): StateAdapter { + let blocked = false; + return new Proxy(state, { + get(target, prop, receiver) { + if (prop === "acquireLock") { + return async (key: string, ttlMs: number) => { + if (!blocked && key === "junior:conversation-work:index:lock") { + blocked = true; + return null; + } + return target.acquireLock(key, ttlMs); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as StateAdapter; +} + +function delayMutationLockUntil(args: { + conversationId: string; + readyAtMs: number; + state: StateAdapter; +}): StateAdapter { + const mutationLockKey = `junior:conversation-work:mutation:${args.conversationId}`; + return new Proxy(args.state, { + get(target, prop, receiver) { + if (prop === "acquireLock") { + return async (key: string, ttlMs: number) => { + if (key === mutationLockKey && Date.now() < args.readyAtMs) { + return null; + } + return target.acquireLock(key, ttlMs); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as StateAdapter; +} + function signSlackBody(body: string, timestamp: string): string { return `v0=${createHmac("sha256", SLACK_SIGNING_SECRET) .update(`v0:${timestamp}:${body}`) @@ -204,6 +245,51 @@ describe("conversation work execution", () => { expect(queue.sent).toHaveLength(2); }); + it("retries transient conversation work index lock contention", async () => { + const queue = new FakeQueue(); + const state = delayIndexLockOnce(getStateAdapter()); + + await expect( + appendAndEnqueueInboundMessage({ + message: inboundMessage("m1"), + nowMs: 2_000, + queue, + state, + }), + ).resolves.toMatchObject({ status: "appended", queueMessageId: "queue-1" }); + + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.messages).toHaveLength(1); + expect(queue.sent).toHaveLength(1); + }); + + it("waits through same-conversation mutation lock contention", async () => { + vi.useFakeTimers({ now: 1_000 }); + const queue = new FakeQueue(); + const state = delayMutationLockUntil({ + conversationId: CONVERSATION_ID, + readyAtMs: 3_500, + state: getStateAdapter(), + }); + + const append = appendAndEnqueueInboundMessage({ + message: inboundMessage("m1"), + nowMs: 2_000, + queue, + state, + }); + + await vi.advanceTimersByTimeAsync(2_500); + await expect(append).resolves.toMatchObject({ + status: "appended", + queueMessageId: "queue-1", + }); + expect(queue.sent).toHaveLength(1); + }); + it("repairs pending mailbox work when the initial queue send fails", async () => { const queue = new FakeQueue(); queue.fail = true; @@ -1017,6 +1103,85 @@ describe("conversation work execution", () => { expect(work ? countPendingConversationMessages(work) : 0).toBe(0); }); + it("drains Slack messages that arrive during an active turn into steering", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createJuniorSlackAdapter({ + botToken: "xoxb-test", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); + const ingressServices = { + getSlackAdapter: () => slackAdapter, + queue, + runtime: { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }, + state, + }; + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + ts: "1712345.0001", + }), + ), + services: ingressServices, + }); + + const injected: string[][] = []; + const drained: string[][] = []; + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (_thread, _message, hooks) => { + await hooks?.onTurnStatePersisted?.(); + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> steer this`, + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + services: ingressServices, + }); + const messages = + (await hooks?.drainSteeringMessages?.(async (steering) => { + injected.push(steering.map((message) => message.id)); + })) ?? []; + drained.push(messages.map((message) => message.id)); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(injected).toEqual([["1712345.0002"]]); + expect(drained).toEqual([["1712345.0002"]]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.messages.map((message) => message.injectedAtMs)).toEqual([ + expect.any(Number), + expect.any(Number), + ]); + expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + }); + it("does not replay injected Slack mailbox records after lease recovery", async () => { const queue = new FakeQueue(); const state = getStateAdapter(); diff --git a/packages/junior/tests/integration/slack/app-home-webhook.test.ts b/packages/junior/tests/integration/slack/app-home-webhook.test.ts index 851b750eb..67db34f84 100644 --- a/packages/junior/tests/integration/slack/app-home-webhook.test.ts +++ b/packages/junior/tests/integration/slack/app-home-webhook.test.ts @@ -343,7 +343,7 @@ describe("Slack webhook: App Home events", () => { }, }, state, - userTokenStore, + getUserTokenStore: () => userTokenStore, }, }); diff --git a/packages/junior/tests/integration/slack/webhook-auth-boundary.test.ts b/packages/junior/tests/integration/slack/webhook-auth-boundary.test.ts new file mode 100644 index 000000000..8e8d47ed6 --- /dev/null +++ b/packages/junior/tests/integration/slack/webhook-auth-boundary.test.ts @@ -0,0 +1,57 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const ORIGINAL_ENV = { ...process.env }; + +type WaitUntilTask = Promise; + +function invalidSlackRequest(): Request { + const body = JSON.stringify({ type: "event_callback" }); + return new Request("https://example.test/api/webhooks/slack", { + method: "POST", + headers: { + "content-type": "application/json", + "x-slack-request-timestamp": String(Math.floor(Date.now() / 1000)), + "x-slack-signature": "v0=invalid", + }, + body, + }); +} + +function collectWaitUntil(tasks: WaitUntilTask[]) { + return (task: WaitUntilTask | (() => WaitUntilTask)) => { + tasks.push(typeof task === "function" ? task() : task); + }; +} + +describe("Slack webhook auth boundary", () => { + beforeEach(() => { + vi.resetModules(); + process.env = { + ...ORIGINAL_ENV, + SLACK_BOT_TOKEN: "xoxb-test-token", + SLACK_SIGNING_SECRET: "test-signing-secret", + }; + delete process.env.JUNIOR_STATE_ADAPTER; + delete process.env.REDIS_URL; + }); + + afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + vi.resetModules(); + }); + + it("rejects invalid Slack signatures before durable state is required", async () => { + const { handlePlatformWebhook } = await import("@/handlers/webhooks"); + const waitUntilTasks: WaitUntilTask[] = []; + + const response = await handlePlatformWebhook( + invalidSlackRequest(), + "slack", + collectWaitUntil(waitUntilTasks), + ); + + expect(response.status).toBe(401); + expect(await response.text()).toBe("Invalid signature"); + expect(waitUntilTasks).toHaveLength(0); + }); +}); diff --git a/packages/junior/tests/unit/cli/init-cli.test.ts b/packages/junior/tests/unit/cli/init-cli.test.ts index 80a58a96a..f0e403fc4 100644 --- a/packages/junior/tests/unit/cli/init-cli.test.ts +++ b/packages/junior/tests/unit/cli/init-cli.test.ts @@ -58,7 +58,7 @@ describe("init cli", () => { }, ]); expect(vercelConfig.functions).toEqual({ - "server.ts": { + "api/internal/agent/continue.ts": { maxDuration: 300, experimentalTriggers: [ { diff --git a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts index 557d86062..7076569a0 100644 --- a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts +++ b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts @@ -1,7 +1,10 @@ import { Buffer } from "node:buffer"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const { counters } = vi.hoisted(() => ({ +const { agentMode, counters } = vi.hoisted(() => ({ + agentMode: { + value: "providerRetry" as "providerRetry" | "steering", + }, counters: { continueCalls: 0, promptCalls: 0, @@ -16,6 +19,8 @@ vi.mock("@earendil-works/pi-agent-core", () => { systemPrompt: string; tools: unknown[]; }; + private prepareNextTurn?: () => Promise | unknown; + private steeringMessages: unknown[] = []; constructor(input: { initialState: { @@ -23,6 +28,7 @@ vi.mock("@earendil-works/pi-agent-core", () => { systemPrompt: string; tools: unknown[]; }; + prepareNextTurn?: () => Promise | unknown; }) { this.state = { messages: [], @@ -30,12 +36,17 @@ vi.mock("@earendil-works/pi-agent-core", () => { systemPrompt: input.initialState.systemPrompt, tools: input.initialState.tools, }; + this.prepareNextTurn = input.prepareNextTurn; } subscribe() { return () => undefined; } + steer(message: unknown) { + this.steeringMessages.push(message); + } + abort() { return undefined; } @@ -43,6 +54,20 @@ vi.mock("@earendil-works/pi-agent-core", () => { async prompt(message: unknown) { counters.promptCalls += 1; this.state.messages.push(message); + if (agentMode.value === "steering") { + await this.prepareNextTurn?.(); + this.state.messages.push(...this.steeringMessages); + this.state.messages.push({ + role: "assistant", + content: [{ type: "text", text: "Steered." }], + stopReason: "stop", + usage: { + input: 2, + output: 2, + }, + }); + return {}; + } this.state.messages.push({ role: "toolResult", toolName: "bash", @@ -176,6 +201,7 @@ import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; describe("generateAssistantReply provider retry", () => { beforeEach(async () => { + agentMode.value = "providerRetry"; counters.continueCalls = 0; counters.promptCalls = 0; process.env.JUNIOR_STATE_ADAPTER = "memory"; @@ -224,4 +250,38 @@ describe("generateAssistantReply provider retry", () => { "assistant", ]); }); + + it("persists and queues steering messages at the next Pi boundary", async () => { + agentMode.value = "steering"; + const injectedTexts: string[] = []; + + const reply = await generateAssistantReply("help me", { + requester: { userId: "U123" }, + correlation: { + conversationId: "conversation-steering", + turnId: "turn-steering", + channelId: "C123", + threadTs: "1712345.0001", + }, + drainSteeringMessages: async (inject) => { + const messages = [ + { text: "actually do the other thing", timestampMs: 2_000 }, + ]; + await inject(messages); + injectedTexts.push(...messages.map((message) => message.text)); + return messages; + }, + }); + + expect(reply.text).toBe("Steered."); + expect(injectedTexts).toEqual(["actually do the other thing"]); + + const sessionRecord = await getAgentTurnSessionRecord( + "conversation-steering", + "turn-steering", + ); + const serializedMessages = JSON.stringify(sessionRecord?.piMessages); + expect(serializedMessages).toContain("help me"); + expect(serializedMessages).toContain("actually do the other thing"); + }); }); diff --git a/packages/junior/tests/unit/turn-result.test.ts b/packages/junior/tests/unit/turn-result.test.ts index 4e63e352c..038356126 100644 --- a/packages/junior/tests/unit/turn-result.test.ts +++ b/packages/junior/tests/unit/turn-result.test.ts @@ -110,6 +110,43 @@ describe("buildTurnResult", () => { expect(reply.diagnostics.usedPrimaryText).toBe(true); }); + it("uses only assistant text after the latest steered user message", () => { + const reply = buildTurnResult({ + newMessages: [ + { + role: "user", + content: [{ type: "text", text: "first request" }], + }, + { + role: "assistant", + content: [{ type: "text", text: "Stale answer." }], + stopReason: "stop", + }, + { + role: "user", + content: [{ type: "text", text: "actually do this instead" }], + }, + { + role: "assistant", + content: [{ type: "text", text: "Updated answer." }], + stopReason: "stop", + }, + ], + userInput: "first request", + replyFiles: [], + artifactStatePatch: {}, + toolCalls: [], + generatedFileCount: 0, + shouldTrace: false, + spanContext: {}, + thinkingSelection, + }); + + expect(reply.text).toBe("Updated answer."); + expect(reply.diagnostics.outcome).toBe("success"); + expect(reply.diagnostics.assistantMessageCount).toBe(2); + }); + it("removes leaked thinking blocks from terminal assistant text", () => { const reply = buildTurnResult({ newMessages: [ diff --git a/packages/junior/tests/unit/vercel.test.ts b/packages/junior/tests/unit/vercel.test.ts index f7906bfff..64e71325e 100644 --- a/packages/junior/tests/unit/vercel.test.ts +++ b/packages/junior/tests/unit/vercel.test.ts @@ -1,8 +1,14 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { resolveConversationWorkVisibilityTimeoutSeconds } from "@/chat/task-execution/vercel-callback"; import { DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC } from "@/chat/task-execution/vercel-queue"; import { juniorVercelConfig } from "@/vercel"; +const TEST_DIR = path.dirname(fileURLToPath(import.meta.url)); +const WORKSPACE_ROOT = path.resolve(TEST_DIR, "../../../.."); + describe("juniorVercelConfig", () => { it("returns config with default buildCommand", () => { const config = juniorVercelConfig(); @@ -16,7 +22,7 @@ describe("juniorVercelConfig", () => { }, ]); expect(config.functions).toEqual({ - "server.ts": { + "api/internal/agent/continue.ts": { maxDuration: 300, experimentalTriggers: [ { @@ -33,6 +39,17 @@ describe("juniorVercelConfig", () => { expect(config.buildCommand).toBeUndefined(); }); + + it("keeps the example app Vercel config aligned with queue triggers", () => { + const config = JSON.parse( + fs.readFileSync( + path.join(WORKSPACE_ROOT, "apps/example/vercel.json"), + "utf8", + ), + ); + + expect(config).toEqual(juniorVercelConfig()); + }); }); describe("resolveConversationWorkVisibilityTimeoutSeconds", () => { From 1d0ac175f9e2d8590ae9a80af17bec6c8c1ce016 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 18:40:41 +0200 Subject: [PATCH 13/45] fix(slack): Strip normalized bot mentions in worker turns Real Slack webhook parsing normalizes leading bot mentions to @U_BOT. Preserve the mention-free prompt contract by passing the Slack bot user ID through the runtime strip helper. Add a webhook-to-worker integration test that fakes only Slack requests and the agent reply boundary, proving rapid thread follow-ups steer into one active turn and do not create duplicate replies. Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/app/factory.ts | 6 +- .../junior/src/chat/runtime/reply-executor.ts | 1 + .../junior/src/chat/runtime/slack-runtime.ts | 1 + .../junior/src/chat/runtime/thread-context.ts | 18 +- ...onversation-turn-steering-behavior.test.ts | 306 ++++++++++++++++++ .../tests/unit/runtime/thread-context.test.ts | 21 ++ 6 files changed, 349 insertions(+), 4 deletions(-) create mode 100644 packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts diff --git a/packages/junior/src/chat/app/factory.ts b/packages/junior/src/chat/app/factory.ts index 7dfe2a0fe..edc3e618a 100644 --- a/packages/junior/src/chat/app/factory.ts +++ b/packages/junior/src/chat/app/factory.ts @@ -83,7 +83,11 @@ export function createSlackRuntime( getThreadId, getChannelId, getRunId, - stripLeadingBotMention, + stripLeadingBotMention: (text, stripOptions) => + stripLeadingBotMention(text, { + ...stripOptions, + botUserId: options.getSlackAdapter().botUserId, + }), withSpan, logWarn, logException, diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 7f016ec1c..f05899425 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -311,6 +311,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }, async () => { const strippedUserText = stripLeadingBotMention(message.text, { + botUserId: deps.getSlackAdapter().botUserId, stripLeadingSlackMentionToken: options.explicitMention || Boolean(message.isMention), }); diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index 99a44809e..1f1f6d414 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -155,6 +155,7 @@ export interface SlackTurnRuntimeDependencies { stripLeadingBotMention: ( text: string, options: { + botUserId?: string; stripLeadingSlackMentionToken?: boolean; }, ) => string; diff --git a/packages/junior/src/chat/runtime/thread-context.ts b/packages/junior/src/chat/runtime/thread-context.ts index 8bf2dc3b8..1c8e28324 100644 --- a/packages/junior/src/chat/runtime/thread-context.ts +++ b/packages/junior/src/chat/runtime/thread-context.ts @@ -19,15 +19,27 @@ function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } +interface StripLeadingBotMentionOptions { + botUserId?: string; + stripLeadingSlackMentionToken?: boolean; +} + export function stripLeadingBotMention( text: string, - options: { - stripLeadingSlackMentionToken?: boolean; - } = {}, + options: StripLeadingBotMentionOptions = {}, ): string { if (!text.trim()) return text; let next = text; + if (options.stripLeadingSlackMentionToken && options.botUserId) { + const botUserId = escapeRegExp(options.botUserId); + const mentionByBotUserIdRe = new RegExp( + `^\\s*(?:<@${botUserId}(?:\\|[^>]+)?>|@${botUserId})[\\s,:-]*`, + "i", + ); + next = next.replace(mentionByBotUserIdRe, "").trim(); + } + if (options.stripLeadingSlackMentionToken) { next = next.replace(/^\s*<@[^>]+>[\s,:-]*/, "").trim(); } diff --git a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts new file mode 100644 index 000000000..b8a2630f6 --- /dev/null +++ b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts @@ -0,0 +1,306 @@ +import { createHmac } from "node:crypto"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { StateAdapter } from "chat"; +import { slackEventsApiEnvelope } from "../../fixtures/slack/factories/events"; +import { + getCapturedSlackApiCalls, + resetSlackApiMockState, +} from "../../msw/handlers/slack-api"; +import { createSlackRuntime } from "@/chat/app/factory"; +import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; +import type { ReplyExecutorServices } from "@/chat/runtime/reply-executor"; +import type { ReplySteeringMessage } from "@/chat/respond"; +import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; +import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import type { + ConversationQueueMessage, + ConversationQueueSendOptions, + ConversationWorkQueue, +} from "@/chat/task-execution/queue"; +import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; +import { getConversationWorkState } from "@/chat/task-execution/store"; +import { processConversationWork } from "@/chat/task-execution/worker"; +import type { WaitUntilFn } from "@/handlers/types"; + +const BOT_USER_ID = "U_BOT"; +const CHANNEL_ID = "C_STEER"; +const SIGNING_SECRET = "slack-signature-fixture"; +const THREAD_TS = "1712345.000100"; + +class FakeQueue implements ConversationWorkQueue { + sent: Array<{ + conversationId: string; + delayMs?: number; + idempotencyKey?: string; + }> = []; + + async send( + message: ConversationQueueMessage, + options?: ConversationQueueSendOptions, + ): Promise<{ messageId: string }> { + this.sent.push({ + conversationId: message.conversationId, + delayMs: options?.delayMs, + idempotencyKey: options?.idempotencyKey, + }); + return { messageId: `fake-queue-${this.sent.length}` }; + } +} + +type WaitUntilTask = () => Promise; + +function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { + return (task) => { + tasks.push(typeof task === "function" ? task : () => task); + }; +} + +async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { + for (let index = 0; index < tasks.length; index += 1) { + await tasks[index]?.(); + } +} + +function signSlackBody(body: string, timestamp: string): string { + return `v0=${createHmac("sha256", SIGNING_SECRET) + .update(`v0:${timestamp}:${body}`) + .digest("hex")}`; +} + +function slackRequest(body: unknown): Request { + const serialized = JSON.stringify(body); + const timestamp = String(Math.floor(Date.now() / 1000)); + return new Request("https://example.test/api/webhooks/slack", { + method: "POST", + headers: { + "content-type": "application/json", + "x-slack-request-timestamp": timestamp, + "x-slack-signature": signSlackBody(serialized, timestamp), + }, + body: serialized, + }); +} + +async function handleSlackWebhookAndFlush(args: { + request: Request; + services: Parameters[0]["services"]; +}): Promise { + const waitUntilTasks: WaitUntilTask[] = []; + const response = await handleSlackWebhook({ + ...args, + waitUntil: collectWaitUntil(waitUntilTasks), + }); + await flushWaitUntil(waitUntilTasks); + return response; +} + +function makeMessageEvent(args: { + eventType: "app_mention" | "message"; + text: string; + ts: string; +}) { + return slackEventsApiEnvelope({ + channel: CHANNEL_ID, + eventType: args.eventType, + text: args.text, + threadTs: args.ts === THREAD_TS ? undefined : THREAD_TS, + ts: args.ts, + }); +} + +function makeDiagnostics() { + return { + assistantMessageCount: 1, + modelId: "fake-agent-model", + outcome: "success" as const, + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: true, + }; +} + +function deferred(): { + promise: Promise; + resolve(value: T): void; +} { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +function createTurnHarness(args: { + generateAssistantReply: ReplyExecutorServices["generateAssistantReply"]; + state: StateAdapter; +}) { + const queue = new FakeQueue(); + const adapter = createJuniorSlackAdapter({ + botToken: "slack-bot-fixture", + botUserId: BOT_USER_ID, + signingSecret: SIGNING_SECRET, + }); + const runtime = createSlackRuntime({ + getSlackAdapter: () => adapter, + services: { + replyExecutor: { + generateAssistantReply: args.generateAssistantReply, + }, + }, + }); + const services = { + getSlackAdapter: () => adapter, + queue, + runtime, + state: args.state, + }; + const conversationId = adapter.encodeThreadId({ + channel: CHANNEL_ID, + threadTs: THREAD_TS, + }); + const runWorker = () => + processConversationWork(conversationId, { + queue, + run: createSlackConversationWorker({ + getSlackAdapter: () => adapter, + runtime, + state: args.state, + }), + state: args.state, + }); + + return { + conversationId, + queue, + runWorker, + services, + }; +} + +describe("Slack behavior: durable turn steering", () => { + beforeEach(async () => { + resetSlackApiMockState(); + await disconnectStateAdapter(); + }); + + afterEach(async () => { + resetSlackApiMockState(); + await disconnectStateAdapter(); + }); + + it("steers rapid Slack webhook follow-ups into one active worker turn", async () => { + const agentEntered = deferred(); + const releaseAgent = deferred(); + const agentCalls: Array<{ + prompt: string; + steeringTexts: string[]; + }> = []; + const state = getStateAdapter(); + const generateAssistantReply: ReplyExecutorServices["generateAssistantReply"] = + async (prompt, context) => { + agentEntered.resolve(); + await releaseAgent.promise; + + const steeringMessages: ReplySteeringMessage[] = []; + const drained = await context?.drainSteeringMessages?.( + async (messages) => { + steeringMessages.push(...messages); + }, + ); + if (steeringMessages.length === 0 && drained) { + steeringMessages.push(...drained); + } + + const steeringTexts = steeringMessages.map((message) => message.text); + agentCalls.push({ prompt, steeringTexts }); + return { + text: [ + `Handled initial: ${prompt}`, + `Steered: ${steeringTexts.join(" | ")}`, + ].join("\n"), + diagnostics: makeDiagnostics(), + }; + }; + const { conversationId, queue, runWorker, services } = createTurnHarness({ + generateAssistantReply, + state, + }); + + const firstResponse = await handleSlackWebhookAndFlush({ + request: slackRequest( + makeMessageEvent({ + eventType: "app_mention", + text: `<@${BOT_USER_ID}> start the incident summary`, + ts: THREAD_TS, + }), + ), + services, + }); + expect(firstResponse.status).toBe(200); + expect(queue.sent).toHaveLength(1); + + const activeTurn = runWorker(); + await agentEntered.promise; + + for (const followUp of [ + { text: "add customer impact", ts: "1712345.000200" }, + { text: "include the rollback owner", ts: "1712345.000300" }, + { text: "finish with the next action", ts: "1712345.000400" }, + ]) { + const response = await handleSlackWebhookAndFlush({ + request: slackRequest( + makeMessageEvent({ + eventType: "message", + text: followUp.text, + ts: followUp.ts, + }), + ), + services, + }); + expect(response.status).toBe(200); + } + + releaseAgent.resolve(); + await expect(activeTurn).resolves.toEqual({ status: "completed" }); + expect(queue.sent).toHaveLength(4); + + expect(agentCalls).toEqual([ + { + prompt: "start the incident summary", + steeringTexts: [ + "add customer impact", + "include the rollback owner", + "finish with the next action", + ], + }, + ]); + + const postCalls = getCapturedSlackApiCalls("chat.postMessage"); + expect(postCalls).toHaveLength(1); + expect(postCalls[0]?.params).toEqual( + expect.objectContaining({ + channel: CHANNEL_ID, + thread_ts: THREAD_TS, + text: expect.stringContaining("Steered: add customer impact"), + }), + ); + + const queuedWakeups = queue.sent.length; + for (let index = 1; index < queuedWakeups; index += 1) { + await expect(runWorker()).resolves.toEqual({ status: "no_work" }); + } + + expect(agentCalls).toHaveLength(1); + expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(1); + const work = await getConversationWorkState({ + conversationId, + state, + }); + expect(work?.messages).toHaveLength(4); + expect( + work?.messages.every((message) => message.injectedAtMs !== undefined), + ).toBe(true); + expect(work?.needsRun).toBe(false); + }); +}); diff --git a/packages/junior/tests/unit/runtime/thread-context.test.ts b/packages/junior/tests/unit/runtime/thread-context.test.ts index 39af30fad..34ef107f6 100644 --- a/packages/junior/tests/unit/runtime/thread-context.test.ts +++ b/packages/junior/tests/unit/runtime/thread-context.test.ts @@ -2,9 +2,30 @@ import { describe, expect, it } from "vitest"; import { getAssistantThreadContext, getTeamId, + stripLeadingBotMention, } from "@/chat/runtime/thread-context"; import { runWithWorkspaceTeamId } from "@/chat/slack/workspace-context"; +describe("stripLeadingBotMention", () => { + it("strips the Slack adapter's normalized bot user id mention", () => { + expect( + stripLeadingBotMention("@U_BOT start the incident summary", { + botUserId: "U_BOT", + stripLeadingSlackMentionToken: true, + }), + ).toBe("start the incident summary"); + }); + + it("keeps non-bot normalized mentions intact", () => { + expect( + stripLeadingBotMention("@U_OTHER ask junior for help", { + botUserId: "U_BOT", + stripLeadingSlackMentionToken: true, + }), + ).toBe("@U_OTHER ask junior for help"); + }); +}); + describe("getAssistantThreadContext", () => { it("uses the current raw message ts for the first non-DM thread reply", () => { expect( From b2fc2ebd36a16fe1098fed687d49d16fbe004d86 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 19:12:46 +0200 Subject: [PATCH 14/45] test(slack): Split conversation work component coverage Move Slack-specific durable mailbox coverage out of the generic conversation work component test and share local fixtures across the split files. This keeps each component test file below the large-file threshold without changing the covered behavior. Co-Authored-By: GPT-5 Codex --- .../conversation-work-fixtures.ts | 194 ++++ .../task-execution/conversation-work.test.ts | 846 +----------------- .../slack-conversation-work.test.ts | 602 +++++++++++++ 3 files changed, 804 insertions(+), 838 deletions(-) create mode 100644 packages/junior/tests/component/task-execution/conversation-work-fixtures.ts create mode 100644 packages/junior/tests/component/task-execution/slack-conversation-work.test.ts diff --git a/packages/junior/tests/component/task-execution/conversation-work-fixtures.ts b/packages/junior/tests/component/task-execution/conversation-work-fixtures.ts new file mode 100644 index 000000000..ac7da5a0d --- /dev/null +++ b/packages/junior/tests/component/task-execution/conversation-work-fixtures.ts @@ -0,0 +1,194 @@ +import { createHmac } from "node:crypto"; +import type { StateAdapter } from "chat"; +import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; +import type { InboundMessageRecord } from "@/chat/task-execution/store"; +import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; +import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; +import type { WaitUntilFn } from "@/handlers/types"; + +export const CONVERSATION_ID = "slack:C123:1712345.0001"; +export const SLACK_BOT_USER_ID = "U_BOT"; + +const SLACK_SIGNING_SECRET = "slack-signature-fixture"; + +export class FakeQueue implements ConversationWorkQueue { + fail = false; + sent: Array<{ + conversationId: string; + delayMs?: number; + idempotencyKey?: string; + }> = []; + + async send( + message: { conversationId: string }, + options?: { delayMs?: number; idempotencyKey?: string }, + ): Promise<{ messageId: string }> { + if (this.fail) { + throw new Error("queue unavailable"); + } + this.sent.push({ + conversationId: message.conversationId, + delayMs: options?.delayMs, + idempotencyKey: options?.idempotencyKey, + }); + return { messageId: `queue-${this.sent.length}` }; + } +} + +export function inboundMessage( + inboundMessageId: string, + overrides: Partial = {}, +): InboundMessageRecord { + return { + conversationId: CONVERSATION_ID, + inboundMessageId, + source: "slack", + createdAtMs: 1_000, + receivedAtMs: 1_100, + input: { + text: `message ${inboundMessageId}`, + authorId: "U123", + }, + ...overrides, + }; +} + +export function delayIndexLockOnce(state: StateAdapter): StateAdapter { + let blocked = false; + return new Proxy(state, { + get(target, prop, receiver) { + if (prop === "acquireLock") { + return async (key: string, ttlMs: number) => { + if (!blocked && key === "junior:conversation-work:index:lock") { + blocked = true; + return null; + } + return target.acquireLock(key, ttlMs); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as StateAdapter; +} + +export function delayMutationLockUntil(args: { + conversationId: string; + readyAtMs: number; + state: StateAdapter; +}): StateAdapter { + const mutationLockKey = `junior:conversation-work:mutation:${args.conversationId}`; + return new Proxy(args.state, { + get(target, prop, receiver) { + if (prop === "acquireLock") { + return async (key: string, ttlMs: number) => { + if (key === mutationLockKey && Date.now() < args.readyAtMs) { + return null; + } + return target.acquireLock(key, ttlMs); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as StateAdapter; +} + +function signSlackBody(body: string, timestamp: string): string { + return `v0=${createHmac("sha256", SLACK_SIGNING_SECRET) + .update(`v0:${timestamp}:${body}`) + .digest("hex")}`; +} + +export function slackWebhookRequest(body: unknown): Request { + const serialized = JSON.stringify(body); + const timestamp = String(Math.floor(Date.now() / 1000)); + return new Request("https://example.test/api/webhooks/slack", { + method: "POST", + headers: { + "content-type": "application/json", + "x-slack-request-timestamp": timestamp, + "x-slack-signature": signSlackBody(serialized, timestamp), + }, + body: serialized, + }); +} + +export function slackEnvelope(input: { + channel?: string; + eventType?: "app_mention" | "message"; + text?: string; + threadTs?: string; + ts?: string; +}) { + const channel = input.channel ?? "C123"; + const ts = input.ts ?? "1712345.0001"; + return { + team_id: "T123", + type: "event_callback", + event: { + type: input.eventType ?? "app_mention", + user: "U123", + text: input.text ?? `<@${SLACK_BOT_USER_ID}> hello`, + channel, + ts, + event_ts: ts, + channel_type: channel.startsWith("D") ? "im" : "channel", + ...(input.threadTs ? { thread_ts: input.threadTs } : {}), + }, + }; +} + +export function deferred(): { + promise: Promise; + resolve(value: T): void; +} { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +type WaitUntilTask = () => Promise; + +function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { + return (task) => { + tasks.push(typeof task === "function" ? task : () => task); + }; +} + +async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { + for (let index = 0; index < tasks.length; index += 1) { + await tasks[index]?.(); + } +} + +export async function handleSlackWebhookAndFlush( + args: Omit[0], "waitUntil">, +): Promise { + const waitUntilTasks: WaitUntilTask[] = []; + const response = await handleSlackWebhook({ + ...args, + waitUntil: collectWaitUntil(waitUntilTasks), + }); + await flushWaitUntil(waitUntilTasks); + return response; +} + +export function createSlackAdapterFixture() { + return createJuniorSlackAdapter({ + botToken: "slack-bot-fixture", + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, + }); +} + +export function createNoopSlackWebhookRuntime() { + return { + handleAssistantContextChanged: async () => {}, + handleAssistantThreadStarted: async () => {}, + handleNewMention: async () => {}, + handleSubscribedMessage: async () => {}, + }; +} diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index cbc726f34..8d647ff8f 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -1,7 +1,4 @@ -import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { Message, StateAdapter, Thread } from "chat"; -import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; import { runHeartbeat } from "@/chat/agent-dispatch/heartbeat"; import { @@ -22,190 +19,17 @@ import { CONVERSATION_WORK_DEFER_DELAY_MS, processConversationWork, } from "@/chat/task-execution/worker"; -import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; -import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import { createVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; -import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; -import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; -import type { WaitUntilFn } from "@/handlers/types"; - -vi.hoisted(() => { - process.env.JUNIOR_STATE_ADAPTER = "memory"; -}); - -const CONVERSATION_ID = "slack:C123:1712345.0001"; -const SLACK_SIGNING_SECRET = "test-signing-secret"; -const SLACK_BOT_USER_ID = "U_BOT"; - -class FakeQueue implements ConversationWorkQueue { - fail = false; - sent: Array<{ - conversationId: string; - delayMs?: number; - idempotencyKey?: string; - }> = []; - - async send( - message: { conversationId: string }, - options?: { delayMs?: number; idempotencyKey?: string }, - ): Promise<{ messageId: string }> { - if (this.fail) { - throw new Error("queue unavailable"); - } - this.sent.push({ - conversationId: message.conversationId, - delayMs: options?.delayMs, - idempotencyKey: options?.idempotencyKey, - }); - return { messageId: `queue-${this.sent.length}` }; - } -} - -function inboundMessage( - inboundMessageId: string, - overrides: Partial = {}, -): InboundMessageRecord { - return { - conversationId: CONVERSATION_ID, - inboundMessageId, - source: "slack", - createdAtMs: 1_000, - receivedAtMs: 1_100, - input: { - text: `message ${inboundMessageId}`, - authorId: "U123", - }, - ...overrides, - }; -} - -function delayIndexLockOnce(state: StateAdapter): StateAdapter { - let blocked = false; - return new Proxy(state, { - get(target, prop, receiver) { - if (prop === "acquireLock") { - return async (key: string, ttlMs: number) => { - if (!blocked && key === "junior:conversation-work:index:lock") { - blocked = true; - return null; - } - return target.acquireLock(key, ttlMs); - }; - } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as StateAdapter; -} - -function delayMutationLockUntil(args: { - conversationId: string; - readyAtMs: number; - state: StateAdapter; -}): StateAdapter { - const mutationLockKey = `junior:conversation-work:mutation:${args.conversationId}`; - return new Proxy(args.state, { - get(target, prop, receiver) { - if (prop === "acquireLock") { - return async (key: string, ttlMs: number) => { - if (key === mutationLockKey && Date.now() < args.readyAtMs) { - return null; - } - return target.acquireLock(key, ttlMs); - }; - } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as StateAdapter; -} - -function signSlackBody(body: string, timestamp: string): string { - return `v0=${createHmac("sha256", SLACK_SIGNING_SECRET) - .update(`v0:${timestamp}:${body}`) - .digest("hex")}`; -} - -function slackWebhookRequest(body: unknown): Request { - const serialized = JSON.stringify(body); - const timestamp = String(Math.floor(Date.now() / 1000)); - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(serialized, timestamp), - }, - body: serialized, - }); -} - -function slackEnvelope(input: { - channel?: string; - eventType?: "app_mention" | "message"; - text?: string; - threadTs?: string; - ts?: string; -}) { - const channel = input.channel ?? "C123"; - const ts = input.ts ?? "1712345.0001"; - return { - team_id: "T123", - type: "event_callback", - event: { - type: input.eventType ?? "app_mention", - user: "U123", - text: input.text ?? `<@${SLACK_BOT_USER_ID}> hello`, - channel, - ts, - event_ts: ts, - channel_type: channel.startsWith("D") ? "im" : "channel", - ...(input.threadTs ? { thread_ts: input.threadTs } : {}), - }, - }; -} - -function deferred(): { - promise: Promise; - reject(error: unknown): void; - resolve(value: T): void; -} { - let resolve!: (value: T) => void; - let reject!: (error: unknown) => void; - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise; - reject = rejectPromise; - }); - return { promise, resolve, reject }; -} - -type WaitUntilTask = () => Promise; - -function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task : () => task); - }; -} - -async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { - for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]?.(); - } -} - -async function handleSlackWebhookAndFlush( - args: Omit[0], "waitUntil">, -): Promise { - const waitUntilTasks: WaitUntilTask[] = []; - const response = await handleSlackWebhook({ - ...args, - waitUntil: collectWaitUntil(waitUntilTasks), - }); - await flushWaitUntil(waitUntilTasks); - return response; -} +import { + CONVERSATION_ID, + FakeQueue, + deferred, + delayIndexLockOnce, + delayMutationLockUntil, + inboundMessage, +} from "./conversation-work-fixtures"; describe("conversation work execution", () => { beforeEach(async () => { @@ -789,660 +613,6 @@ describe("conversation work execution", () => { expect(injected).toEqual(["m1"]); }); - it("persists Slack mentions into the durable mailbox and wakes the queue", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - - const response = await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> deploy status`, - }), - ), - services: { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }, - }); - - expect(response.status).toBe(200); - expect(queue.sent).toEqual([ - expect.objectContaining({ - conversationId: CONVERSATION_ID, - }), - ]); - const work = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect(work?.needsRun).toBe(true); - expect(work?.messages).toEqual([ - expect.objectContaining({ - conversationId: CONVERSATION_ID, - source: "slack", - input: expect.objectContaining({ - authorId: "U123", - metadata: expect.objectContaining({ - platform: "slack", - route: "mention", - }), - }), - }), - ]); - }); - - it("runs queued Slack mailbox work through the Slack runtime", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - const calls: Array<{ - message: Message; - skipped: Message[]; - thread: Thread; - }> = []; - - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> first`, - ts: "1712345.0001", - }), - ), - services: { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }, - }); - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> second`, - ts: "1712345.0002", - threadTs: "1712345.0001", - }), - ), - services: { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }, - }); - - await expect( - processConversationWork(CONVERSATION_ID, { - queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (thread, message, hooks) => { - calls.push({ - thread, - message, - skipped: hooks?.messageContext?.skipped ?? [], - }); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }), - }), - ).resolves.toEqual({ status: "completed" }); - - expect(calls).toHaveLength(1); - expect(calls[0]?.thread.id).toBe(CONVERSATION_ID); - expect(calls[0]?.message.id).toBe("1712345.0002"); - expect(calls[0]?.message.text).toContain("second"); - expect(calls[0]?.skipped.map((message) => message.id)).toEqual([ - "1712345.0001", - ]); - const work = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect(work ? countPendingConversationMessages(work) : 0).toBe(0); - }); - - it("keeps restored thread context aligned with promoted mention routing", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - const calls: Array<{ - message: Message; - skipped: Message[]; - thread: Thread; - }> = []; - const subscribedValues: boolean[] = []; - const ingressServices = { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }; - - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> first`, - ts: "1712345.0001", - }), - ), - services: ingressServices, - }); - await state.subscribe(CONVERSATION_ID); - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - eventType: "message", - text: "follow-up without an explicit mention", - ts: "1712345.0002", - threadTs: "1712345.0001", - }), - ), - services: ingressServices, - }); - const workBeforeProcessing = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect( - workBeforeProcessing?.messages.map((record) => record.input.metadata), - ).toEqual([ - expect.objectContaining({ route: "mention" }), - expect.objectContaining({ route: "subscribed" }), - ]); - await state.unsubscribe(CONVERSATION_ID); - - await expect( - processConversationWork(CONVERSATION_ID, { - queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (thread, message, hooks) => { - subscribedValues.push(await thread.isSubscribed()); - calls.push({ - thread, - message, - skipped: hooks?.messageContext?.skipped ?? [], - }); - }, - handleSubscribedMessage: async () => { - throw new Error( - "mixed mention batches should promote to mention", - ); - }, - }, - state, - }), - }), - ).resolves.toEqual({ status: "completed" }); - - expect(calls).toHaveLength(1); - expect(calls[0]?.message.id).toBe("1712345.0002"); - expect(calls[0]?.skipped.map((message) => message.id)).toEqual([ - "1712345.0001", - ]); - expect(subscribedValues).toEqual([false]); - }); - - it("processes pending Slack follow-ups before timeout continuation", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - await upsertAgentTurnSessionRecord({ - conversationId: CONVERSATION_ID, - sessionId: "turn-timeout", - sliceId: 2, - state: "awaiting_resume", - resumeReason: "timeout", - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "original request" }], - timestamp: 1_000, - }, - ], - }); - - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> follow-up`, - ts: "1712345.0002", - threadTs: "1712345.0001", - }), - ), - services: { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }, - }); - - const calls: string[] = []; - await expect( - processConversationWork(CONVERSATION_ID, { - queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (_thread, message) => { - calls.push(message.text); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }), - }), - ).resolves.toEqual({ status: "completed" }); - - expect(calls).toEqual([expect.stringContaining("follow-up")]); - const work = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect(work ? countPendingConversationMessages(work) : 0).toBe(0); - }); - - it("drains Slack messages that arrive during an active turn into steering", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - const ingressServices = { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }; - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> first`, - ts: "1712345.0001", - }), - ), - services: ingressServices, - }); - - const injected: string[][] = []; - const drained: string[][] = []; - await expect( - processConversationWork(CONVERSATION_ID, { - queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (_thread, _message, hooks) => { - await hooks?.onTurnStatePersisted?.(); - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> steer this`, - ts: "1712345.0002", - threadTs: "1712345.0001", - }), - ), - services: ingressServices, - }); - const messages = - (await hooks?.drainSteeringMessages?.(async (steering) => { - injected.push(steering.map((message) => message.id)); - })) ?? []; - drained.push(messages.map((message) => message.id)); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }), - }), - ).resolves.toEqual({ status: "completed" }); - - expect(injected).toEqual([["1712345.0002"]]); - expect(drained).toEqual([["1712345.0002"]]); - const work = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect(work?.messages.map((message) => message.injectedAtMs)).toEqual([ - expect.any(Number), - expect.any(Number), - ]); - expect(work ? countPendingConversationMessages(work) : 0).toBe(0); - }); - - it("does not replay injected Slack mailbox records after lease recovery", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> first`, - }), - ), - services: { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }, - }); - const lease = await startConversationWork({ - conversationId: CONVERSATION_ID, - nowMs: 2_000, - state, - }); - expect(lease.status).toBe("acquired"); - if (lease.status !== "acquired") { - return; - } - const work = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - const inboundMessageIds = - work?.messages.map((message) => message.inboundMessageId) ?? []; - await markConversationMessagesInjected({ - conversationId: CONVERSATION_ID, - inboundMessageIds, - leaseToken: lease.leaseToken, - nowMs: 3_000, - state, - }); - await recoverConversationWork({ - nowMs: 2_000 + CONVERSATION_WORK_LEASE_TTL_MS, - queue, - state, - }); - - await expect( - processConversationWork(CONVERSATION_ID, { - queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async () => { - throw new Error("injected messages should not replay"); - }, - handleSubscribedMessage: async () => { - throw new Error("injected messages should not replay"); - }, - }, - state, - }), - }), - ).resolves.toEqual({ status: "no_work" }); - - const recovered = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect(recovered?.needsRun).toBe(false); - expect(recovered ? countPendingConversationMessages(recovered) : 0).toBe(0); - }); - - it("keeps Slack mailbox records pending when the runtime handoff fails", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> first`, - }), - ), - services: { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }, - }); - - await expect( - processConversationWork(CONVERSATION_ID, { - queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async () => { - throw new Error("runtime failed before durable handoff"); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }), - }), - ).rejects.toThrow("runtime failed before durable handoff"); - - const work = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect(work?.lease).toBeUndefined(); - expect(work ? countPendingConversationMessages(work) : 0).toBe(1); - expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); - }); - - it("reports lost lease when Slack injection marking loses ownership", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> first`, - }), - ), - services: { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }, - }); - - let handled = 0; - const worker = createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async () => { - handled += 1; - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }); - - await expect( - worker({ - checkIn: async () => true, - conversationId: CONVERSATION_ID, - drainMailbox: async () => [], - leaseToken: "stale-lease", - shouldYield: () => false, - }), - ).resolves.toEqual({ status: "lost_lease" }); - - expect(handled).toBe(1); - const work = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect(work ? countPendingConversationMessages(work) : 0).toBe(1); - }); - - it("completes Slack mailbox work when the handler finishes after the soft deadline", async () => { - const queue = new FakeQueue(); - const state = getStateAdapter(); - await state.connect(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: SLACK_BOT_USER_ID, - signingSecret: SLACK_SIGNING_SECRET, - }); - let currentNowMs = 1_000; - - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> first`, - }), - ), - services: { - getSlackAdapter: () => slackAdapter, - queue, - runtime: { - handleAssistantContextChanged: async () => {}, - handleAssistantThreadStarted: async () => {}, - handleNewMention: async () => {}, - handleSubscribedMessage: async () => {}, - }, - state, - }, - }); - queue.sent = []; - - await expect( - processConversationWork(CONVERSATION_ID, { - nowMs: () => currentNowMs, - queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async () => { - currentNowMs = 242_000; - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }), - }), - ).resolves.toEqual({ status: "completed" }); - - expect(queue.sent).toEqual([]); - const work = await getConversationWorkState({ - conversationId: CONVERSATION_ID, - state, - }); - expect(work?.needsRun).toBe(false); - expect(work ? countPendingConversationMessages(work) : 0).toBe(0); - }); - it("rejects malformed Vercel Queue payloads", async () => { const queue = new FakeQueue(); diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts new file mode 100644 index 000000000..79a87eadf --- /dev/null +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -0,0 +1,602 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { Message, Thread } from "chat"; +import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; +import { + CONVERSATION_WORK_LEASE_TTL_MS, + countPendingConversationMessages, + getConversationWorkState, + markConversationMessagesInjected, + startConversationWork, +} from "@/chat/task-execution/store"; +import { processConversationWork } from "@/chat/task-execution/worker"; +import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; +import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { + CONVERSATION_ID, + FakeQueue, + SLACK_BOT_USER_ID, + createNoopSlackWebhookRuntime, + createSlackAdapterFixture, + handleSlackWebhookAndFlush, + slackEnvelope, + slackWebhookRequest, +} from "./conversation-work-fixtures"; + +describe("Slack conversation work execution", () => { + beforeEach(async () => { + await disconnectStateAdapter(); + }); + + afterEach(async () => { + await disconnectStateAdapter(); + }); + + it("persists Slack mentions into the durable mailbox and wakes the queue", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + + const response = await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> deploy status`, + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + + expect(response.status).toBe(200); + expect(queue.sent).toEqual([ + expect.objectContaining({ + conversationId: CONVERSATION_ID, + }), + ]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.needsRun).toBe(true); + expect(work?.messages).toEqual([ + expect.objectContaining({ + conversationId: CONVERSATION_ID, + source: "slack", + input: expect.objectContaining({ + authorId: "U123", + metadata: expect.objectContaining({ + platform: "slack", + route: "mention", + }), + }), + }), + ]); + }); + + it("runs queued Slack mailbox work through the Slack runtime", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + const calls: Array<{ + message: Message; + skipped: Message[]; + thread: Thread; + }> = []; + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + ts: "1712345.0001", + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> second`, + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (thread, message, hooks) => { + calls.push({ + thread, + message, + skipped: hooks?.messageContext?.skipped ?? [], + }); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.thread.id).toBe(CONVERSATION_ID); + expect(calls[0]?.message.id).toBe("1712345.0002"); + expect(calls[0]?.message.text).toContain("second"); + expect(calls[0]?.skipped.map((message) => message.id)).toEqual([ + "1712345.0001", + ]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + }); + + it("keeps restored thread context aligned with promoted mention routing", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + const calls: Array<{ + message: Message; + skipped: Message[]; + thread: Thread; + }> = []; + const subscribedValues: boolean[] = []; + const ingressServices = { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }; + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + ts: "1712345.0001", + }), + ), + services: ingressServices, + }); + await state.subscribe(CONVERSATION_ID); + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + eventType: "message", + text: "follow-up without an explicit mention", + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + services: ingressServices, + }); + const workBeforeProcessing = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect( + workBeforeProcessing?.messages.map((record) => record.input.metadata), + ).toEqual([ + expect.objectContaining({ route: "mention" }), + expect.objectContaining({ route: "subscribed" }), + ]); + await state.unsubscribe(CONVERSATION_ID); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (thread, message, hooks) => { + subscribedValues.push(await thread.isSubscribed()); + calls.push({ + thread, + message, + skipped: hooks?.messageContext?.skipped ?? [], + }); + }, + handleSubscribedMessage: async () => { + throw new Error( + "mixed mention batches should promote to mention", + ); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.message.id).toBe("1712345.0002"); + expect(calls[0]?.skipped.map((message) => message.id)).toEqual([ + "1712345.0001", + ]); + expect(subscribedValues).toEqual([false]); + }); + + it("processes pending Slack follow-ups before timeout continuation", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + await upsertAgentTurnSessionRecord({ + conversationId: CONVERSATION_ID, + sessionId: "turn-timeout", + sliceId: 2, + state: "awaiting_resume", + resumeReason: "timeout", + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "original request" }], + timestamp: 1_000, + }, + ], + }); + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> follow-up`, + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + + const calls: string[] = []; + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (_thread, message) => { + calls.push(message.text); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(calls).toEqual([expect.stringContaining("follow-up")]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + }); + + it("drains Slack messages that arrive during an active turn into steering", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + const ingressServices = { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }; + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + ts: "1712345.0001", + }), + ), + services: ingressServices, + }); + + const injected: string[][] = []; + const drained: string[][] = []; + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (_thread, _message, hooks) => { + await hooks?.onTurnStatePersisted?.(); + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> steer this`, + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + services: ingressServices, + }); + const messages = + (await hooks?.drainSteeringMessages?.(async (steering) => { + injected.push(steering.map((message) => message.id)); + })) ?? []; + drained.push(messages.map((message) => message.id)); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(injected).toEqual([["1712345.0002"]]); + expect(drained).toEqual([["1712345.0002"]]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.messages.map((message) => message.injectedAtMs)).toEqual([ + expect.any(Number), + expect.any(Number), + ]); + expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + }); + + it("does not replay injected Slack mailbox records after lease recovery", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + const lease = await startConversationWork({ + conversationId: CONVERSATION_ID, + nowMs: 2_000, + state, + }); + expect(lease.status).toBe("acquired"); + if (lease.status !== "acquired") { + return; + } + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + const inboundMessageIds = + work?.messages.map((message) => message.inboundMessageId) ?? []; + await markConversationMessagesInjected({ + conversationId: CONVERSATION_ID, + inboundMessageIds, + leaseToken: lease.leaseToken, + nowMs: 3_000, + state, + }); + await recoverConversationWork({ + nowMs: 2_000 + CONVERSATION_WORK_LEASE_TTL_MS, + queue, + state, + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + throw new Error("injected messages should not replay"); + }, + handleSubscribedMessage: async () => { + throw new Error("injected messages should not replay"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "no_work" }); + + const recovered = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(recovered?.needsRun).toBe(false); + expect(recovered ? countPendingConversationMessages(recovered) : 0).toBe(0); + }); + + it("keeps Slack mailbox records pending when the runtime handoff fails", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + throw new Error("runtime failed before durable handoff"); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).rejects.toThrow("runtime failed before durable handoff"); + + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.lease).toBeUndefined(); + expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); + }); + + it("reports lost lease when Slack injection marking loses ownership", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + + let handled = 0; + const worker = createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + handled += 1; + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }); + + await expect( + worker({ + checkIn: async () => true, + conversationId: CONVERSATION_ID, + drainMailbox: async () => [], + leaseToken: "stale-lease", + shouldYield: () => false, + }), + ).resolves.toEqual({ status: "lost_lease" }); + + expect(handled).toBe(1); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + }); + + it("completes Slack mailbox work when the handler finishes after the soft deadline", async () => { + const queue = new FakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + let currentNowMs = 1_000; + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + queue.sent = []; + + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + currentNowMs = 242_000; + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(queue.sent).toEqual([]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.needsRun).toBe(false); + expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + }); +}); From 502ebd1854da36cec57cad809e35bdb061d45bdc Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 20:10:39 +0200 Subject: [PATCH 15/45] test(slack): Share conversation work fixtures Move durable conversation work test helpers into the shared fixture tree and replace the test-only queue class with a small object factory. This keeps the steering integration test on the same fake request and queue seam as the component coverage while reducing duplicate Slack signing plumbing. Co-Authored-By: GPT-5 Codex --- .../task-execution/conversation-work.test.ts | 42 +++---- .../slack-conversation-work.test.ts | 22 ++-- .../conversation-work.ts} | 65 ++++++---- ...onversation-turn-steering-behavior.test.ts | 112 +++--------------- 4 files changed, 91 insertions(+), 150 deletions(-) rename packages/junior/tests/{component/task-execution/conversation-work-fixtures.ts => fixtures/conversation-work.ts} (73%) diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index 8d647ff8f..edba6f566 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -24,12 +24,12 @@ import { createVercelConversationWorkQueue } from "@/chat/task-execution/vercel- import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { CONVERSATION_ID, - FakeQueue, + createFakeQueue, deferred, delayIndexLockOnce, delayMutationLockUntil, inboundMessage, -} from "./conversation-work-fixtures"; +} from "../../fixtures/conversation-work"; describe("conversation work execution", () => { beforeEach(async () => { @@ -42,7 +42,7 @@ describe("conversation work execution", () => { }); it("stores inbound mailbox messages idempotently", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await expect( appendAndEnqueueInboundMessage({ message: inboundMessage("m1"), @@ -70,7 +70,7 @@ describe("conversation work execution", () => { }); it("retries transient conversation work index lock contention", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = delayIndexLockOnce(getStateAdapter()); await expect( @@ -92,7 +92,7 @@ describe("conversation work execution", () => { it("waits through same-conversation mutation lock contention", async () => { vi.useFakeTimers({ now: 1_000 }); - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = delayMutationLockUntil({ conversationId: CONVERSATION_ID, readyAtMs: 3_500, @@ -115,7 +115,7 @@ describe("conversation work execution", () => { }); it("repairs pending mailbox work when the initial queue send fails", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); queue.fail = true; await expect( appendAndEnqueueInboundMessage({ @@ -141,7 +141,7 @@ describe("conversation work execution", () => { }); it("defers duplicate queue nudges while a conversation lease is active", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const entered = deferred(); const finish = deferred(); @@ -181,7 +181,7 @@ describe("conversation work execution", () => { }); it("requeues work requested while a lease is running", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); let currentNowMs = 1_000; await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -216,7 +216,7 @@ describe("conversation work execution", () => { }); it("uses fresh queue idempotency keys for repeated worker requeues", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); let currentNowMs = 1_000; await requestConversationWork({ conversationId: CONVERSATION_ID, @@ -250,7 +250,7 @@ describe("conversation work execution", () => { }); it("drains pending messages and completes the leased conversation", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const injected: InboundMessageRecord[][] = []; @@ -277,7 +277,7 @@ describe("conversation work execution", () => { it("extends the lease with worker check-ins during long execution", async () => { vi.useFakeTimers({ now: 1_000 }); - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const entered = deferred(); const finish = deferred(); @@ -314,7 +314,7 @@ describe("conversation work execution", () => { }); it("requeues an expired conversation lease from heartbeat", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( startConversationWork({ conversationId: CONVERSATION_ID, nowMs: 2_000 }), @@ -339,7 +339,7 @@ describe("conversation work execution", () => { }); it("requeues pending mailbox work with no recent queue marker", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( @@ -352,7 +352,7 @@ describe("conversation work execution", () => { }); it("uses fresh queue idempotency keys for repeated heartbeat recovery", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( @@ -375,7 +375,7 @@ describe("conversation work execution", () => { }); it("runs conversation work recovery from the core heartbeat", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await runHeartbeat({ @@ -392,7 +392,7 @@ describe("conversation work execution", () => { }); it("injects messages that arrive during active execution at a safe boundary", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const injected: string[][] = []; @@ -420,7 +420,7 @@ describe("conversation work execution", () => { }); it("clears the run marker after draining messages that arrived during active execution", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( @@ -449,7 +449,7 @@ describe("conversation work execution", () => { }); it("requeues instead of completing when final mailbox work remains", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); let currentNowMs = 1_000; await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -480,7 +480,7 @@ describe("conversation work execution", () => { }); it("yields cooperatively and leaves the conversation resumable", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); let currentNowMs = 1_000; await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -590,7 +590,7 @@ describe("conversation work execution", () => { }); it("processes Vercel Queue payloads through the leased worker", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const injected: string[] = []; @@ -614,7 +614,7 @@ describe("conversation work execution", () => { }); it("rejects malformed Vercel Queue payloads", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); await expect( processConversationQueueMessage( diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index 79a87eadf..b67fc3ae1 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -14,14 +14,14 @@ import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; import { CONVERSATION_ID, - FakeQueue, + createFakeQueue, SLACK_BOT_USER_ID, createNoopSlackWebhookRuntime, createSlackAdapterFixture, handleSlackWebhookAndFlush, slackEnvelope, slackWebhookRequest, -} from "./conversation-work-fixtures"; +} from "../../fixtures/conversation-work"; describe("Slack conversation work execution", () => { beforeEach(async () => { @@ -33,7 +33,7 @@ describe("Slack conversation work execution", () => { }); it("persists Slack mentions into the durable mailbox and wakes the queue", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -79,7 +79,7 @@ describe("Slack conversation work execution", () => { }); it("runs queued Slack mailbox work through the Slack runtime", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -157,7 +157,7 @@ describe("Slack conversation work execution", () => { }); it("keeps restored thread context aligned with promoted mention routing", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -242,7 +242,7 @@ describe("Slack conversation work execution", () => { }); it("processes pending Slack follow-ups before timeout continuation", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -306,7 +306,7 @@ describe("Slack conversation work execution", () => { }); it("drains Slack messages that arrive during an active turn into steering", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -376,7 +376,7 @@ describe("Slack conversation work execution", () => { }); it("does not replay injected Slack mailbox records after lease recovery", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -450,7 +450,7 @@ describe("Slack conversation work execution", () => { }); it("keeps Slack mailbox records pending when the runtime handoff fails", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -498,7 +498,7 @@ describe("Slack conversation work execution", () => { }); it("reports lost lease when Slack injection marking loses ownership", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -550,7 +550,7 @@ describe("Slack conversation work execution", () => { }); it("completes Slack mailbox work when the handler finishes after the soft deadline", async () => { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); diff --git a/packages/junior/tests/component/task-execution/conversation-work-fixtures.ts b/packages/junior/tests/fixtures/conversation-work.ts similarity index 73% rename from packages/junior/tests/component/task-execution/conversation-work-fixtures.ts rename to packages/junior/tests/fixtures/conversation-work.ts index ac7da5a0d..62939bd95 100644 --- a/packages/junior/tests/component/task-execution/conversation-work-fixtures.ts +++ b/packages/junior/tests/fixtures/conversation-work.ts @@ -1,6 +1,10 @@ import { createHmac } from "node:crypto"; import type { StateAdapter } from "chat"; -import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; +import type { + ConversationQueueMessage, + ConversationQueueSendOptions, + ConversationWorkQueue, +} from "@/chat/task-execution/queue"; import type { InboundMessageRecord } from "@/chat/task-execution/store"; import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; @@ -8,33 +12,42 @@ import type { WaitUntilFn } from "@/handlers/types"; export const CONVERSATION_ID = "slack:C123:1712345.0001"; export const SLACK_BOT_USER_ID = "U_BOT"; +export const SLACK_SIGNING_SECRET = "slack-signature-fixture"; -const SLACK_SIGNING_SECRET = "slack-signature-fixture"; - -export class FakeQueue implements ConversationWorkQueue { - fail = false; +/** Queue fixture state exposed for recovery and idempotency assertions. */ +export interface FakeConversationWorkQueue extends ConversationWorkQueue { + fail: boolean; sent: Array<{ conversationId: string; delayMs?: number; idempotencyKey?: string; - }> = []; - - async send( - message: { conversationId: string }, - options?: { delayMs?: number; idempotencyKey?: string }, - ): Promise<{ messageId: string }> { - if (this.fail) { - throw new Error("queue unavailable"); - } - this.sent.push({ - conversationId: message.conversationId, - delayMs: options?.delayMs, - idempotencyKey: options?.idempotencyKey, - }); - return { messageId: `queue-${this.sent.length}` }; - } + }>; +} + +/** Create a queue port fixture that records durable wake-up sends. */ +export function createFakeQueue(): FakeConversationWorkQueue { + const queue: FakeConversationWorkQueue = { + fail: false, + sent: [], + async send( + message: ConversationQueueMessage, + options?: ConversationQueueSendOptions, + ): Promise<{ messageId: string }> { + if (queue.fail) { + throw new Error("queue unavailable"); + } + queue.sent.push({ + conversationId: message.conversationId, + delayMs: options?.delayMs, + idempotencyKey: options?.idempotencyKey, + }); + return { messageId: `queue-${queue.sent.length}` }; + }, + }; + return queue; } +/** Build a durable mailbox record for component-level conversation work tests. */ export function inboundMessage( inboundMessageId: string, overrides: Partial = {}, @@ -53,6 +66,7 @@ export function inboundMessage( }; } +/** Delay the global work index lock once so retry behavior is observable. */ export function delayIndexLockOnce(state: StateAdapter): StateAdapter { let blocked = false; return new Proxy(state, { @@ -72,6 +86,7 @@ export function delayIndexLockOnce(state: StateAdapter): StateAdapter { }) as StateAdapter; } +/** Delay one conversation's mutation lock until the fake clock reaches a point. */ export function delayMutationLockUntil(args: { conversationId: string; readyAtMs: number; @@ -100,6 +115,7 @@ function signSlackBody(body: string, timestamp: string): string { .digest("hex")}`; } +/** Build a signed Slack JSON webhook request for Slack ingress tests. */ export function slackWebhookRequest(body: unknown): Request { const serialized = JSON.stringify(body); const timestamp = String(Math.floor(Date.now() / 1000)); @@ -114,6 +130,7 @@ export function slackWebhookRequest(body: unknown): Request { }); } +/** Build the minimal Slack Events API envelope used by durable ingress tests. */ export function slackEnvelope(input: { channel?: string; eventType?: "app_mention" | "message"; @@ -139,7 +156,8 @@ export function slackEnvelope(input: { }; } -export function deferred(): { +/** Create a manually-resolved promise for coordinating async worker tests. */ +export function deferred(): { promise: Promise; resolve(value: T): void; } { @@ -164,6 +182,7 @@ async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { } } +/** Run Slack webhook ingress and flush every scheduled waitUntil task. */ export async function handleSlackWebhookAndFlush( args: Omit[0], "waitUntil">, ): Promise { @@ -176,6 +195,7 @@ export async function handleSlackWebhookAndFlush( return response; } +/** Create a Slack adapter that shares the signed-request fixture credentials. */ export function createSlackAdapterFixture() { return createJuniorSlackAdapter({ botToken: "slack-bot-fixture", @@ -184,6 +204,7 @@ export function createSlackAdapterFixture() { }); } +/** Provide no-op Slack runtime handlers when tests only care about ingress. */ export function createNoopSlackWebhookRuntime() { return { handleAssistantContextChanged: async () => {}, diff --git a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts index b8a2630f6..9e4b16ee3 100644 --- a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts +++ b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts @@ -1,105 +1,36 @@ -import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { StateAdapter } from "chat"; -import { slackEventsApiEnvelope } from "../../fixtures/slack/factories/events"; +import { + SLACK_BOT_USER_ID, + SLACK_SIGNING_SECRET, + createFakeQueue, + deferred, + handleSlackWebhookAndFlush, + slackEnvelope, + slackWebhookRequest, +} from "../../fixtures/conversation-work"; import { getCapturedSlackApiCalls, resetSlackApiMockState, } from "../../msw/handlers/slack-api"; import { createSlackRuntime } from "@/chat/app/factory"; -import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; import type { ReplyExecutorServices } from "@/chat/runtime/reply-executor"; import type { ReplySteeringMessage } from "@/chat/respond"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; -import type { - ConversationQueueMessage, - ConversationQueueSendOptions, - ConversationWorkQueue, -} from "@/chat/task-execution/queue"; import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; import { getConversationWorkState } from "@/chat/task-execution/store"; import { processConversationWork } from "@/chat/task-execution/worker"; -import type { WaitUntilFn } from "@/handlers/types"; -const BOT_USER_ID = "U_BOT"; const CHANNEL_ID = "C_STEER"; -const SIGNING_SECRET = "slack-signature-fixture"; const THREAD_TS = "1712345.000100"; -class FakeQueue implements ConversationWorkQueue { - sent: Array<{ - conversationId: string; - delayMs?: number; - idempotencyKey?: string; - }> = []; - - async send( - message: ConversationQueueMessage, - options?: ConversationQueueSendOptions, - ): Promise<{ messageId: string }> { - this.sent.push({ - conversationId: message.conversationId, - delayMs: options?.delayMs, - idempotencyKey: options?.idempotencyKey, - }); - return { messageId: `fake-queue-${this.sent.length}` }; - } -} - -type WaitUntilTask = () => Promise; - -function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task : () => task); - }; -} - -async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { - for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]?.(); - } -} - -function signSlackBody(body: string, timestamp: string): string { - return `v0=${createHmac("sha256", SIGNING_SECRET) - .update(`v0:${timestamp}:${body}`) - .digest("hex")}`; -} - -function slackRequest(body: unknown): Request { - const serialized = JSON.stringify(body); - const timestamp = String(Math.floor(Date.now() / 1000)); - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(serialized, timestamp), - }, - body: serialized, - }); -} - -async function handleSlackWebhookAndFlush(args: { - request: Request; - services: Parameters[0]["services"]; -}): Promise { - const waitUntilTasks: WaitUntilTask[] = []; - const response = await handleSlackWebhook({ - ...args, - waitUntil: collectWaitUntil(waitUntilTasks), - }); - await flushWaitUntil(waitUntilTasks); - return response; -} - function makeMessageEvent(args: { eventType: "app_mention" | "message"; text: string; ts: string; }) { - return slackEventsApiEnvelope({ + return slackEnvelope({ channel: CHANNEL_ID, eventType: args.eventType, text: args.text, @@ -120,26 +51,15 @@ function makeDiagnostics() { }; } -function deferred(): { - promise: Promise; - resolve(value: T): void; -} { - let resolve!: (value: T) => void; - const promise = new Promise((resolvePromise) => { - resolve = resolvePromise; - }); - return { promise, resolve }; -} - function createTurnHarness(args: { generateAssistantReply: ReplyExecutorServices["generateAssistantReply"]; state: StateAdapter; }) { - const queue = new FakeQueue(); + const queue = createFakeQueue(); const adapter = createJuniorSlackAdapter({ botToken: "slack-bot-fixture", - botUserId: BOT_USER_ID, - signingSecret: SIGNING_SECRET, + botUserId: SLACK_BOT_USER_ID, + signingSecret: SLACK_SIGNING_SECRET, }); const runtime = createSlackRuntime({ getSlackAdapter: () => adapter, @@ -228,10 +148,10 @@ describe("Slack behavior: durable turn steering", () => { }); const firstResponse = await handleSlackWebhookAndFlush({ - request: slackRequest( + request: slackWebhookRequest( makeMessageEvent({ eventType: "app_mention", - text: `<@${BOT_USER_ID}> start the incident summary`, + text: `<@${SLACK_BOT_USER_ID}> start the incident summary`, ts: THREAD_TS, }), ), @@ -249,7 +169,7 @@ describe("Slack behavior: durable turn steering", () => { { text: "finish with the next action", ts: "1712345.000400" }, ]) { const response = await handleSlackWebhookAndFlush({ - request: slackRequest( + request: slackWebhookRequest( makeMessageEvent({ eventType: "message", text: followUp.text, From e78f16ec5e68c96b7322d7af42c904b014244d66 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 20:19:02 +0200 Subject: [PATCH 16/45] fix(slack): Preserve user mentions after bot token When the Slack bot user id is known, only strip that specific leading mention. Keep subsequent user mention tokens in the prompt so referenced-user context still reaches the agent. Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/runtime/thread-context.ts | 20 +++++++++---------- .../tests/unit/runtime/thread-context.test.ts | 9 +++++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/junior/src/chat/runtime/thread-context.ts b/packages/junior/src/chat/runtime/thread-context.ts index 1c8e28324..a094a6323 100644 --- a/packages/junior/src/chat/runtime/thread-context.ts +++ b/packages/junior/src/chat/runtime/thread-context.ts @@ -31,17 +31,17 @@ export function stripLeadingBotMention( if (!text.trim()) return text; let next = text; - if (options.stripLeadingSlackMentionToken && options.botUserId) { - const botUserId = escapeRegExp(options.botUserId); - const mentionByBotUserIdRe = new RegExp( - `^\\s*(?:<@${botUserId}(?:\\|[^>]+)?>|@${botUserId})[\\s,:-]*`, - "i", - ); - next = next.replace(mentionByBotUserIdRe, "").trim(); - } - if (options.stripLeadingSlackMentionToken) { - next = next.replace(/^\s*<@[^>]+>[\s,:-]*/, "").trim(); + if (options.botUserId) { + const botUserId = escapeRegExp(options.botUserId); + const mentionByBotUserIdRe = new RegExp( + `^\\s*(?:<@${botUserId}(?:\\|[^>]+)?>|@${botUserId})[\\s,:-]*`, + "i", + ); + next = next.replace(mentionByBotUserIdRe, "").trim(); + } else { + next = next.replace(/^\s*<@[^>]+>[\s,:-]*/, "").trim(); + } } const mentionByNameRe = new RegExp( diff --git a/packages/junior/tests/unit/runtime/thread-context.test.ts b/packages/junior/tests/unit/runtime/thread-context.test.ts index 34ef107f6..f700cdf7d 100644 --- a/packages/junior/tests/unit/runtime/thread-context.test.ts +++ b/packages/junior/tests/unit/runtime/thread-context.test.ts @@ -24,6 +24,15 @@ describe("stripLeadingBotMention", () => { }), ).toBe("@U_OTHER ask junior for help"); }); + + it("preserves a referenced user after the leading bot mention", () => { + expect( + stripLeadingBotMention("<@U_BOT> <@U_OTHER> status?", { + botUserId: "U_BOT", + stripLeadingSlackMentionToken: true, + }), + ).toBe("<@U_OTHER> status?"); + }); }); describe("getAssistantThreadContext", () => { From 44ec1ba27b9fe70383e855ccd1a26c77f8e53b84 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 20:21:35 +0200 Subject: [PATCH 17/45] fix(slack): Preserve form workspace context Wrap Slack form handlers in the same workspace-team context used by event webhooks so slash commands and interactive payloads keep team-scoped lookup context. Also simplify timeout resume handling when queued Slack work has no pending mailbox entries. Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/ingress/slack-webhook.ts | 47 ++++++++++--------- .../src/chat/task-execution/slack-work.ts | 11 ++--- .../slack/app-home-webhook.test.ts | 4 ++ 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/packages/junior/src/chat/ingress/slack-webhook.ts b/packages/junior/src/chat/ingress/slack-webhook.ts index 9b06880b6..6aaf13e2f 100644 --- a/packages/junior/src/chat/ingress/slack-webhook.ts +++ b/packages/junior/src/chat/ingress/slack-webhook.ts @@ -539,17 +539,19 @@ async function handleSlackForm(args: { const installation = installationFromForm(params); enqueue( args.waitUntil, - runWithSlackInstallation({ - adapter, - installation, - state, - task: () => - handleSlashCommandForm({ - adapter, - params, - state, - }), - }).catch((error) => { + runWithWorkspaceTeamId(installation.teamId, () => + runWithSlackInstallation({ + adapter, + installation, + state, + task: () => + handleSlashCommandForm({ + adapter, + params, + state, + }), + }), + ).catch((error) => { logException(error, "slash_command_failed", { slackUserId: params.get("user_id") ?? undefined, }); @@ -566,19 +568,22 @@ async function handleSlackForm(args: { if (!payload) { return new Response("Invalid payload JSON", { status: 400 }); } + const installation = installationFromInteractive(payload); enqueue( args.waitUntil, - runWithSlackInstallation({ - adapter, - installation: installationFromInteractive(payload), - state, - task: () => - handleInteractivePayload({ - payload, - userTokenStore: getUserTokenStore(args.services), - }), - }).catch((error) => { + runWithWorkspaceTeamId(installation.teamId, () => + runWithSlackInstallation({ + adapter, + installation, + state, + task: () => + handleInteractivePayload({ + payload, + userTokenStore: getUserTokenStore(args.services), + }), + }), + ).catch((error) => { logException(error, "slack_interactive_payload_failed", { slackUserId: buildAuthorFromInteractive(payload.user).userId, }); diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 356d99a7e..ff5f1cdbb 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -142,12 +142,12 @@ function latestTimeoutResume( ); } -async function resumeAwaitingTimeout(conversationId: string): Promise { +async function resumeAwaitingTimeout(conversationId: string): Promise { const summary = latestTimeoutResume( await listAgentTurnSessionSummariesForConversation(conversationId), ); if (!summary) { - return false; + return; } const request = await getAwaitingTurnContinuationRequest({ @@ -155,11 +155,10 @@ async function resumeAwaitingTimeout(conversationId: string): Promise { sessionId: summary.sessionId, }); if (!request) { - return false; + return; } await resumeTimedOutTurnWithLockRetry(request); - return true; } function getInstallation( @@ -201,9 +200,7 @@ export function createSlackConversationWorker( }), ); if (records.length === 0) { - if (await resumeAwaitingTimeout(context.conversationId)) { - return { status: "completed" }; - } + await resumeAwaitingTimeout(context.conversationId); return { status: "completed" }; } diff --git a/packages/junior/tests/integration/slack/app-home-webhook.test.ts b/packages/junior/tests/integration/slack/app-home-webhook.test.ts index 67db34f84..07f6c125b 100644 --- a/packages/junior/tests/integration/slack/app-home-webhook.test.ts +++ b/packages/junior/tests/integration/slack/app-home-webhook.test.ts @@ -4,6 +4,7 @@ import { createMemoryState } from "@chat-adapter/state-memory"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import type { UserTokenStore } from "@/chat/credentials/user-token-store"; import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; +import { getWorkspaceTeamId } from "@/chat/slack/workspace-context"; import { disconnectStateAdapter } from "@/chat/state/adapter"; import type { WaitUntilFn } from "@/handlers/types"; import { @@ -305,7 +306,9 @@ describe("Slack webhook: App Home events", () => { it("refreshes App Home after disconnect unlink failure", async () => { const state = createMemoryState(); const waitUntilTasks: WaitUntilTask[] = []; + const workspaceTeamIds: Array = []; const deleteToken = vi.fn(async () => { + workspaceTeamIds.push(getWorkspaceTeamId()); throw new Error("token store unavailable"); }); const userTokenStore = createTokenStore({ delete: deleteToken }); @@ -351,6 +354,7 @@ describe("Slack webhook: App Home events", () => { expect(waitUntilTasks).toHaveLength(1); await flushWaitUntil(waitUntilTasks); expect(deleteToken).toHaveBeenCalledWith("U123", "notion"); + expect(workspaceTeamIds).toEqual(["T123"]); expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(1); }); }); From eb283e7766d292d11da7b93e54eb50d6a5f7217b Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 20:53:17 +0200 Subject: [PATCH 18/45] test(slack): Cover durable edited mentions Add a component regression that sends a Slack message_changed request through the durable webhook and queued worker path. This keeps edited mention coverage on the new mailbox architecture, not only the legacy Chat SDK webhook path. Refs GH-470 Co-Authored-By: GPT-5 Codex --- .../slack-conversation-work.test.ts | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index b67fc3ae1..379cf9406 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -78,6 +78,83 @@ describe("Slack conversation work execution", () => { ]); }); + it("routes edited Slack mentions through the durable mailbox", async () => { + const queue = createFakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + const editedTs = "1712345.0003"; + const editedText = `<@${SLACK_BOT_USER_ID}> edited ask`; + + const response = await handleSlackWebhookAndFlush({ + request: slackWebhookRequest({ + ...slackEnvelope({ + eventType: "message", + text: "edited ask", + ts: editedTs, + }), + event: { + type: "message", + subtype: "message_changed", + channel: "C123", + hidden: true, + message: { + type: "message", + user: "U123", + text: editedText, + ts: editedTs, + }, + previous_message: { + type: "message", + user: "U123", + text: "edited ask", + ts: editedTs, + }, + }, + }), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + + expect(response.status).toBe(200); + expect(queue.sent).toEqual([ + expect.objectContaining({ + conversationId: `slack:C123:${editedTs}`, + idempotencyKey: `slack:T123:slack:C123:${editedTs}:${editedTs}:message_changed_mention`, + }), + ]); + + const calls: Array<{ message: Message; thread: Thread }> = []; + await expect( + processConversationWork(`slack:C123:${editedTs}`, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (thread, message) => { + calls.push({ thread, message }); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.thread.id).toBe(`slack:C123:${editedTs}`); + expect(calls[0]?.message.id).toBe(`${editedTs}:message_changed_mention`); + expect(calls[0]?.message.text).toBe(editedText); + expect(calls[0]?.message.isMention).toBe(true); + }); + it("runs queued Slack mailbox work through the Slack runtime", async () => { const queue = createFakeQueue(); const state = getStateAdapter(); From d93ae980ad8333d82805dfa26b91d0879e73b2d5 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 20:56:42 +0200 Subject: [PATCH 19/45] fix(runtime): Report effective turn timeout Use the active turn deadline budget in timeout errors and timeout telemetry. This keeps resumed turns with shorter host request deadlines from reporting the configured maximum instead of the operative timeout. Refs GH-470 Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond.ts | 5 ++-- .../runtime/respond-timeout-resume.test.ts | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index 59c284367..1b80c7b8e 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -446,6 +446,7 @@ export async function generateAssistantReply( contextTurnDeadlineAtMs === undefined ? configuredTurnDeadlineAtMs : Math.min(configuredTurnDeadlineAtMs, contextTurnDeadlineAtMs); + const turnTimeoutBudgetMs = Math.max(0, turnDeadlineAtMs - replyStartedAtMs); let timeoutResumeConversationId: string | undefined; let timeoutResumeSessionId: string | undefined; let timeoutResumeSliceId = 1; @@ -1258,7 +1259,7 @@ export async function generateAssistantReply( agent.abort(); reject( new Error( - `Agent turn timed out after ${botConfig.turnTimeoutMs}ms`, + `Agent turn timed out after ${turnTimeoutBudgetMs}ms`, ), ); }; @@ -1287,7 +1288,7 @@ export async function generateAssistantReply( thinkingSelection.thinkingLevel, } : {}), - "app.ai.turn_timeout_ms": botConfig.turnTimeoutMs, + "app.ai.turn_timeout_ms": turnTimeoutBudgetMs, "app.ai.turn_deadline_remaining_ms": Math.max( 0, turnDeadlineAtMs - Date.now(), diff --git a/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts b/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts index fe2b442eb..c1f5cbef3 100644 --- a/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts +++ b/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts @@ -246,6 +246,33 @@ describe("generateAssistantReply timeout resume", () => { ]); }); + it("records the effective request deadline timeout budget", async () => { + const startedAtMs = Date.now(); + const replyPromise = generateAssistantReply("help me", { + requester: { userId: "U123" }, + turnDeadlineAtMs: startedAtMs + 2_500, + correlation: { + conversationId: "conversation-short-deadline", + turnId: "turn-short-deadline", + channelId: "C123", + threadTs: "1712345.0005", + }, + }).catch((caught) => caught); + + await vi.advanceTimersByTimeAsync(2_500); + const error = await replyPromise; + + expect(promptAborted.value).toBe(true); + expect(isRetryableTurnError(error, "turn_timeout_resume")).toBe(true); + const sessionRecord = await getAgentTurnSessionRecord( + "conversation-short-deadline", + "turn-short-deadline", + ); + expect(sessionRecord?.errorMessage).toBe( + "Agent turn timed out after 2500ms", + ); + }); + it("persists omitted-image context in the session-recorded Pi user message", async () => { const replyPromise = generateAssistantReply("what is in this image?", { requester: { userId: "U123" }, From ee692916a39394f9ffc723e5cfe7a12af0818cac Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 21:00:43 +0200 Subject: [PATCH 20/45] fix(runtime): Preserve steered assistant text Treat tool results, not steering user messages, as the terminal assistant output boundary. This prevents mid-turn steering from truncating assistant text that belongs to the same finalized reply. Refs GH-470 Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond-helpers.ts | 11 ++++------- packages/junior/tests/unit/turn-result.test.ts | 8 +++++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/junior/src/chat/respond-helpers.ts b/packages/junior/src/chat/respond-helpers.ts index a0b099917..246237ff3 100644 --- a/packages/junior/src/chat/respond-helpers.ts +++ b/packages/junior/src/chat/respond-helpers.ts @@ -379,18 +379,15 @@ export function extractAssistantText(message: AssistantMessage): string { export function getTerminalAssistantMessages( messages: readonly unknown[], ): AssistantMessage[] { - let lastInputBoundaryIndex = -1; + let lastToolResultIndex = -1; for (let index = messages.length - 1; index >= 0; index -= 1) { - if ( - isToolResultMessage(messages[index]) || - getPiMessageRole(messages[index]) === "user" - ) { - lastInputBoundaryIndex = index; + if (isToolResultMessage(messages[index])) { + lastToolResultIndex = index; break; } } - return messages.slice(lastInputBoundaryIndex + 1).filter(isAssistantMessage); + return messages.slice(lastToolResultIndex + 1).filter(isAssistantMessage); } /** Upsert a skill into the active skills list by name. */ diff --git a/packages/junior/tests/unit/turn-result.test.ts b/packages/junior/tests/unit/turn-result.test.ts index 038356126..7fdd41ac2 100644 --- a/packages/junior/tests/unit/turn-result.test.ts +++ b/packages/junior/tests/unit/turn-result.test.ts @@ -110,7 +110,7 @@ describe("buildTurnResult", () => { expect(reply.diagnostics.usedPrimaryText).toBe(true); }); - it("uses only assistant text after the latest steered user message", () => { + it("keeps assistant text across steered user messages", () => { const reply = buildTurnResult({ newMessages: [ { @@ -119,7 +119,7 @@ describe("buildTurnResult", () => { }, { role: "assistant", - content: [{ type: "text", text: "Stale answer." }], + content: [{ type: "text", text: "Initial answer." }], stopReason: "stop", }, { @@ -142,7 +142,9 @@ describe("buildTurnResult", () => { thinkingSelection, }); - expect(reply.text).toBe("Updated answer."); + expect(reply.text).toBe( + ["Initial answer.", "Updated answer."].join("\n\n"), + ); expect(reply.diagnostics.outcome).toBe("success"); expect(reply.diagnostics.assistantMessageCount).toBe(2); }); From dafb26c61c6df356a4f8dc930bf9d12ec7be7545 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 21:04:20 +0200 Subject: [PATCH 21/45] fix(runtime): Keep failed steering pending Apply Pi steering inside the durable injection callback so a steering failure rejects before mailbox records are marked injected. This keeps Slack follow-ups pending for a later worker instead of silently dropping them. Refs GH-470 Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond.ts | 15 ++++--- .../runtime/respond-provider-retry.test.ts | 45 ++++++++++++++++++- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index 1b80c7b8e..6134c6429 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -1123,23 +1123,24 @@ export async function generateAssistantReply( } try { - let piMessages: PiMessage[] = []; + let steeredMessageCount = 0; await context.drainSteeringMessages(async (messages) => { - piMessages = messages.map(buildSteeringPiMessage); + const piMessages = messages.map(buildSteeringPiMessage); if (piMessages.length === 0) { return; } await persistSafeBoundary([...agent!.state.messages, ...piMessages]); + for (const message of piMessages) { + agent!.steer(message); + } + steeredMessageCount += piMessages.length; }); - for (const message of piMessages) { - agent!.steer(message); - } - if (piMessages.length > 0) { + if (steeredMessageCount > 0) { logInfo( "agent_turn_steering_messages_injected", spanContext, { - "app.ai.steering_message_count": piMessages.length, + "app.ai.steering_message_count": steeredMessageCount, }, "Agent turn steering messages injected", ); diff --git a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts index 7076569a0..b7f3a39a6 100644 --- a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts +++ b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts @@ -3,7 +3,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const { agentMode, counters } = vi.hoisted(() => ({ agentMode: { - value: "providerRetry" as "providerRetry" | "steering", + value: "providerRetry" as + | "providerRetry" + | "steering" + | "steeringSteerThrows", }, counters: { continueCalls: 0, @@ -44,6 +47,9 @@ vi.mock("@earendil-works/pi-agent-core", () => { } steer(message: unknown) { + if (agentMode.value === "steeringSteerThrows") { + throw new Error("steer failed"); + } this.steeringMessages.push(message); } @@ -54,7 +60,10 @@ vi.mock("@earendil-works/pi-agent-core", () => { async prompt(message: unknown) { counters.promptCalls += 1; this.state.messages.push(message); - if (agentMode.value === "steering") { + if ( + agentMode.value === "steering" || + agentMode.value === "steeringSteerThrows" + ) { await this.prepareNextTurn?.(); this.state.messages.push(...this.steeringMessages); this.state.messages.push({ @@ -284,4 +293,36 @@ describe("generateAssistantReply provider retry", () => { expect(serializedMessages).toContain("help me"); expect(serializedMessages).toContain("actually do the other thing"); }); + + it("rejects steering injection when Pi steer fails", async () => { + agentMode.value = "steeringSteerThrows"; + let injectRejected = false; + let injectCompleted = false; + + await generateAssistantReply("help me", { + requester: { userId: "U123" }, + correlation: { + conversationId: "conversation-steering-failure", + turnId: "turn-steering-failure", + channelId: "C123", + threadTs: "1712345.0002", + }, + drainSteeringMessages: async (inject) => { + const messages = [ + { text: "actually do the other thing", timestampMs: 2_000 }, + ]; + try { + await inject(messages); + injectCompleted = true; + return messages; + } catch { + injectRejected = true; + throw new Error("inject rejected"); + } + }, + }); + + expect(injectRejected).toBe(true); + expect(injectCompleted).toBe(false); + }); }); From a691cf7a32342cb8aa2263de872801c6ff6f0c78 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 21:16:27 +0200 Subject: [PATCH 22/45] fix(runtime): Honor continuation safety gates Thread durable worker yield checks into Pi safe boundaries so Slack turns pause before starting another model iteration when the worker soft deadline has elapsed. Carry host request deadlines into resumed Slack turns and skip heartbeat timeout-resume recovery when the persisted conversation no longer marks that session as the active turn. Refs GH-470 Co-Authored-By: GPT-5 Codex --- .../src/chat/agent-dispatch/heartbeat.ts | 10 +++ packages/junior/src/chat/respond.ts | 16 +++++ .../junior/src/chat/runtime/reply-executor.ts | 2 + .../junior/src/chat/runtime/slack-resume.ts | 4 ++ .../junior/src/chat/runtime/slack-runtime.ts | 4 ++ .../src/chat/task-execution/slack-work.ts | 2 + packages/junior/src/handlers/turn-resume.ts | 27 ++++---- .../tests/integration/heartbeat.test.ts | 69 +++++++++++++++++++ .../integration/turn-resume-slack.test.ts | 3 + .../runtime/respond-provider-retry.test.ts | 34 +++++++++ 10 files changed, 159 insertions(+), 12 deletions(-) diff --git a/packages/junior/src/chat/agent-dispatch/heartbeat.ts b/packages/junior/src/chat/agent-dispatch/heartbeat.ts index bbd919ecc..9565697eb 100644 --- a/packages/junior/src/chat/agent-dispatch/heartbeat.ts +++ b/packages/junior/src/chat/agent-dispatch/heartbeat.ts @@ -4,6 +4,8 @@ import { getAwaitingTurnContinuationRequest, scheduleTurnTimeoutResume, } from "@/chat/services/timeout-resume"; +import { getPersistedThreadState } from "@/chat/runtime/thread-state"; +import { coerceThreadConversationState } from "@/chat/state/conversation"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { getVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; @@ -120,6 +122,14 @@ export async function recoverStaleTimeoutResumes(args: { } try { + const persistedState = await getPersistedThreadState( + summary.conversationId, + ); + const conversation = coerceThreadConversationState(persistedState); + if (conversation.processing.activeTurnId !== summary.sessionId) { + continue; + } + const request = await getAwaitingTurnContinuationRequest({ conversationId: summary.conversationId, sessionId: summary.sessionId, diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index 6134c6429..11b59d469 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -232,6 +232,8 @@ export interface ReplyRequestContext { drainSteeringMessages?: ( inject: (messages: ReplySteeringMessage[]) => Promise, ) => Promise; + /** Return true when the durable worker should pause at the next Pi boundary. */ + shouldYield?: () => boolean; onAuthPending?: ( pendingAuth: ConversationPendingAuthState, ) => void | Promise; @@ -1157,6 +1159,19 @@ export async function generateAssistantReply( ); } }; + const yieldAtSafeBoundaryIfDue = (): void => { + if (!context.shouldYield?.()) { + return; + } + + timedOut = true; + timeoutResumeMessages = [...agent!.state.messages]; + throw new Error( + `Agent turn yielded at a safe boundary after ${ + Date.now() - replyStartedAtMs + }ms`, + ); + }; agent = new Agent({ getApiKey: () => getPiGatewayApiKeyOverride(), @@ -1164,6 +1179,7 @@ export async function generateAssistantReply( steeringMode: "all", prepareNextTurn: async () => { await drainSteeringMessages(); + yieldAtSafeBoundaryIfDue(); return undefined; }, initialState: { diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index f05899425..df653622d 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -274,6 +274,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { drainSteeringMessages?: ( inject: (messages: QueuedTurnMessage[]) => Promise, ) => Promise; + shouldYield?: () => boolean; } = {}, ) { if (message.author.isMe) { @@ -749,6 +750,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { onStatus: (nextStatus) => status.update(nextStatus), onToolInvocation: options.onToolInvocation, drainSteeringMessages, + shouldYield: options.shouldYield, }, ); const diagnosticsContext = { diff --git a/packages/junior/src/chat/runtime/slack-resume.ts b/packages/junior/src/chat/runtime/slack-resume.ts index 62e670062..7ab157e36 100644 --- a/packages/junior/src/chat/runtime/slack-resume.ts +++ b/packages/junior/src/chat/runtime/slack-resume.ts @@ -34,6 +34,7 @@ import { type ProcessingReactionSession, } from "@/chat/runtime/processing-reaction"; import { buildAuthPauseResponse } from "@/chat/services/auth-pause-response"; +import { getTurnRequestDeadline } from "@/chat/runtime/request-deadline"; function resolveReplyTimeoutMs(explicitTimeoutMs?: number): number | undefined { if (typeof explicitTimeoutMs === "number" && explicitTimeoutMs > 0) { @@ -201,6 +202,7 @@ function createResumeReplyContext( statusSession: AssistantStatusSession, ): ReplyRequestContext { const replyContext = args.replyContext ?? {}; + const requestDeadline = getTurnRequestDeadline(); const threadId = args.lockKey ?? getDefaultLockKey(args.channelId, args.threadTs); const persistedChannelConfiguration = @@ -211,6 +213,8 @@ function createResumeReplyContext( return { ...replyContext, + turnDeadlineAtMs: + replyContext.turnDeadlineAtMs ?? requestDeadline?.deadlineAtMs, correlation: { ...replyContext.correlation, threadId: replyContext.correlation?.threadId ?? threadId, diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index 1f1f6d414..ceb1c6b13 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -51,6 +51,7 @@ export interface ReplyHooks { messageContext?: MessageContext; onToolInvocation?: (invocation: TurnToolInvocation) => void; onTurnStatePersisted?: () => Promise; + shouldYield?: () => boolean; } const THREAD_OPTOUT_ACK = @@ -149,6 +150,7 @@ export interface SlackTurnRuntimeDependencies { drainSteeringMessages?: ( inject: (messages: QueuedTurnMessage[]) => Promise, ) => Promise; + shouldYield?: () => boolean; }, ) => Promise; decideSubscribedReply: SubscribedReplyPolicy; @@ -411,6 +413,7 @@ export function createSlackTurnRuntime< onToolInvocation: toolInvocationHook, drainSteeringMessages, onTurnStatePersisted: hooks?.onTurnStatePersisted, + shouldYield: hooks?.shouldYield, }); }); } catch (error) { @@ -611,6 +614,7 @@ export function createSlackTurnRuntime< onToolInvocation: toolInvocationHook, drainSteeringMessages, onTurnStatePersisted: hooks?.onTurnStatePersisted, + shouldYield: hooks?.shouldYield, }); }); } catch (error) { diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index ff5f1cdbb..07a458bbb 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -291,6 +291,7 @@ export function createSlackConversationWorker( messageContext, drainSteeringMessages, onTurnStatePersisted, + shouldYield: context.shouldYield, }); return; } @@ -299,6 +300,7 @@ export function createSlackConversationWorker( messageContext, drainSteeringMessages, onTurnStatePersisted, + shouldYield: context.shouldYield, }); }, }); diff --git a/packages/junior/src/handlers/turn-resume.ts b/packages/junior/src/handlers/turn-resume.ts index cd918ee44..53608de11 100644 --- a/packages/junior/src/handlers/turn-resume.ts +++ b/packages/junior/src/handlers/turn-resume.ts @@ -6,6 +6,7 @@ * durable conversation queue. */ import { logException } from "@/chat/logging"; +import { runWithTurnRequestDeadline } from "@/chat/runtime/request-deadline"; import { resumeTimedOutTurnWithLockRetry } from "@/chat/runtime/timeout-resume-runner"; import { verifyTurnTimeoutResumeRequest } from "@/chat/services/timeout-resume"; import type { WaitUntilFn } from "@/handlers/types"; @@ -21,18 +22,20 @@ export async function POST( } waitUntil(() => - resumeTimedOutTurnWithLockRetry(payload).catch((error) => { - logException( - error, - "timeout_resume_handler_failed", - {}, - { - "app.ai.conversation_id": payload.conversationId, - "app.ai.session_id": payload.sessionId, - }, - "Timeout resume handler failed", - ); - }), + runWithTurnRequestDeadline(() => + resumeTimedOutTurnWithLockRetry(payload).catch((error) => { + logException( + error, + "timeout_resume_handler_failed", + {}, + { + "app.ai.conversation_id": payload.conversationId, + "app.ai.session_id": payload.sessionId, + }, + "Timeout resume handler failed", + ); + }), + ), ); return new Response("Accepted", { status: 202 }); } diff --git a/packages/junior/tests/integration/heartbeat.test.ts b/packages/junior/tests/integration/heartbeat.test.ts index 1bf3c92df..f4312967c 100644 --- a/packages/junior/tests/integration/heartbeat.test.ts +++ b/packages/junior/tests/integration/heartbeat.test.ts @@ -19,6 +19,7 @@ import { import type { DispatchRecord } from "@/chat/agent-dispatch/types"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { persistThreadStateById } from "@/chat/runtime/thread-state"; import { getConversationWorkState } from "@/chat/task-execution/store"; import type { PiMessage } from "@/chat/pi/messages"; import { setAgentPlugins } from "@/chat/plugins/agent-hooks"; @@ -150,6 +151,33 @@ function createCredentialSubject( return subject; } +async function persistActiveTurn( + conversationId: string, + activeTurnId?: string, +): Promise { + await persistThreadStateById(conversationId, { + conversation: { + schemaVersion: 1, + backfill: {}, + compactions: [], + messages: [], + piMessages: [], + processing: { + activeTurnId, + }, + stats: { + compactedMessageCount: 0, + estimatedContextTokens: 0, + totalMessageCount: 0, + updatedAtMs: TEST_NOW_MS, + }, + vision: { + byFileId: {}, + }, + }, + }); +} + describe("trusted plugin heartbeat", () => { const originalFetch = global.fetch; @@ -233,6 +261,7 @@ describe("trusted plugin heartbeat", () => { } as PiMessage, ], }); + await persistActiveTurn(conversationId, sessionId); vi.setSystemTime(TEST_NOW_MS); const waitUntilTasks: Promise[] = []; @@ -262,6 +291,46 @@ describe("trusted plugin heartbeat", () => { }); }); + it("skips stale timeout resume records for inactive turns", async () => { + const queue = new FakeConversationWorkQueue(); + const conversationId = "slack:C123:1712345.0007"; + const sessionId = "turn-timeout-inactive"; + const staleNowMs = TEST_NOW_MS - 3 * 60 * 1000; + vi.setSystemTime(staleNowMs); + await upsertAgentTurnSessionRecord({ + conversationId, + sessionId, + sliceId: 2, + state: "awaiting_resume", + resumeReason: "timeout", + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "finish this" }], + timestamp: staleNowMs, + } as PiMessage, + ], + }); + await persistActiveTurn(conversationId, "turn-newer"); + vi.setSystemTime(TEST_NOW_MS); + + const waitUntilTasks: Promise[] = []; + const response = await heartbeat( + new Request("https://example.invalid/api/internal/heartbeat", { + headers: { authorization: "Bearer heartbeat-secret" }, + }), + collectWaitUntil(waitUntilTasks), + { conversationWorkQueue: queue }, + ); + + expect(response.status).toBe(202); + await Promise.all(waitUntilTasks); + expect(queue.sent).toEqual([]); + await expect(getConversationWorkState({ conversationId })).resolves.toBe( + undefined, + ); + }); + it("scopes dispatch lookup to the plugin that created it", async () => { const fetchMock = vi.fn(async () => { return new Response("Accepted", { status: 202 }); diff --git a/packages/junior/tests/integration/turn-resume-slack.test.ts b/packages/junior/tests/integration/turn-resume-slack.test.ts index ca2b3bdf1..ed9b53ba5 100644 --- a/packages/junior/tests/integration/turn-resume-slack.test.ts +++ b/packages/junior/tests/integration/turn-resume-slack.test.ts @@ -214,7 +214,10 @@ describe("turn resume slack integration", () => { channelConfiguration?: { resolve: (key: string) => Promise; }; + turnDeadlineAtMs?: number; }; + expect(resumeContext.turnDeadlineAtMs).toEqual(expect.any(Number)); + expect(resumeContext.turnDeadlineAtMs).toBeGreaterThan(Date.now()); expect(await resumeContext.channelConfiguration?.resolve("demo.org")).toBe( "acme", ); diff --git a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts index b7f3a39a6..fc45b25e3 100644 --- a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts +++ b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts @@ -5,6 +5,7 @@ const { agentMode, counters } = vi.hoisted(() => ({ agentMode: { value: "providerRetry" as | "providerRetry" + | "cooperativeYield" | "steering" | "steeringSteerThrows", }, @@ -61,6 +62,7 @@ vi.mock("@earendil-works/pi-agent-core", () => { counters.promptCalls += 1; this.state.messages.push(message); if ( + agentMode.value === "cooperativeYield" || agentMode.value === "steering" || agentMode.value === "steeringSteerThrows" ) { @@ -207,6 +209,7 @@ vi.mock("@/chat/skills", async (importOriginal) => ({ import { generateAssistantReply } from "@/chat/respond"; import { disconnectStateAdapter } from "@/chat/state/adapter"; import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { isRetryableTurnError } from "@/chat/runtime/turn"; describe("generateAssistantReply provider retry", () => { beforeEach(async () => { @@ -294,6 +297,37 @@ describe("generateAssistantReply provider retry", () => { expect(serializedMessages).toContain("actually do the other thing"); }); + it("parks the turn when the worker asks to yield at a Pi boundary", async () => { + agentMode.value = "cooperativeYield"; + + const error = await generateAssistantReply("help me", { + requester: { userId: "U123" }, + correlation: { + conversationId: "conversation-yield", + turnId: "turn-yield", + channelId: "C123", + threadTs: "1712345.0003", + }, + shouldYield: () => true, + }).then( + () => undefined, + (caught: unknown) => caught, + ); + + expect(isRetryableTurnError(error, "turn_timeout_resume")).toBe(true); + const sessionRecord = await getAgentTurnSessionRecord( + "conversation-yield", + "turn-yield", + ); + expect(sessionRecord).toMatchObject({ + state: "awaiting_resume", + resumeReason: "timeout", + }); + expect(sessionRecord?.piMessages.map((message) => message.role)).toEqual([ + "user", + ]); + }); + it("rejects steering injection when Pi steer fails", async () => { agentMode.value = "steeringSteerThrows"; let injectRejected = false; From 11702aa81b072d5945c676c45123ae74f7b9307f Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 22:33:50 +0200 Subject: [PATCH 23/45] fix(runtime): Separate cooperative yield resumes Persist soft-yield boundaries with a distinct yield resume reason so routine worker continuation does not consume timeout resume slices. Bubble cooperative yield through Slack runtime handling so the generic conversation worker releases the lease and requeues the next slice at the durable worker boundary. Refs GH-470 Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond.ts | 40 ++++++++++- .../junior/src/chat/runtime/reply-executor.ts | 10 ++- .../junior/src/chat/runtime/slack-runtime.ts | 11 ++- .../src/chat/runtime/timeout-resume-runner.ts | 3 +- packages/junior/src/chat/runtime/turn.ts | 16 +++++ .../src/chat/services/timeout-resume.ts | 5 +- .../src/chat/services/turn-session-record.ts | 72 +++++++++++++++++++ .../junior/src/chat/state/turn-session.ts | 6 +- .../src/chat/task-execution/slack-work.ts | 46 +++++++----- .../slack-conversation-work.test.ts | 62 ++++++++++++++++ .../runtime/respond-provider-retry.test.ts | 18 ++++- 11 files changed, 260 insertions(+), 29 deletions(-) diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index 11b59d469..c23d0a571 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -82,7 +82,11 @@ import type { AssistantStatusSpec } from "@/chat/slack/assistant-thread/status"; import type { SlackConversationContext } from "@/chat/slack/conversation-context"; import { createAgentTools } from "@/chat/tools/agent-tools"; import { mergeArtifactsState } from "@/chat/runtime/thread-state"; -import { RetryableTurnError, isRetryableTurnError } from "@/chat/runtime/turn"; +import { + CooperativeTurnYieldError, + RetryableTurnError, + isRetryableTurnError, +} from "@/chat/runtime/turn"; import { buildUserTurnText, encodeNonImageAttachmentForPrompt, @@ -119,6 +123,7 @@ import { persistAuthPauseSessionRecord, persistRunningSessionRecord, persistTimeoutSessionRecord, + persistYieldSessionRecord, } from "@/chat/services/turn-session-record"; import type { AgentTurnRequester } from "@/chat/state/turn-session"; import type { CredentialContext } from "@/chat/credentials/context"; @@ -463,6 +468,7 @@ export async function generateAssistantReply( let canRecordMcpProviders = false; let sandboxExecutor: SandboxExecutor | undefined; let timedOut = false; + let yielded = false; let turnUsage: AgentTurnUsage | undefined; let thinkingSelection: TurnThinkingSelection | undefined; const requester = requesterFromContext( @@ -1164,9 +1170,9 @@ export async function generateAssistantReply( return; } - timedOut = true; + yielded = true; timeoutResumeMessages = [...agent!.state.messages]; - throw new Error( + throw new CooperativeTurnYieldError( `Agent turn yielded at a safe boundary after ${ Date.now() - replyStartedAtMs }ms`, @@ -1477,6 +1483,34 @@ export async function generateAssistantReply( assistantUserName: botConfig.userName, }); } catch (error) { + if ( + yielded && + error instanceof CooperativeTurnYieldError && + timeoutResumeConversationId && + timeoutResumeSessionId + ) { + turnUsage = + turnUsage ?? + extractSliceUsage(timeoutResumeMessages, beforeMessageCount); + await recordActiveMcpProviders(); + const sessionRecord = await persistYieldSessionRecord({ + channelName: context.correlation?.channelName, + conversationId: timeoutResumeConversationId, + sessionId: timeoutResumeSessionId, + currentSliceId: timeoutResumeSliceId, + currentDurationMs: Date.now() - replyStartedAtMs, + currentUsage: turnUsage, + messages: timeoutResumeMessages, + errorMessage: error.message, + loadedSkillNames: loadedSkillNamesForResume, + logContext: sessionRecordLogContext, + requester, + }); + if (sessionRecord) { + throw error; + } + } + if (timedOut && timeoutResumeConversationId && timeoutResumeSessionId) { turnUsage = turnUsage ?? diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index df653622d..9795508d3 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -83,7 +83,10 @@ import { appendSlackLegacyAttachmentText } from "@/chat/slack/legacy-attachments import { type ThreadArtifactsState } from "@/chat/state/artifacts"; import { lookupSlackUser } from "@/chat/slack/user"; import type { TurnContinuationRequest } from "@/chat/services/timeout-resume"; -import { isRetryableTurnError } from "@/chat/runtime/turn"; +import { + isCooperativeTurnYieldError, + isRetryableTurnError, +} from "@/chat/runtime/turn"; import { buildDeterministicTurnId } from "@/chat/runtime/turn"; import { markTurnClosed, markTurnFailed } from "@/chat/runtime/turn"; import { startActiveTurn } from "@/chat/runtime/turn"; @@ -913,6 +916,11 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { ); } } catch (error) { + if (isCooperativeTurnYieldError(error)) { + shouldPersistFailureState = false; + throw error; + } + if ( isRetryableTurnError(error, "mcp_auth_resume") || isRetryableTurnError(error, "plugin_auth_resume") diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index ceb1c6b13..774c8c2f0 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -8,7 +8,10 @@ */ import type { Message, MessageContext, Thread } from "chat"; import { getSubscribedReplyPreflightDecision } from "@/chat/services/subscribed-decision"; -import { isRetryableTurnError } from "@/chat/runtime/turn"; +import { + isCooperativeTurnYieldError, + isRetryableTurnError, +} from "@/chat/runtime/turn"; import { buildTurnFailureResponse } from "@/chat/logging"; import { getSlackErrorObservabilityAttributes } from "@/chat/slack/errors"; import type { @@ -417,6 +420,9 @@ export function createSlackTurnRuntime< }); }); } catch (error) { + if (isCooperativeTurnYieldError(error)) { + throw error; + } const errorContext = logContext({ threadId: deps.getThreadId(thread, message), requesterId: message.author.userId, @@ -618,6 +624,9 @@ export function createSlackTurnRuntime< }); }); } catch (error) { + if (isCooperativeTurnYieldError(error)) { + throw error; + } const errorContext = logContext({ threadId: deps.getThreadId(thread, message), requesterId: message.author.userId, diff --git a/packages/junior/src/chat/runtime/timeout-resume-runner.ts b/packages/junior/src/chat/runtime/timeout-resume-runner.ts index f733bbd99..db717815c 100644 --- a/packages/junior/src/chat/runtime/timeout-resume-runner.ts +++ b/packages/junior/src/chat/runtime/timeout-resume-runner.ts @@ -149,7 +149,8 @@ export async function resumeTimedOutTurn( if ( !sessionRecord || sessionRecord.state !== "awaiting_resume" || - sessionRecord.resumeReason !== "timeout" || + (sessionRecord.resumeReason !== "timeout" && + sessionRecord.resumeReason !== "yield") || sessionRecord.version !== payload.expectedVersion ) { return false; diff --git a/packages/junior/src/chat/runtime/turn.ts b/packages/junior/src/chat/runtime/turn.ts index 7db188277..6ee27e7c7 100644 --- a/packages/junior/src/chat/runtime/turn.ts +++ b/packages/junior/src/chat/runtime/turn.ts @@ -61,6 +61,22 @@ export function isRetryableTurnError( return error.reason === reason; } +/** Error indicating the turn paused voluntarily at a safe continuation boundary. */ +export class CooperativeTurnYieldError extends Error { + readonly code = "cooperative_turn_yield"; + + constructor(message = "Agent turn yielded at a safe boundary") { + super(message); + this.name = "CooperativeTurnYieldError"; + } +} + +export function isCooperativeTurnYieldError( + error: unknown, +): error is CooperativeTurnYieldError { + return error instanceof CooperativeTurnYieldError; +} + // --------------------------------------------------------------------------- // Turn lifecycle mutations // --------------------------------------------------------------------------- diff --git a/packages/junior/src/chat/services/timeout-resume.ts b/packages/junior/src/chat/services/timeout-resume.ts index 29e304bdb..56f5557e6 100644 --- a/packages/junior/src/chat/services/timeout-resume.ts +++ b/packages/junior/src/chat/services/timeout-resume.ts @@ -45,8 +45,9 @@ export async function getAwaitingTurnContinuationRequest(args: { if ( !sessionRecord || sessionRecord.state !== "awaiting_resume" || - sessionRecord.resumeReason !== "timeout" || - sessionRecord.sliceId < 2 + (sessionRecord.resumeReason !== "timeout" && + sessionRecord.resumeReason !== "yield") || + (sessionRecord.resumeReason === "timeout" && sessionRecord.sliceId < 2) ) { return undefined; } diff --git a/packages/junior/src/chat/services/turn-session-record.ts b/packages/junior/src/chat/services/turn-session-record.ts index e9e37f550..7c87ccb89 100644 --- a/packages/junior/src/chat/services/turn-session-record.ts +++ b/packages/junior/src/chat/services/turn-session-record.ts @@ -417,3 +417,75 @@ export async function persistTimeoutSessionRecord(args: { return undefined; } } + +/** + * Persist a cooperative-yield boundary without advancing timeout slice counts. + */ +export async function persistYieldSessionRecord(args: { + channelName?: string; + conversationId: string; + sessionId: string; + currentSliceId: number; + currentDurationMs?: number; + currentUsage?: AgentTurnUsage; + messages: PiMessage[]; + loadedSkillNames?: string[]; + errorMessage: string; + logContext: SessionRecordLogContext; + requester?: AgentTurnRequester; +}): Promise { + try { + const latestSessionRecord = await getAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + const piMessages = resumableBoundary( + args.messages, + latestSessionRecord?.piMessages, + ); + if (piMessages.length === 0 || !isContinuableBoundary(piMessages)) { + return undefined; + } + return await upsertAgentTurnSessionRecord({ + ...((args.channelName ?? latestSessionRecord?.channelName) + ? { channelName: args.channelName ?? latestSessionRecord?.channelName } + : {}), + conversationId: args.conversationId, + cumulativeDurationMs: addDurationMs( + latestSessionRecord?.cumulativeDurationMs, + args.currentDurationMs, + ), + cumulativeUsage: addAgentTurnUsage( + latestSessionRecord?.cumulativeUsage, + args.currentUsage, + ), + sessionId: args.sessionId, + sliceId: args.currentSliceId, + state: "awaiting_resume", + piMessages, + ...(args.loadedSkillNames + ? { loadedSkillNames: args.loadedSkillNames } + : {}), + resumeReason: "yield", + resumedFromSliceId: latestSessionRecord?.resumedFromSliceId, + errorMessage: args.errorMessage, + ...((args.requester ?? latestSessionRecord?.requester) + ? { requester: args.requester ?? latestSessionRecord?.requester } + : {}), + ...((getActiveTraceId() ?? latestSessionRecord?.traceId) + ? { traceId: getActiveTraceId() ?? latestSessionRecord?.traceId } + : {}), + }); + } catch (recordError) { + logSessionRecordError( + recordError, + "agent_turn_yield_session_record_failed", + args, + { + "app.ai.resume_slice_id": args.currentSliceId, + }, + "Failed to persist cooperative yield session record", + ); + return undefined; + } +} diff --git a/packages/junior/src/chat/state/turn-session.ts b/packages/junior/src/chat/state/turn-session.ts index 57531215e..cb59e3326 100644 --- a/packages/junior/src/chat/state/turn-session.ts +++ b/packages/junior/src/chat/state/turn-session.ts @@ -25,7 +25,7 @@ export type AgentTurnSessionStatus = | "failed" | "abandoned"; -export type AgentTurnResumeReason = "timeout" | "auth"; +export type AgentTurnResumeReason = "timeout" | "auth" | "yield"; export interface AgentTurnRequester { email?: string; @@ -231,7 +231,9 @@ function parseAgentTurnSessionFields( ), } : {}), - ...(parsed.resumeReason === "timeout" || parsed.resumeReason === "auth" + ...(parsed.resumeReason === "timeout" || + parsed.resumeReason === "auth" || + parsed.resumeReason === "yield" ? { resumeReason: parsed.resumeReason } : {}), ...(typeof parsed.errorMessage === "string" diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 07a458bbb..a6997c58a 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -8,6 +8,7 @@ import { type StateAdapter, } from "chat"; import type { SlackTurnRuntime } from "@/chat/runtime/slack-runtime"; +import { isCooperativeTurnYieldError } from "@/chat/runtime/turn"; import { normalizeIncomingSlackThreadId } from "@/chat/ingress/message-router"; import { rehydrateAttachmentFetchers } from "@/chat/queue/thread-message-dispatcher"; import { getAwaitingTurnContinuationRequest } from "@/chat/services/timeout-resume"; @@ -133,17 +134,20 @@ function restoreThread(args: { }); } -function latestTimeoutResume( +function latestContinuationResume( summaries: AgentTurnSessionSummary[], ): AgentTurnSessionSummary | undefined { return summaries.find( (summary) => - summary.state === "awaiting_resume" && summary.resumeReason === "timeout", + summary.state === "awaiting_resume" && + (summary.resumeReason === "timeout" || summary.resumeReason === "yield"), ); } -async function resumeAwaitingTimeout(conversationId: string): Promise { - const summary = latestTimeoutResume( +async function resumeAwaitingContinuation( + conversationId: string, +): Promise { + const summary = latestContinuationResume( await listAgentTurnSessionSummariesForConversation(conversationId), ); if (!summary) { @@ -200,7 +204,7 @@ export function createSlackConversationWorker( }), ); if (records.length === 0) { - await resumeAwaitingTimeout(context.conversationId); + await resumeAwaitingContinuation(context.conversationId); return { status: "completed" }; } @@ -220,7 +224,7 @@ export function createSlackConversationWorker( return { status: "lost_lease" }; } - await runWithSlackInstallation({ + const turnResult = await runWithSlackInstallation({ adapter, installation: getInstallation(records), state, @@ -286,24 +290,34 @@ export function createSlackConversationWorker( ); }; - if (route === "mention") { - await options.runtime.handleNewMention(thread, latestMessage, { + try { + if (route === "mention") { + await options.runtime.handleNewMention(thread, latestMessage, { + messageContext, + drainSteeringMessages, + onTurnStatePersisted, + shouldYield: context.shouldYield, + }); + return; + } + + await options.runtime.handleSubscribedMessage(thread, latestMessage, { messageContext, drainSteeringMessages, onTurnStatePersisted, shouldYield: context.shouldYield, }); - return; + } catch (error) { + if (isCooperativeTurnYieldError(error)) { + return { status: "yielded" } satisfies ConversationWorkerResult; + } + throw error; } - - await options.runtime.handleSubscribedMessage(thread, latestMessage, { - messageContext, - drainSteeringMessages, - onTurnStatePersisted, - shouldYield: context.shouldYield, - }); }, }); + if (turnResult?.status === "yielded") { + return turnResult; + } const messagesMarked = await markConversationMessagesInjected({ conversationId: context.conversationId, diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index 379cf9406..dd85239b1 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { Message, Thread } from "chat"; +import { CooperativeTurnYieldError } from "@/chat/runtime/turn"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; import { CONVERSATION_WORK_LEASE_TTL_MS, @@ -676,4 +677,65 @@ describe("Slack conversation work execution", () => { expect(work?.needsRun).toBe(false); expect(work ? countPendingConversationMessages(work) : 0).toBe(0); }); + + it("yields Slack mailbox work after a persisted safe boundary", async () => { + const queue = createFakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + let currentNowMs = 1_000; + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> first`, + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + queue.sent = []; + + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (_thread, _message, hooks) => { + await hooks?.onTurnStatePersisted?.(); + currentNowMs = 242_000; + throw new CooperativeTurnYieldError(); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "yielded" }); + + expect(queue.sent).toMatchObject([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `yield:${CONVERSATION_ID}:242000`, + }, + ]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.lease).toBeUndefined(); + expect(work?.needsRun).toBe(true); + expect(work?.messages.map((message) => message.injectedAtMs)).toEqual([ + expect.any(Number), + ]); + }); }); diff --git a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts index fc45b25e3..bf0937a11 100644 --- a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts +++ b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts @@ -207,9 +207,10 @@ vi.mock("@/chat/skills", async (importOriginal) => ({ })); import { generateAssistantReply } from "@/chat/respond"; +import { isCooperativeTurnYieldError } from "@/chat/runtime/turn"; +import { getAwaitingTurnContinuationRequest } from "@/chat/services/timeout-resume"; import { disconnectStateAdapter } from "@/chat/state/adapter"; import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; -import { isRetryableTurnError } from "@/chat/runtime/turn"; describe("generateAssistantReply provider retry", () => { beforeEach(async () => { @@ -314,18 +315,29 @@ describe("generateAssistantReply provider retry", () => { (caught: unknown) => caught, ); - expect(isRetryableTurnError(error, "turn_timeout_resume")).toBe(true); + expect(isCooperativeTurnYieldError(error)).toBe(true); const sessionRecord = await getAgentTurnSessionRecord( "conversation-yield", "turn-yield", ); expect(sessionRecord).toMatchObject({ state: "awaiting_resume", - resumeReason: "timeout", + resumeReason: "yield", + sliceId: 1, }); expect(sessionRecord?.piMessages.map((message) => message.role)).toEqual([ "user", ]); + await expect( + getAwaitingTurnContinuationRequest({ + conversationId: "conversation-yield", + sessionId: "turn-yield", + }), + ).resolves.toMatchObject({ + conversationId: "conversation-yield", + sessionId: "turn-yield", + expectedVersion: sessionRecord?.version, + }); }); it("rejects steering injection when Pi steer fails", async () => { From 7da20794a0c4719a81c04dcbcbd399b44afa5234 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 22:40:27 +0200 Subject: [PATCH 24/45] fix(runtime): Recover continuation work after stale leases Mark expired conversation leases runnable so recovered queue nudges can reach continuation scanning even when mailbox messages were already injected. Include cooperative yield records in stale continuation heartbeat recovery. Co-Authored-By: GPT-5 Codex --- .../src/chat/agent-dispatch/heartbeat.ts | 5 +- .../junior/src/chat/task-execution/store.ts | 1 + .../task-execution/conversation-work.test.ts | 33 ++++++++++++ .../slack-conversation-work.test.ts | 2 +- .../tests/integration/heartbeat.test.ts | 50 +++++++++++++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/packages/junior/src/chat/agent-dispatch/heartbeat.ts b/packages/junior/src/chat/agent-dispatch/heartbeat.ts index 9565697eb..926f27dec 100644 --- a/packages/junior/src/chat/agent-dispatch/heartbeat.ts +++ b/packages/junior/src/chat/agent-dispatch/heartbeat.ts @@ -99,7 +99,7 @@ async function runWithTimeout( } } -/** Re-drive stale turn timeout continuations whose internal callback vanished. */ +/** Re-drive stale turn continuations whose internal callback vanished. */ export async function recoverStaleTimeoutResumes(args: { conversationWorkQueue?: ConversationWorkQueue; limit?: number; @@ -115,7 +115,8 @@ export async function recoverStaleTimeoutResumes(args: { } if ( summary.state !== "awaiting_resume" || - summary.resumeReason !== "timeout" || + (summary.resumeReason !== "timeout" && + summary.resumeReason !== "yield") || summary.updatedAtMs + TIMEOUT_RESUME_STALE_MS > args.nowMs ) { continue; diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index 6979c5bcf..45642fb86 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -803,6 +803,7 @@ export async function clearExpiredConversationLease(args: { await writeWorkState(state, { ...current, lease: undefined, + needsRun: true, updatedAtMs: nowMs, }); return true; diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index edba6f566..1d0cb94da 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -330,6 +330,7 @@ describe("conversation work execution", () => { conversationId: CONVERSATION_ID, }); expect(state?.lease).toBeUndefined(); + expect(state?.needsRun).toBe(true); expect(queue.sent).toMatchObject([ { conversationId: CONVERSATION_ID, @@ -338,6 +339,38 @@ describe("conversation work execution", () => { ]); }); + it("keeps an expired injected-message lease runnable for continuation recovery", async () => { + const queue = createFakeQueue(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + const lease = await startConversationWork({ + conversationId: CONVERSATION_ID, + nowMs: 2_000, + }); + expect(lease.status).toBe("acquired"); + if (lease.status !== "acquired") { + return; + } + await markConversationMessagesInjected({ + conversationId: CONVERSATION_ID, + inboundMessageIds: ["m1"], + leaseToken: lease.leaseToken, + nowMs: 3_000, + }); + + await expect( + recoverConversationWork({ + nowMs: 2_000 + CONVERSATION_WORK_LEASE_TTL_MS, + queue, + }), + ).resolves.toEqual({ expiredLeaseCount: 1, pendingCount: 0 }); + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + run: async () => ({ status: "completed" }), + }), + ).resolves.toEqual({ status: "completed" }); + }); + it("requeues pending mailbox work with no recent queue marker", async () => { const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index dd85239b1..ffa9213f1 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -517,7 +517,7 @@ describe("Slack conversation work execution", () => { state, }), }), - ).resolves.toEqual({ status: "no_work" }); + ).resolves.toEqual({ status: "completed" }); const recovered = await getConversationWorkState({ conversationId: CONVERSATION_ID, diff --git a/packages/junior/tests/integration/heartbeat.test.ts b/packages/junior/tests/integration/heartbeat.test.ts index f4312967c..26e2d6514 100644 --- a/packages/junior/tests/integration/heartbeat.test.ts +++ b/packages/junior/tests/integration/heartbeat.test.ts @@ -291,6 +291,56 @@ describe("trusted plugin heartbeat", () => { }); }); + it("reschedules stale cooperative yield resume records", async () => { + const queue = new FakeConversationWorkQueue(); + const conversationId = "slack:C123:1712345.0008"; + const sessionId = "turn-yield"; + const staleNowMs = TEST_NOW_MS - 3 * 60 * 1000; + vi.setSystemTime(staleNowMs); + await upsertAgentTurnSessionRecord({ + conversationId, + sessionId, + sliceId: 1, + state: "awaiting_resume", + resumeReason: "yield", + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "keep going" }], + timestamp: staleNowMs, + } as PiMessage, + ], + }); + await persistActiveTurn(conversationId, sessionId); + vi.setSystemTime(TEST_NOW_MS); + + const waitUntilTasks: Promise[] = []; + const response = await heartbeat( + new Request("https://example.invalid/api/internal/heartbeat", { + headers: { authorization: "Bearer heartbeat-secret" }, + }), + collectWaitUntil(waitUntilTasks), + { conversationWorkQueue: queue }, + ); + + expect(response.status).toBe(202); + await Promise.all(waitUntilTasks); + expect(queue.sent).toEqual([ + { + conversationId, + idempotencyKey: expect.stringContaining( + `timeout:${conversationId}:${sessionId}:`, + ), + }, + ]); + await expect( + getConversationWorkState({ conversationId }), + ).resolves.toMatchObject({ + conversationId, + needsRun: true, + }); + }); + it("skips stale timeout resume records for inactive turns", async () => { const queue = new FakeConversationWorkQueue(); const conversationId = "slack:C123:1712345.0007"; From 228eb3b8b8b53648c8ecc1a7aa4291824b91fec8 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 22:47:45 +0200 Subject: [PATCH 25/45] fix(runtime): Preserve invalid idle continuations Treat a timeout or yield continuation summary without a valid resume request as a worker failure instead of completed idle work. This keeps the conversation runnable for queue retry or heartbeat repair instead of clearing needsRun after recovered idle work. Co-Authored-By: GPT-5 Codex --- .../src/chat/task-execution/slack-work.ts | 4 +- .../slack-conversation-work.test.ts | 51 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index a6997c58a..4fce39c4c 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -159,7 +159,9 @@ async function resumeAwaitingContinuation( sessionId: summary.sessionId, }); if (!request) { - return; + throw new Error( + `Unable to build continuation request for turn session "${summary.sessionId}" in conversation "${conversationId}"`, + ); } await resumeTimedOutTurnWithLockRetry(request); diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index ffa9213f1..6ffac9600 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -7,6 +7,7 @@ import { countPendingConversationMessages, getConversationWorkState, markConversationMessagesInjected, + requestConversationWork, startConversationWork, } from "@/chat/task-execution/store"; import { processConversationWork } from "@/chat/task-execution/worker"; @@ -527,6 +528,56 @@ describe("Slack conversation work execution", () => { expect(recovered ? countPendingConversationMessages(recovered) : 0).toBe(0); }); + it("keeps idle Slack work runnable when continuation metadata is invalid", async () => { + const queue = createFakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + + await requestConversationWork({ + conversationId: CONVERSATION_ID, + nowMs: 1_000, + state, + }); + await upsertAgentTurnSessionRecord({ + conversationId: CONVERSATION_ID, + sessionId: "turn-invalid-timeout", + sliceId: 1, + state: "awaiting_resume", + resumeReason: "timeout", + piMessages: [], + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + throw new Error("injected messages should not replay"); + }, + handleSubscribedMessage: async () => { + throw new Error("injected messages should not replay"); + }, + }, + state, + }), + }), + ).rejects.toThrow( + 'Unable to build continuation request for turn session "turn-invalid-timeout"', + ); + + const recovered = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(recovered?.lease).toBeUndefined(); + expect(recovered?.needsRun).toBe(true); + expect(recovered?.messages).toEqual([]); + }); + it("keeps Slack mailbox records pending when the runtime handoff fails", async () => { const queue = createFakeQueue(); const state = getStateAdapter(); From 69724215bf90c3810b82986b1c01ee550125be12 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 23:19:02 +0200 Subject: [PATCH 26/45] fix(runtime): Terminalize stale continuations Make Slack resume startup report whether a continuation actually started so idle durable work can distinguish a real resume from a stale no-op. Terminalize invalid or skipped awaiting timeout/yield sessions before completing idle work, and cover both invalid metadata and stale active-turn mismatch with component tests. Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/runtime/slack-resume.ts | 19 +-- .../src/chat/runtime/timeout-resume-runner.ts | 27 ++-- .../src/chat/task-execution/slack-work.ts | 73 +++++++---- .../slack-conversation-work.test.ts | 116 +++++++++++++++++- 4 files changed, 190 insertions(+), 45 deletions(-) diff --git a/packages/junior/src/chat/runtime/slack-resume.ts b/packages/junior/src/chat/runtime/slack-resume.ts index 7ab157e36..be70bb2ca 100644 --- a/packages/junior/src/chat/runtime/slack-resume.ts +++ b/packages/junior/src/chat/runtime/slack-resume.ts @@ -245,12 +245,13 @@ function createResumeReplyContext( /** * Resume a paused Slack turn under the normal thread lock. * - * Success is defined by final reply delivery, not only by successful assistant - * generation. If the final visible Slack post fails, the resumed turn is - * treated as failed so thread state does not claim the user saw a reply that - * never arrived. + * Started resumes own their terminal side effects: final delivery, pause + * persistence, or failure response. Returns false only when `beforeStart` + * proves the resume is stale before generation begins. */ -export async function resumeSlackTurn(args: ResumeSlackTurnArgs) { +export async function resumeSlackTurn( + args: ResumeSlackTurnArgs, +): Promise { const stateAdapter = getStateAdapter(); await stateAdapter.connect(); const lockKey = @@ -274,7 +275,7 @@ export async function resumeSlackTurn(args: ResumeSlackTurnArgs) { try { const preparedArgs = await args.beforeStart?.(); if (preparedArgs === false) { - return; + return false; } if (preparedArgs) { runArgs = { ...args, ...preparedArgs }; @@ -446,7 +447,7 @@ export async function resumeSlackTurn(args: ResumeSlackTurnArgs) { buildAuthPauseResponse(), ); } - return; + return true; } catch (pauseError) { await handleResumeFailure({ body: "Failed to handle resumed turn pause", @@ -455,13 +456,15 @@ export async function resumeSlackTurn(args: ResumeSlackTurnArgs) { lockKey, resumeArgs: runArgs, }); - return; + return true; } } if (deferredFailureHandler) { await deferredFailureHandler(); } + + return true; } /** Resume an OAuth-paused Slack request through the shared resume runner. */ diff --git a/packages/junior/src/chat/runtime/timeout-resume-runner.ts b/packages/junior/src/chat/runtime/timeout-resume-runner.ts index db717815c..d21b61b31 100644 --- a/packages/junior/src/chat/runtime/timeout-resume-runner.ts +++ b/packages/junior/src/chat/runtime/timeout-resume-runner.ts @@ -125,10 +125,14 @@ async function persistFailedReplyState( }); } -/** Resume one durable timeout continuation for a Slack thread. */ +/** + * Resume one durable timeout continuation for a Slack thread. + * + * Returns false when the session became stale before generation began. + */ export async function resumeTimedOutTurn( payload: TurnContinuationRequest, -): Promise { +): Promise { const thread = parseSlackThreadId(payload.conversationId); if (!thread) { throw new Error( @@ -136,7 +140,7 @@ export async function resumeTimedOutTurn( ); } - await resumeSlackTurn({ + return await resumeSlackTurn({ messageText: "", channelId: thread.channelId, threadTs: thread.threadTs, @@ -267,17 +271,22 @@ export async function resumeTimedOutTurn( }); } -/** Retry timeout continuation when the normal Slack thread lock is briefly busy. */ +/** + * Retry timeout continuation when the normal Slack thread lock is briefly busy. + * + * Returns false when the session became stale before generation began. A busy + * lock that is rescheduled still returns true because runnable work remains + * durable. + */ export async function resumeTimedOutTurnWithLockRetry( payload: TurnContinuationRequest, -): Promise { +): Promise { for (const [attempt, delayMs] of [ ...TIMEOUT_RESUME_LOCK_RETRY_DELAYS_MS, undefined, ].entries()) { try { - await resumeTimedOutTurn(payload); - return; + return await resumeTimedOutTurn(payload); } catch (error) { if (!(error instanceof ResumeTurnBusyError)) { throw error; @@ -294,7 +303,7 @@ export async function resumeTimedOutTurnWithLockRetry( "Rescheduling timeout resume because another turn still owns the thread lock", ); await scheduleTurnTimeoutResume(payload); - return; + return true; } logWarn( @@ -311,4 +320,6 @@ export async function resumeTimedOutTurnWithLockRetry( await sleep(delayMs); } } + + return true; } diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 4fce39c4c..e29d6814f 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -14,6 +14,7 @@ import { rehydrateAttachmentFetchers } from "@/chat/queue/thread-message-dispatc import { getAwaitingTurnContinuationRequest } from "@/chat/services/timeout-resume"; import { resumeTimedOutTurnWithLockRetry } from "@/chat/runtime/timeout-resume-runner"; import { + failAgentTurnSessionRecord, listAgentTurnSessionSummariesForConversation, type AgentTurnSessionSummary, } from "@/chat/state/turn-session"; @@ -134,37 +135,63 @@ function restoreThread(args: { }); } -function latestContinuationResume( - summaries: AgentTurnSessionSummary[], -): AgentTurnSessionSummary | undefined { - return summaries.find( - (summary) => - summary.state === "awaiting_resume" && - (summary.resumeReason === "timeout" || summary.resumeReason === "yield"), +function isContinuationResume(summary: AgentTurnSessionSummary): boolean { + return ( + summary.state === "awaiting_resume" && + (summary.resumeReason === "timeout" || summary.resumeReason === "yield") ); } +async function failUnresumableContinuation(args: { + conversationId: string; + errorMessage: string; + expectedVersion?: number; + summary: AgentTurnSessionSummary; +}): Promise { + await failAgentTurnSessionRecord({ + conversationId: args.conversationId, + expectedVersion: args.expectedVersion ?? args.summary.version, + sessionId: args.summary.sessionId, + errorMessage: args.errorMessage, + }); +} + async function resumeAwaitingContinuation( conversationId: string, ): Promise { - const summary = latestContinuationResume( - await listAgentTurnSessionSummariesForConversation(conversationId), - ); - if (!summary) { - return; - } + const summaries = + await listAgentTurnSessionSummariesForConversation(conversationId); - const request = await getAwaitingTurnContinuationRequest({ - conversationId, - sessionId: summary.sessionId, - }); - if (!request) { - throw new Error( - `Unable to build continuation request for turn session "${summary.sessionId}" in conversation "${conversationId}"`, - ); - } + for (const summary of summaries) { + if (!isContinuationResume(summary)) { + continue; + } + + const request = await getAwaitingTurnContinuationRequest({ + conversationId, + sessionId: summary.sessionId, + }); + if (!request) { + await failUnresumableContinuation({ + conversationId, + summary, + errorMessage: + "Awaiting turn continuation metadata could not be materialized", + }); + continue; + } - await resumeTimedOutTurnWithLockRetry(request); + if (await resumeTimedOutTurnWithLockRetry(request)) { + return; + } + + await failUnresumableContinuation({ + conversationId, + expectedVersion: request.expectedVersion, + summary, + errorMessage: "Awaiting turn continuation was stale before resuming", + }); + } } function getInstallation( diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index 6ffac9600..3bd6101ca 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -13,7 +13,11 @@ import { import { processConversationWork } from "@/chat/task-execution/worker"; import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; -import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { + getAgentTurnSessionRecord, + upsertAgentTurnSessionRecord, +} from "@/chat/state/turn-session"; +import { persistThreadStateById } from "@/chat/runtime/thread-state"; import { CONVERSATION_ID, createFakeQueue, @@ -528,7 +532,7 @@ describe("Slack conversation work execution", () => { expect(recovered ? countPendingConversationMessages(recovered) : 0).toBe(0); }); - it("keeps idle Slack work runnable when continuation metadata is invalid", async () => { + it("terminalizes invalid idle continuation metadata", async () => { const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); @@ -565,17 +569,117 @@ describe("Slack conversation work execution", () => { state, }), }), - ).rejects.toThrow( - 'Unable to build continuation request for turn session "turn-invalid-timeout"', - ); + ).resolves.toEqual({ status: "completed" }); const recovered = await getConversationWorkState({ conversationId: CONVERSATION_ID, state, }); expect(recovered?.lease).toBeUndefined(); - expect(recovered?.needsRun).toBe(true); + expect(recovered?.needsRun).toBe(false); expect(recovered?.messages).toEqual([]); + await expect( + getAgentTurnSessionRecord(CONVERSATION_ID, "turn-invalid-timeout"), + ).resolves.toMatchObject({ + state: "failed", + errorMessage: + "Awaiting turn continuation metadata could not be materialized", + }); + }); + + it("terminalizes stale idle continuations skipped by resume startup", async () => { + const queue = createFakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + const sessionId = "turn_1712345_0001"; + + await requestConversationWork({ + conversationId: CONVERSATION_ID, + nowMs: 1_000, + state, + }); + await upsertAgentTurnSessionRecord({ + conversationId: CONVERSATION_ID, + sessionId, + sliceId: 2, + state: "awaiting_resume", + resumeReason: "timeout", + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "original request" }], + timestamp: 1_000, + }, + ], + }); + await persistThreadStateById(CONVERSATION_ID, { + artifacts: { + listColumnMap: {}, + }, + conversation: { + schemaVersion: 1, + backfill: {}, + compactions: [], + piMessages: [], + messages: [ + { + id: "1712345.0001", + role: "user", + text: "original request", + createdAtMs: 1_000, + author: { + userId: "U123", + }, + }, + ], + processing: { + activeTurnId: "turn-newer", + }, + stats: { + compactedMessageCount: 0, + estimatedContextTokens: 0, + totalMessageCount: 1, + updatedAtMs: 1_000, + }, + vision: { + byFileId: {}, + }, + }, + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + throw new Error("injected messages should not replay"); + }, + handleSubscribedMessage: async () => { + throw new Error("injected messages should not replay"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "completed" }); + + const recovered = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(recovered?.lease).toBeUndefined(); + expect(recovered?.needsRun).toBe(false); + expect(recovered?.messages).toEqual([]); + await expect( + getAgentTurnSessionRecord(CONVERSATION_ID, sessionId), + ).resolves.toMatchObject({ + state: "failed", + errorMessage: "Awaiting turn continuation was stale before resuming", + }); }); it("keeps Slack mailbox records pending when the runtime handoff fails", async () => { From fe903f437fce986c6cb2f342c3ed48105172d588 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 2 Jun 2026 23:36:30 +0200 Subject: [PATCH 27/45] fix(runtime): Keep resumed follow-ups pending When a new Slack message arrives while a previous turn is awaiting resume, schedule the old continuation without marking the new message as replied. This keeps the follow-up available for steering or the next handled turn instead of silently treating it as answered. Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/runtime/reply-executor.ts | 8 -------- .../tests/integration/slack/bot-handlers.test.ts | 16 +++++++--------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 9795508d3..25cb7f69f 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -423,14 +423,6 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { throw error; } - markConversationMessage( - preparedState.conversation, - preparedState.userMessageId, - { - replied: true, - skippedReason: undefined, - }, - ); await persistThreadState(thread, { conversation: preparedState.conversation, }); diff --git a/packages/junior/tests/integration/slack/bot-handlers.test.ts b/packages/junior/tests/integration/slack/bot-handlers.test.ts index 1231103b2..edd21ff15 100644 --- a/packages/junior/tests/integration/slack/bot-handlers.test.ts +++ b/packages/junior/tests/integration/slack/bot-handlers.test.ts @@ -776,7 +776,7 @@ describe("bot handlers (integration)", () => { expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([]); }); - it("reschedules an awaiting turn continuation instead of starting a new turn", async () => { + it("reschedules an awaiting turn continuation without replying to the follow-up", async () => { const conversationId = "slack:C_TIMEOUT_RETRY:1700000000.000"; const activeSessionId = "turn_msg-original"; const scheduleTurnTimeoutResume = vi.fn().mockResolvedValue(undefined); @@ -838,14 +838,12 @@ describe("bot handlers (integration)", () => { } ).conversation; expect(conversation?.processing?.activeTurnId).toBe(activeSessionId); - expect( - conversation?.messages?.find((message) => message.id === "msg-retry"), - ).toMatchObject({ - meta: { - replied: true, - skippedReason: undefined, - }, - }); + const followUp = conversation?.messages?.find( + (message) => message.id === "msg-retry", + ); + expect(followUp).toBeDefined(); + expect(followUp?.meta?.replied).toBeUndefined(); + expect(followUp?.meta?.skippedReason).toBeUndefined(); }); it("reschedules an awaiting continuation for repeated delivery of the active message", async () => { From efbce200dd769625b861cb6af010e035ee444c68 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 01:49:15 +0200 Subject: [PATCH 28/45] fix(mcp): Recover missing resumed tool calls Return a model-visible MCP tool error when a resumed slice asks for a tool name that is no longer present in the rebuilt provider catalog. Include recovery guidance so the agent can refresh the provider catalog before retrying. Fixes GH-492 Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/mcp/errors.ts | 2 +- .../src/chat/tools/skill/call-mcp-tool.ts | 41 +++++++++++++++++-- .../tests/unit/tools/call-mcp-tool.test.ts | 36 ++++++++++++---- 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/packages/junior/src/chat/mcp/errors.ts b/packages/junior/src/chat/mcp/errors.ts index 75957ea88..50322b555 100644 --- a/packages/junior/src/chat/mcp/errors.ts +++ b/packages/junior/src/chat/mcp/errors.ts @@ -1,4 +1,4 @@ -/** Thrown when an MCP tool returns an error result. */ +/** Thrown when an MCP failure should be returned as a model-visible tool error. */ export class McpToolError extends Error { constructor(message: string) { super(message); diff --git a/packages/junior/src/chat/tools/skill/call-mcp-tool.ts b/packages/junior/src/chat/tools/skill/call-mcp-tool.ts index a21d9159b..ae6f42f10 100644 --- a/packages/junior/src/chat/tools/skill/call-mcp-tool.ts +++ b/packages/junior/src/chat/tools/skill/call-mcp-tool.ts @@ -1,4 +1,6 @@ import { Type } from "@sinclair/typebox"; +import { setSpanAttributes } from "@/chat/logging"; +import { McpToolError } from "@/chat/mcp/errors"; import type { ManagedMcpTool } from "@/chat/mcp/tool-manager"; import { parseMcpProviderFromToolName } from "@/chat/mcp/tool-name"; import { tool } from "@/chat/tools/definition"; @@ -34,6 +36,19 @@ function resolveMcpArguments( return {}; } +function activeProviderNames(tools: ManagedMcpTool[]): string[] { + return [...new Set(tools.map((toolDef) => toolDef.provider))].sort((a, b) => + a.localeCompare(b), + ); +} + +function missingToolMessage(toolName: string, provider: string | undefined) { + const retryHint = provider + ? `Call searchMcpTools with provider "${provider}" to refresh the catalog, then retry with an exact returned tool_name.` + : "Call searchMcpTools to refresh the catalog, then retry with an exact returned tool_name."; + return `MCP tool is not active for this turn: ${toolName}. ${retryHint}`; +} + /** Create the stable dispatcher for active MCP provider tools. */ export function createCallMcpToolTool(mcpToolManager: CallMcpToolManager) { return tool({ @@ -60,11 +75,29 @@ export function createCallMcpToolTool(mcpToolManager: CallMcpToolManager) { if (provider) { await mcpToolManager.activateProvider(provider); } - const mcpTool = mcpToolManager - .getResolvedActiveTools() - .find((candidate) => candidate.name === tool_name); + const activeTools = mcpToolManager.getResolvedActiveTools(); + const mcpTool = activeTools.find( + (candidate) => candidate.name === tool_name, + ); if (!mcpTool) { - throw new Error(`MCP tool is not active for this turn: ${tool_name}`); + const providerTools = provider + ? activeTools.filter((candidate) => candidate.provider === provider) + : []; + setSpanAttributes({ + "app.mcp.requested_tool_name": tool_name, + ...(provider ? { "app.mcp.requested_provider": provider } : {}), + "app.mcp.active_provider_names": activeProviderNames(activeTools), + "app.mcp.active_tool_count": activeTools.length, + ...(provider + ? { + "app.mcp.matching_provider_tool_count": providerTools.length, + "app.mcp.matching_provider_tool_names": providerTools + .map((candidate) => candidate.name) + .sort((a, b) => a.localeCompare(b)), + } + : {}), + }); + throw new McpToolError(missingToolMessage(tool_name, provider)); } return await mcpTool.execute( resolveMcpArguments(input as Record), diff --git a/packages/junior/tests/unit/tools/call-mcp-tool.test.ts b/packages/junior/tests/unit/tools/call-mcp-tool.test.ts index 6ecc0cbd5..e122410d7 100644 --- a/packages/junior/tests/unit/tools/call-mcp-tool.test.ts +++ b/packages/junior/tests/unit/tools/call-mcp-tool.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { McpToolError } from "@/chat/mcp/errors"; import { createCallMcpToolTool } from "@/chat/tools/skill/call-mcp-tool"; describe("callMcpTool", () => { @@ -142,20 +143,41 @@ describe("callMcpTool", () => { ).rejects.toThrow("callMcpTool arguments must be an object"); }); - it("rejects tools that are not active for the turn", async () => { + it("returns an expected MCP error when a resumed catalog is missing the requested tool", async () => { const manager = { activateProvider: vi.fn(async () => true), - getResolvedActiveTools: vi.fn(() => []), + getResolvedActiveTools: vi.fn(() => [ + { + name: "mcp__demo__other", + rawName: "other", + provider: "demo", + description: "Other", + parameters: {}, + execute: vi.fn(), + }, + ]), }; const callMcpTool = createCallMcpToolTool(manager); - await expect( - callMcpTool.execute!( + let error: unknown; + try { + await callMcpTool.execute!( { - tool_name: "mcp__demo__missing", + tool_name: "mcp__demo__missing_after_resume", }, {}, - ), - ).rejects.toThrow("MCP tool is not active for this turn"); + ); + } catch (caught: unknown) { + error = caught; + } + + expect(error).toBeInstanceOf(McpToolError); + if (!(error instanceof Error)) { + throw new Error("expected callMcpTool to throw an error"); + } + expect(error.message).toContain( + 'Call searchMcpTools with provider "demo" to refresh the catalog', + ); + expect(manager.activateProvider).toHaveBeenCalledWith("demo"); }); }); From 1c65e3604f8927291029e267a86032d0837baf95 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 01:49:30 +0200 Subject: [PATCH 29/45] fix(runtime): Keep unhanded Slack work pending Only mark Slack mailbox records injected after the runtime has durably persisted the turn handoff. Preserve pending mailbox work when a handler only reschedules an awaiting continuation, while still marking already-handled early replies after their thread state is saved. Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/runtime/reply-executor.ts | 2 + .../src/chat/task-execution/slack-work.ts | 15 ++--- .../slack-conversation-work.test.ts | 62 +++++++++++-------- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 25cb7f69f..38a6e4612 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -433,6 +433,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { await persistThreadState(thread, { conversation: preparedState.conversation, }); + await options.onTurnStatePersisted?.(); return; } const configReply = await maybeApplyProviderDefaultConfigRequest({ @@ -467,6 +468,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { await persistThreadState(thread, { conversation: preparedState.conversation, }); + await options.onTurnStatePersisted?.(); return; } startActiveTurn({ diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index e29d6814f..7fd62bd14 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -282,6 +282,7 @@ export function createSlackConversationWorker( (record) => record.inboundMessageId, ); let initialMessagesPersisted = false; + let leaseLostDuringTurnHandoff = false; const markInitialMessagesInjected = async (): Promise => { if (initialMessagesPersisted) { return true; @@ -297,6 +298,7 @@ export function createSlackConversationWorker( }; const onTurnStatePersisted = async (): Promise => { if (!(await markInitialMessagesInjected())) { + leaseLostDuringTurnHandoff = true; throw new Error( `Conversation work lease lost before Slack turn handoff for ${context.conversationId}`, ); @@ -340,6 +342,9 @@ export function createSlackConversationWorker( if (isCooperativeTurnYieldError(error)) { return { status: "yielded" } satisfies ConversationWorkerResult; } + if (leaseLostDuringTurnHandoff) { + return { status: "lost_lease" } satisfies ConversationWorkerResult; + } throw error; } }, @@ -348,16 +353,6 @@ export function createSlackConversationWorker( return turnResult; } - const messagesMarked = await markConversationMessagesInjected({ - conversationId: context.conversationId, - inboundMessageIds: records.map((record) => record.inboundMessageId), - leaseToken: context.leaseToken, - state, - }); - if (!messagesMarked) { - return { status: "lost_lease" }; - } - return { status: "completed" }; }; } diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index 3bd6101ca..16c98b36c 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -142,7 +142,8 @@ describe("Slack conversation work execution", () => { run: createSlackConversationWorker({ getSlackAdapter: () => slackAdapter, runtime: { - handleNewMention: async (thread, message) => { + handleNewMention: async (thread, message, hooks) => { + await hooks?.onTurnStatePersisted?.(); calls.push({ thread, message }); }, handleSubscribedMessage: async () => { @@ -210,6 +211,7 @@ describe("Slack conversation work execution", () => { getSlackAdapter: () => slackAdapter, runtime: { handleNewMention: async (thread, message, hooks) => { + await hooks?.onTurnStatePersisted?.(); calls.push({ thread, message, @@ -298,6 +300,7 @@ describe("Slack conversation work execution", () => { getSlackAdapter: () => slackAdapter, runtime: { handleNewMention: async (thread, message, hooks) => { + await hooks?.onTurnStatePersisted?.(); subscribedValues.push(await thread.isSubscribed()); calls.push({ thread, @@ -368,7 +371,8 @@ describe("Slack conversation work execution", () => { run: createSlackConversationWorker({ getSlackAdapter: () => slackAdapter, runtime: { - handleNewMention: async (_thread, message) => { + handleNewMention: async (_thread, message, hooks) => { + await hooks?.onTurnStatePersisted?.(); calls.push(message.text); }, handleSubscribedMessage: async () => { @@ -730,7 +734,7 @@ describe("Slack conversation work execution", () => { expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); }); - it("reports lost lease when Slack injection marking loses ownership", async () => { + it("requeues Slack mailbox records when the runtime returns without durable handoff", async () => { const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); @@ -739,7 +743,7 @@ describe("Slack conversation work execution", () => { await handleSlackWebhookAndFlush({ request: slackWebhookRequest( slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> first`, + text: `<@${SLACK_BOT_USER_ID}> follow-up during resume`, }), ), services: { @@ -749,37 +753,44 @@ describe("Slack conversation work execution", () => { state, }, }); + queue.sent = []; let handled = 0; - const worker = createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async () => { - handled += 1; - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }); - await expect( - worker({ - checkIn: async () => true, - conversationId: CONVERSATION_ID, - drainMailbox: async () => [], - leaseToken: "stale-lease", - shouldYield: () => false, + processConversationWork(CONVERSATION_ID, { + nowMs: () => 3_000, + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async () => { + handled += 1; + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), }), - ).resolves.toEqual({ status: "lost_lease" }); + ).resolves.toEqual({ status: "pending_requeued" }); expect(handled).toBe(1); + expect(queue.sent).toEqual([ + expect.objectContaining({ + conversationId: CONVERSATION_ID, + idempotencyKey: `pending:${CONVERSATION_ID}:3000`, + }), + ]); const work = await getConversationWorkState({ conversationId: CONVERSATION_ID, state, }); + expect(work?.lease).toBeUndefined(); + expect(work?.needsRun).toBe(true); expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); }); it("completes Slack mailbox work when the handler finishes after the soft deadline", async () => { @@ -812,8 +823,9 @@ describe("Slack conversation work execution", () => { run: createSlackConversationWorker({ getSlackAdapter: () => slackAdapter, runtime: { - handleNewMention: async () => { + handleNewMention: async (_thread, _message, hooks) => { currentNowMs = 242_000; + await hooks?.onTurnStatePersisted?.(); }, handleSubscribedMessage: async () => { throw new Error("unexpected subscribed route"); From 4172cbef19fee0f747a4aa9474713b6756f9c0aa Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 03:42:30 +0200 Subject: [PATCH 30/45] fix(runtime): Resume turns before pending Slack mail Prefer awaiting turn continuation recovery before routing pending Slack mailbox records. When a continuation starts, leave pending mail runnable so the queue re-drives it after the active turn finishes instead of looping through a no-handoff Slack handler path. Co-Authored-By: GPT-5 Codex --- .../src/chat/task-execution/slack-work.ts | 13 ++- .../slack-conversation-work.test.ts | 83 +++++++++++++++---- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 7fd62bd14..72ba0b674 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -49,6 +49,7 @@ export interface SlackConversationMessageMetadata { export interface CreateSlackConversationWorkerOptions { getSlackAdapter: () => SlackAdapter; + resumeAwaitingContinuation?: (conversationId: string) => Promise; runtime: Pick< SlackTurnRuntime, "handleNewMention" | "handleSubscribedMessage" @@ -158,7 +159,7 @@ async function failUnresumableContinuation(args: { async function resumeAwaitingContinuation( conversationId: string, -): Promise { +): Promise { const summaries = await listAgentTurnSessionSummariesForConversation(conversationId); @@ -182,7 +183,7 @@ async function resumeAwaitingContinuation( } if (await resumeTimedOutTurnWithLockRetry(request)) { - return; + return true; } await failUnresumableContinuation({ @@ -192,6 +193,8 @@ async function resumeAwaitingContinuation( errorMessage: "Awaiting turn continuation was stale before resuming", }); } + + return false; } function getInstallation( @@ -226,6 +229,12 @@ export function createSlackConversationWorker( const state = getConnectedState(options.state); await state.connect(); + const resumeContinuation = + options.resumeAwaitingContinuation ?? resumeAwaitingContinuation; + if (await resumeContinuation(context.conversationId)) { + return { status: "completed" }; + } + const records = getPendingRecords( await getConversationWorkState({ conversationId: context.conversationId, diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index 16c98b36c..6d0afe0a2 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { Message, Thread } from "chat"; import { CooperativeTurnYieldError } from "@/chat/runtime/turn"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; @@ -327,25 +327,12 @@ describe("Slack conversation work execution", () => { expect(subscribedValues).toEqual([false]); }); - it("processes pending Slack follow-ups before timeout continuation", async () => { + it("processes pending Slack follow-ups when no continuation starts", async () => { const queue = createFakeQueue(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); - await upsertAgentTurnSessionRecord({ - conversationId: CONVERSATION_ID, - sessionId: "turn-timeout", - sliceId: 2, - state: "awaiting_resume", - resumeReason: "timeout", - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "original request" }], - timestamp: 1_000, - }, - ], - }); + const resumeAwaitingContinuation = vi.fn(async () => false); await handleSlackWebhookAndFlush({ request: slackWebhookRequest( @@ -370,6 +357,7 @@ describe("Slack conversation work execution", () => { state, run: createSlackConversationWorker({ getSlackAdapter: () => slackAdapter, + resumeAwaitingContinuation, runtime: { handleNewMention: async (_thread, message, hooks) => { await hooks?.onTurnStatePersisted?.(); @@ -384,6 +372,7 @@ describe("Slack conversation work execution", () => { }), ).resolves.toEqual({ status: "completed" }); + expect(resumeAwaitingContinuation).toHaveBeenCalledWith(CONVERSATION_ID); expect(calls).toEqual([expect.stringContaining("follow-up")]); const work = await getConversationWorkState({ conversationId: CONVERSATION_ID, @@ -392,6 +381,68 @@ describe("Slack conversation work execution", () => { expect(work ? countPendingConversationMessages(work) : 0).toBe(0); }); + it("resumes awaiting continuations before routing pending Slack follow-ups", async () => { + const queue = createFakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + const resumeAwaitingContinuation = vi.fn(async () => true); + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> follow-up`, + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + queue.sent = []; + + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => 3_500, + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + resumeAwaitingContinuation, + runtime: { + handleNewMention: async () => { + throw new Error("pending follow-up should wait for resume"); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "pending_requeued" }); + + expect(resumeAwaitingContinuation).toHaveBeenCalledWith(CONVERSATION_ID); + expect(queue.sent).toEqual([ + expect.objectContaining({ + conversationId: CONVERSATION_ID, + idempotencyKey: `pending:${CONVERSATION_ID}:3500`, + }), + ]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.lease).toBeUndefined(); + expect(work?.needsRun).toBe(true); + expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); + }); + it("drains Slack messages that arrive during an active turn into steering", async () => { const queue = createFakeQueue(); const state = getStateAdapter(); From 95e7a71a198a7bcd6d3956f0ac2ea8886ca9ea81 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 03:42:49 +0200 Subject: [PATCH 31/45] fix(queue): Authenticate conversation callbacks Sign Vercel conversation work queue payloads with JUNIOR_SECRET and verify the decoded callback payload before processing work. This keeps the public queue route from executing forged conversation work even outside Vercel trigger enforcement. Co-Authored-By: GPT-5 Codex --- .../src/chat/task-execution/queue-signing.ts | 110 ++++++++++++++++++ .../chat/task-execution/vercel-callback.ts | 7 +- .../src/chat/task-execution/vercel-queue.ts | 15 ++- .../task-execution/conversation-work.test.ts | 43 ++++++- 4 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 packages/junior/src/chat/task-execution/queue-signing.ts diff --git a/packages/junior/src/chat/task-execution/queue-signing.ts b/packages/junior/src/chat/task-execution/queue-signing.ts new file mode 100644 index 000000000..bd367bd97 --- /dev/null +++ b/packages/junior/src/chat/task-execution/queue-signing.ts @@ -0,0 +1,110 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import type { ConversationQueueMessage } from "./queue"; + +const CONVERSATION_WORK_QUEUE_SIGNATURE_CONTEXT = + "junior.conversation_work_queue.v1"; +const CONVERSATION_WORK_QUEUE_SIGNATURE_VERSION = "v1"; + +interface SignedConversationQueueMessage extends ConversationQueueMessage { + signature: string; + signatureVersion: typeof CONVERSATION_WORK_QUEUE_SIGNATURE_VERSION; + signedAtMs: number; +} + +function getConversationWorkQueueSecret(): string | undefined { + return process.env.JUNIOR_SECRET?.trim() || undefined; +} + +function buildSignedPayload( + message: ConversationQueueMessage, + signedAtMs: number, +) { + return [ + CONVERSATION_WORK_QUEUE_SIGNATURE_CONTEXT, + signedAtMs, + message.conversationId, + ].join(":"); +} + +function signPayload( + message: ConversationQueueMessage, + signedAtMs: number, + secret: string, +): string { + return createHmac("sha256", secret) + .update(buildSignedPayload(message, signedAtMs)) + .digest("hex"); +} + +function timingSafeMatch(expected: string, actual: string): boolean { + const expectedBuffer = Buffer.from(expected); + const actualBuffer = Buffer.from(actual); + if (expectedBuffer.length !== actualBuffer.length) { + return false; + } + return timingSafeEqual(expectedBuffer, actualBuffer); +} + +function parseSignedConversationQueueMessage( + value: unknown, +): SignedConversationQueueMessage | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const record = value as Record; + if ( + typeof record.conversationId !== "string" || + !record.conversationId.trim() || + record.signatureVersion !== CONVERSATION_WORK_QUEUE_SIGNATURE_VERSION || + typeof record.signedAtMs !== "number" || + !Number.isFinite(record.signedAtMs) || + typeof record.signature !== "string" || + !record.signature.trim() + ) { + return undefined; + } + + return { + conversationId: record.conversationId, + signature: record.signature, + signatureVersion: CONVERSATION_WORK_QUEUE_SIGNATURE_VERSION, + signedAtMs: record.signedAtMs, + }; +} + +/** Sign a conversation queue payload before it crosses the public callback route. */ +export function signConversationQueueMessage( + message: ConversationQueueMessage, + nowMs = Date.now(), +): SignedConversationQueueMessage { + const secret = getConversationWorkQueueSecret(); + if (!secret) { + throw new Error( + "Cannot sign conversation queue message without JUNIOR_SECRET", + ); + } + return { + ...message, + signedAtMs: nowMs, + signatureVersion: CONVERSATION_WORK_QUEUE_SIGNATURE_VERSION, + signature: signPayload(message, nowMs, secret), + }; +} + +/** Verify a signed conversation queue payload from the Vercel Queue callback. */ +export function verifySignedConversationQueueMessage( + value: unknown, +): ConversationQueueMessage | undefined { + const message = parseSignedConversationQueueMessage(value); + const secret = getConversationWorkQueueSecret(); + if (!message || !secret) { + return undefined; + } + + const expected = signPayload(message, message.signedAtMs, secret); + if (!timingSafeMatch(expected, message.signature)) { + return undefined; + } + + return { conversationId: message.conversationId }; +} diff --git a/packages/junior/src/chat/task-execution/vercel-callback.ts b/packages/junior/src/chat/task-execution/vercel-callback.ts index 44d2c8dbe..77c39c4fc 100644 --- a/packages/junior/src/chat/task-execution/vercel-callback.ts +++ b/packages/junior/src/chat/task-execution/vercel-callback.ts @@ -10,6 +10,7 @@ import { type ConversationWorkerResult, type ConversationWorkerContext, } from "./worker"; +import { verifySignedConversationQueueMessage } from "./queue-signing"; export const CONVERSATION_WORK_VISIBILITY_TIMEOUT_BUFFER_SECONDS = 30; @@ -75,8 +76,12 @@ export function createVercelConversationWorkCallback( ): (request: Request) => Promise { return handleCallback( async (message: unknown) => { + const verified = verifySignedConversationQueueMessage(message); + if (!verified) { + throw new Error("Unauthorized conversation queue message"); + } await runWithTurnRequestDeadline(() => - processConversationQueueMessage(message, options), + processConversationQueueMessage(verified, options), ); }, { diff --git a/packages/junior/src/chat/task-execution/vercel-queue.ts b/packages/junior/src/chat/task-execution/vercel-queue.ts index 874385863..8274efde2 100644 --- a/packages/junior/src/chat/task-execution/vercel-queue.ts +++ b/packages/junior/src/chat/task-execution/vercel-queue.ts @@ -6,6 +6,7 @@ import type { ConversationQueueSendResult, ConversationWorkQueue, } from "./queue"; +import { signConversationQueueMessage } from "./queue-signing"; export const DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC = "junior_conversation_work"; @@ -54,11 +55,15 @@ export function createVercelConversationWorkQueue( message: ConversationQueueMessage, sendOptions?: ConversationQueueSendOptions, ): Promise { - const result = await client.send(topic, message, { - idempotencyKey: sendOptions?.idempotencyKey, - delaySeconds: toDelaySeconds(sendOptions), - retentionSeconds: options.retentionSeconds, - }); + const result = await client.send( + topic, + signConversationQueueMessage(message), + { + idempotencyKey: sendOptions?.idempotencyKey, + delaySeconds: toDelaySeconds(sendOptions), + retentionSeconds: options.retentionSeconds, + }, + ); return result.messageId ? { messageId: result.messageId } : {}; }, }; diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index 1d0cb94da..a0dae45ce 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -21,6 +21,10 @@ import { } from "@/chat/task-execution/worker"; import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import { createVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; +import { + signConversationQueueMessage, + verifySignedConversationQueueMessage, +} from "@/chat/task-execution/queue-signing"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { CONVERSATION_ID, @@ -32,12 +36,19 @@ import { } from "../../fixtures/conversation-work"; describe("conversation work execution", () => { + const originalJuniorSecret = process.env.JUNIOR_SECRET; + beforeEach(async () => { await disconnectStateAdapter(); }); afterEach(async () => { await disconnectStateAdapter(); + if (originalJuniorSecret === undefined) { + delete process.env.JUNIOR_SECRET; + } else { + process.env.JUNIOR_SECRET = originalJuniorSecret; + } vi.useRealTimers(); }); @@ -587,6 +598,7 @@ describe("conversation work execution", () => { }); it("maps the generic queue port to Vercel Queue send options", async () => { + process.env.JUNIOR_SECRET = "conversation-work-secret"; const sends: Array<{ message: unknown; options: unknown; @@ -612,7 +624,12 @@ describe("conversation work execution", () => { expect(sends).toEqual([ { topic: "junior_test_work", - message: { conversationId: CONVERSATION_ID }, + message: expect.objectContaining({ + conversationId: CONVERSATION_ID, + signature: expect.any(String), + signatureVersion: "v1", + signedAtMs: expect.any(Number), + }), options: { delaySeconds: 16, idempotencyKey: "idem-1", @@ -622,6 +639,30 @@ describe("conversation work execution", () => { ]); }); + it("verifies signed Vercel Queue callback payloads", () => { + process.env.JUNIOR_SECRET = "conversation-work-secret"; + const signed = signConversationQueueMessage( + { conversationId: CONVERSATION_ID }, + 12_345, + ); + + expect(verifySignedConversationQueueMessage(signed)).toEqual({ + conversationId: CONVERSATION_ID, + }); + expect( + verifySignedConversationQueueMessage({ + ...signed, + conversationId: "slack:C123:forged", + }), + ).toBeUndefined(); + expect( + verifySignedConversationQueueMessage({ + ...signed, + signature: "deadbeef", + }), + ).toBeUndefined(); + }); + it("processes Vercel Queue payloads through the leased worker", async () => { const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); From cbdf43b308882006af10012897e4ce2824be9932 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 04:54:58 +0200 Subject: [PATCH 32/45] fix(runtime): Preserve continuation failure status Keep failed continuation persistence from becoming a generated assistant error reply, while preserving the terminal timeout slice-cap record as failed state. Propagate Slack handoff lost-lease results through the conversation worker so downstream completion does not treat the run as successful. Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond.ts | 47 ++++++++------ .../src/chat/services/turn-session-record.ts | 5 +- .../src/chat/task-execution/slack-work.ts | 5 +- .../slack-conversation-work.test.ts | 64 +++++++++++++++++++ .../runtime/respond-provider-retry.test.ts | 42 ++++++++++-- .../unit/services/turn-session-record.test.ts | 8 ++- 6 files changed, 144 insertions(+), 27 deletions(-) diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index c23d0a571..1f220cd65 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -1506,9 +1506,12 @@ export async function generateAssistantReply( logContext: sessionRecordLogContext, requester, }); - if (sessionRecord) { - throw error; + if (!sessionRecord) { + throw new Error( + `Failed to persist cooperative yield continuation for conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId}`, + ); } + throw error; } if (timedOut && timeoutResumeConversationId && timeoutResumeSessionId) { @@ -1529,7 +1532,12 @@ export async function generateAssistantReply( logContext: sessionRecordLogContext, requester, }); - if (sessionRecord) { + if (!sessionRecord) { + throw new Error( + `Failed to persist timeout continuation for conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId}`, + ); + } + if (sessionRecord.state === "awaiting_resume") { throw new RetryableTurnError( "turn_timeout_resume", `conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId} slice=${sessionRecord.sliceId} version=${sessionRecord.version}`, @@ -1569,23 +1577,26 @@ export async function generateAssistantReply( logContext: sessionRecordLogContext, requester, }); - if (sessionRecord) { - throw new RetryableTurnError( - error.kind === "plugin" ? "plugin_auth_resume" : "mcp_auth_resume", - `conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId} slice=${sessionRecord.sliceId}`, - { - authDisposition: error.disposition, - authDurationMs: Date.now() - replyStartedAtMs, - authKind: error.kind, - authProvider: error.provider, - authThinkingLevel: thinkingSelection?.thinkingLevel, - authUsage: turnUsage, - conversationId: timeoutResumeConversationId, - sessionId: timeoutResumeSessionId, - sliceId: sessionRecord.sliceId, - }, + if (!sessionRecord) { + throw new Error( + `Failed to persist auth continuation for conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId}`, ); } + throw new RetryableTurnError( + error.kind === "plugin" ? "plugin_auth_resume" : "mcp_auth_resume", + `conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId} slice=${sessionRecord.sliceId}`, + { + authDisposition: error.disposition, + authDurationMs: Date.now() - replyStartedAtMs, + authKind: error.kind, + authProvider: error.provider, + authThinkingLevel: thinkingSelection?.thinkingLevel, + authUsage: turnUsage, + conversationId: timeoutResumeConversationId, + sessionId: timeoutResumeSessionId, + sliceId: sessionRecord.sliceId, + }, + ); } if (isRetryableTurnError(error)) { diff --git a/packages/junior/src/chat/services/turn-session-record.ts b/packages/junior/src/chat/services/turn-session-record.ts index 7c87ccb89..1b35e1bde 100644 --- a/packages/junior/src/chat/services/turn-session-record.ts +++ b/packages/junior/src/chat/services/turn-session-record.ts @@ -313,7 +313,7 @@ export async function persistAuthPauseSessionRecord(args: { /** * Persist a timeout session record at the last safe boundary. Returns the durable - * record when persistence succeeds so callers can enqueue a continuation. + * record so callers can distinguish scheduled continuations from terminal caps. */ export async function persistTimeoutSessionRecord(args: { channelName?: string; @@ -351,7 +351,7 @@ export async function persistTimeoutSessionRecord(args: { args.currentUsage, ); if (nextSliceId > AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES) { - await upsertAgentTurnSessionRecord({ + return await upsertAgentTurnSessionRecord({ ...((args.channelName ?? latestSessionRecord?.channelName) ? { channelName: args.channelName ?? latestSessionRecord?.channelName, @@ -377,7 +377,6 @@ export async function persistTimeoutSessionRecord(args: { ? { traceId: getActiveTraceId() ?? latestSessionRecord?.traceId } : {}), }); - return undefined; } return await upsertAgentTurnSessionRecord({ ...((args.channelName ?? latestSessionRecord?.channelName) diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 72ba0b674..7c2b249b2 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -358,7 +358,10 @@ export function createSlackConversationWorker( } }, }); - if (turnResult?.status === "yielded") { + if ( + turnResult?.status === "yielded" || + turnResult?.status === "lost_lease" + ) { return turnResult; } diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index 6d0afe0a2..06e17d861 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -844,6 +844,70 @@ describe("Slack conversation work execution", () => { expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); }); + it("reports lost lease when Slack turn handoff loses the mailbox lease", async () => { + const queue = createFakeQueue(); + const state = getStateAdapter(); + await state.connect(); + const slackAdapter = createSlackAdapterFixture(); + let currentNowMs = 1_000; + + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> follow-up during lease loss`, + }), + ), + services: { + getSlackAdapter: () => slackAdapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + state, + }, + }); + queue.sent = []; + + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, + queue, + state, + run: createSlackConversationWorker({ + getSlackAdapter: () => slackAdapter, + runtime: { + handleNewMention: async (_thread, _message, hooks) => { + currentNowMs = 1_000 + CONVERSATION_WORK_LEASE_TTL_MS + 1; + await recoverConversationWork({ + nowMs: currentNowMs, + queue, + state, + }); + await hooks?.onTurnStatePersisted?.(); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, + }), + }), + ).resolves.toEqual({ status: "lost_lease" }); + + expect(queue.sent).toEqual([ + expect.objectContaining({ + conversationId: CONVERSATION_ID, + idempotencyKey: `heartbeat:lease:${CONVERSATION_ID}:${currentNowMs}`, + }), + ]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state, + }); + expect(work?.lease).toBeUndefined(); + expect(work?.needsRun).toBe(true); + expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); + }); + it("completes Slack mailbox work when the handler finishes after the soft deadline", async () => { const queue = createFakeQueue(); const state = getStateAdapter(); diff --git a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts index bf0937a11..633d26094 100644 --- a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts +++ b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts @@ -210,7 +210,7 @@ import { generateAssistantReply } from "@/chat/respond"; import { isCooperativeTurnYieldError } from "@/chat/runtime/turn"; import { getAwaitingTurnContinuationRequest } from "@/chat/services/timeout-resume"; import { disconnectStateAdapter } from "@/chat/state/adapter"; -import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import * as turnSessionState from "@/chat/state/turn-session"; describe("generateAssistantReply provider retry", () => { beforeEach(async () => { @@ -252,7 +252,7 @@ describe("generateAssistantReply provider retry", () => { expect(counters.promptCalls).toBe(1); expect(counters.continueCalls).toBe(1); - const sessionRecord = await getAgentTurnSessionRecord( + const sessionRecord = await turnSessionState.getAgentTurnSessionRecord( "conversation-1", "turn-1", ); @@ -289,7 +289,7 @@ describe("generateAssistantReply provider retry", () => { expect(reply.text).toBe("Steered."); expect(injectedTexts).toEqual(["actually do the other thing"]); - const sessionRecord = await getAgentTurnSessionRecord( + const sessionRecord = await turnSessionState.getAgentTurnSessionRecord( "conversation-steering", "turn-steering", ); @@ -316,7 +316,7 @@ describe("generateAssistantReply provider retry", () => { ); expect(isCooperativeTurnYieldError(error)).toBe(true); - const sessionRecord = await getAgentTurnSessionRecord( + const sessionRecord = await turnSessionState.getAgentTurnSessionRecord( "conversation-yield", "turn-yield", ); @@ -340,6 +340,40 @@ describe("generateAssistantReply provider retry", () => { }); }); + it("throws when a cooperative yield cannot persist its resumable boundary", async () => { + agentMode.value = "cooperativeYield"; + const upsertSpy = vi + .spyOn(turnSessionState, "upsertAgentTurnSessionRecord") + .mockRejectedValue(new Error("storage unavailable")); + + const error = await generateAssistantReply("help me", { + requester: { userId: "U123" }, + correlation: { + conversationId: "conversation-yield-persist-failure", + turnId: "turn-yield-persist-failure", + channelId: "C123", + threadTs: "1712345.0004", + }, + shouldYield: () => true, + }).then( + () => undefined, + (caught: unknown) => caught, + ); + upsertSpy.mockRestore(); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain( + "Failed to persist cooperative yield continuation", + ); + expect(isCooperativeTurnYieldError(error)).toBe(false); + await expect( + turnSessionState.getAgentTurnSessionRecord( + "conversation-yield-persist-failure", + "turn-yield-persist-failure", + ), + ).resolves.toBeUndefined(); + }); + it("rejects steering injection when Pi steer fails", async () => { agentMode.value = "steeringSteerThrows"; let injectRejected = false; diff --git a/packages/junior/tests/unit/services/turn-session-record.test.ts b/packages/junior/tests/unit/services/turn-session-record.test.ts index cc5f28651..5a7647541 100644 --- a/packages/junior/tests/unit/services/turn-session-record.test.ts +++ b/packages/junior/tests/unit/services/turn-session-record.test.ts @@ -243,7 +243,13 @@ describe("persistAuthPauseSessionRecord", () => { modelId: "test-model", }, }), - ).resolves.toBeUndefined(); + ).resolves.toMatchObject({ + state: "failed", + sliceId: AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES, + cumulativeDurationMs: 15_000, + errorMessage: expect.stringContaining("slice limit"), + piMessages, + }); await expect( getAgentTurnSessionRecord("conversation-timeout-cap", "turn-timeout-cap"), From 981f4172211362a1a4db9160e05183f15d7f620c Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 06:10:38 +0200 Subject: [PATCH 33/45] fix(runtime): Ignore replied duplicates before resume Handle already-replied Slack deliveries before rescheduling an active continuation so duplicate events complete mailbox handoff without nudging the old turn again. Keep auth-pause persistence failure behavior aligned with the existing provider-error contract while preserving the continuation failure fixes for yield and timeout paths. Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond.ts | 33 ++++++------ .../junior/src/chat/runtime/reply-executor.ts | 14 ++--- .../integration/slack/bot-handlers.test.ts | 52 +++++++++++++++++++ 3 files changed, 74 insertions(+), 25 deletions(-) diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index 1f220cd65..a0c070f52 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -1577,26 +1577,23 @@ export async function generateAssistantReply( logContext: sessionRecordLogContext, requester, }); - if (!sessionRecord) { - throw new Error( - `Failed to persist auth continuation for conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId}`, + if (sessionRecord) { + throw new RetryableTurnError( + error.kind === "plugin" ? "plugin_auth_resume" : "mcp_auth_resume", + `conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId} slice=${sessionRecord.sliceId}`, + { + authDisposition: error.disposition, + authDurationMs: Date.now() - replyStartedAtMs, + authKind: error.kind, + authProvider: error.provider, + authThinkingLevel: thinkingSelection?.thinkingLevel, + authUsage: turnUsage, + conversationId: timeoutResumeConversationId, + sessionId: timeoutResumeSessionId, + sliceId: sessionRecord.sliceId, + }, ); } - throw new RetryableTurnError( - error.kind === "plugin" ? "plugin_auth_resume" : "mcp_auth_resume", - `conversation=${timeoutResumeConversationId} session=${timeoutResumeSessionId} slice=${sessionRecord.sliceId}`, - { - authDisposition: error.disposition, - authDurationMs: Date.now() - replyStartedAtMs, - authKind: error.kind, - authProvider: error.provider, - authThinkingLevel: thinkingSelection?.thinkingLevel, - authUsage: turnUsage, - conversationId: timeoutResumeConversationId, - sessionId: timeoutResumeSessionId, - sliceId: sessionRecord.sliceId, - }, - ); } if (isRetryableTurnError(error)) { diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 38a6e4612..7198d7dbb 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -398,6 +398,13 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { } }; const activeTurnId = preparedState.conversation.processing.activeTurnId; + if (preparedState.userMessageAlreadyReplied) { + await persistThreadState(thread, { + conversation: preparedState.conversation, + }); + await options.onTurnStatePersisted?.(); + return; + } if (conversationId && activeTurnId) { const resumeRequest = await deps.services.getAwaitingTurnContinuationRequest({ @@ -429,13 +436,6 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { return; } } - if (preparedState.userMessageAlreadyReplied) { - await persistThreadState(thread, { - conversation: preparedState.conversation, - }); - await options.onTurnStatePersisted?.(); - return; - } const configReply = await maybeApplyProviderDefaultConfigRequest({ channelConfiguration: preparedState.channelConfiguration, requesterId: message.author.userId, diff --git a/packages/junior/tests/integration/slack/bot-handlers.test.ts b/packages/junior/tests/integration/slack/bot-handlers.test.ts index edd21ff15..68be1d6b0 100644 --- a/packages/junior/tests/integration/slack/bot-handlers.test.ts +++ b/packages/junior/tests/integration/slack/bot-handlers.test.ts @@ -53,6 +53,7 @@ function createRuntime( function createAwaitingContinuationState(args: { activeSessionId: string; + replied?: boolean; userMessageId?: string; userText?: string; }) { @@ -74,6 +75,9 @@ function createAwaitingContinuationState(args: { author: { userId: "U-test", }, + ...(args.replied === undefined + ? {} + : { meta: { replied: args.replied } }), }, ], processing: { @@ -892,6 +896,54 @@ describe("bot handlers (integration)", () => { expect(generateAssistantReply).not.toHaveBeenCalled(); }); + it("does not reschedule an awaiting continuation for an already-replied duplicate", async () => { + const conversationId = "slack:C_TIMEOUT_REPLIED_DUP:1700000000.000"; + const activeSessionId = "turn_msg-replied-duplicate"; + const scheduleTurnTimeoutResume = vi.fn().mockResolvedValue(undefined); + const getAwaitingTurnContinuationRequest = vi.fn().mockResolvedValue({ + conversationId, + sessionId: activeSessionId, + expectedVersion: 4, + }); + const generateAssistantReply = vi.fn(); + const onTurnStatePersisted = vi.fn(); + const { slackRuntime } = createRuntime({ + services: { + replyExecutor: { + generateAssistantReply, + getAwaitingTurnContinuationRequest, + scheduleTurnTimeoutResume, + }, + }, + }); + + const thread = createTestThread({ + id: conversationId, + state: createAwaitingContinuationState({ + activeSessionId, + replied: true, + userMessageId: "msg-replied-duplicate", + }), + }); + + await slackRuntime.handleNewMention( + thread, + createTestMessage({ + id: "msg-replied-duplicate", + threadId: conversationId, + text: "please keep working", + isMention: true, + }), + { onTurnStatePersisted }, + ); + + expect(getAwaitingTurnContinuationRequest).not.toHaveBeenCalled(); + expect(scheduleTurnTimeoutResume).not.toHaveBeenCalled(); + expect(generateAssistantReply).not.toHaveBeenCalled(); + expect(onTurnStatePersisted).toHaveBeenCalledOnce(); + expect(thread.posts).toEqual([]); + }); + it("keeps awaiting continuation state without a visible acknowledgement", async () => { const conversationId = "slack:C_TIMEOUT_NOTICE_FAIL:1700000000.000"; const activeSessionId = "turn_msg-original"; From 715119d1f1bd43d65eb027f641fa30ee3999d367 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 06:31:56 +0200 Subject: [PATCH 34/45] fix(queue): Nudge failed conversation work Worker errors already marked the conversation runnable before releasing its lease, but a recent enqueue marker could make heartbeat defer recovery. Send a fresh wake-up nudge for failed runner slices so runnable work is redelivered promptly. Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/task-execution/worker.ts | 29 ++++++++++++++-- .../task-execution/conversation-work.test.ts | 33 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index d8746e7d9..3537474b1 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -286,17 +286,40 @@ export async function processConversationWork( ); return { status: "completed" }; } catch (error) { + const errorNowMs = now(options); try { - await requestConversationContinuation({ + const continuationMarked = await requestConversationContinuation({ conversationId, leaseToken: lease.leaseToken, - nowMs: now(options), + nowMs: errorNowMs, state: options.state, }); + if (continuationMarked) { + await sendWakeNudge({ + conversationId, + idempotencyKey: nudgeIdempotencyKey( + "error", + conversationId, + errorNowMs, + ), + nowMs: errorNowMs, + options, + }); + } + } catch (requeueError) { + logException( + requeueError, + "conversation_work_requeue_failed", + { conversationId }, + {}, + "Conversation work requeue failed after runner error", + ); + } + try { await releaseConversationWork({ conversationId, leaseToken: lease.leaseToken, - nowMs: now(options), + nowMs: errorNowMs, state: options.state, }); } catch (releaseError) { diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index a0dae45ce..86332d4e1 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -260,6 +260,39 @@ describe("conversation work execution", () => { ]); }); + it("nudges failed worker runs before releasing runnable work", async () => { + const queue = createFakeQueue(); + let currentNowMs = 1_000; + await requestConversationWork({ + conversationId: CONVERSATION_ID, + nowMs: currentNowMs, + }); + + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, + queue, + run: async () => { + currentNowMs = 2_000; + throw new Error("runner failed"); + }, + }), + ).rejects.toThrow("runner failed"); + + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(state?.lease).toBeUndefined(); + expect(state?.needsRun).toBe(true); + expect(state?.lastEnqueuedAtMs).toBe(2_000); + expect(queue.sent).toMatchObject([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `error:${CONVERSATION_ID}:2000`, + }, + ]); + }); + it("drains pending messages and completes the leased conversation", async () => { const queue = createFakeQueue(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); From 7ee7ca716efa4dc5e17a86550aaf574824496b81 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 06:38:59 +0200 Subject: [PATCH 35/45] fix(runtime): Preserve steered yield snapshots Cooperative yields could snapshot Pi state before queued steering messages appeared in agent.state.messages. Keep the latest safe boundary candidate available so yielded, timed-out, and auth-paused resumes do not overwrite a longer steering-aware transcript. Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond.ts | 16 +++++-- .../runtime/respond-provider-retry.test.ts | 43 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index a0c070f52..095c24aa1 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -825,6 +825,13 @@ export async function generateAssistantReply( const toolCalls: string[] = []; let advisorTools: AgentTool[] = []; let agent: Agent | undefined; + let latestSafeBoundaryMessages: PiMessage[] = []; + const getResumeSnapshot = (): PiMessage[] => { + const currentMessages = agent ? [...agent.state.messages] : []; + return latestSafeBoundaryMessages.length > currentMessages.length + ? [...latestSafeBoundaryMessages] + : currentMessages; + }; // ── MCP auth orchestration ─────────────────────────────────────── const mcpAuth = createMcpAuthOrchestration( @@ -1119,6 +1126,7 @@ export async function generateAssistantReply( logContext: sessionRecordLogContext, requester, }); + latestSafeBoundaryMessages = [...messages]; }; const drainSteeringMessages = async (): Promise => { if ( @@ -1171,7 +1179,7 @@ export async function generateAssistantReply( } yielded = true; - timeoutResumeMessages = [...agent!.state.messages]; + timeoutResumeMessages = getResumeSnapshot(); throw new CooperativeTurnYieldError( `Agent turn yielded at a safe boundary after ${ Date.now() - replyStartedAtMs @@ -1334,10 +1342,10 @@ export async function generateAssistantReply( "Timed-out agent run did not settle after abort before resume snapshot", ); } - timeoutResumeMessages = [...agent.state.messages]; + timeoutResumeMessages = getResumeSnapshot(); } if (getPendingAuthPause()) { - timeoutResumeMessages = [...agent.state.messages]; + timeoutResumeMessages = getResumeSnapshot(); throw getPendingAuthPause()!; } throw error; @@ -1382,7 +1390,7 @@ export async function generateAssistantReply( ...extractGenAiUsageAttributes(usageSummary), }); if (getPendingAuthPause()) { - timeoutResumeMessages = [...agent.state.messages]; + timeoutResumeMessages = getResumeSnapshot(); throw getPendingAuthPause()!; } diff --git a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts index 633d26094..327234471 100644 --- a/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts +++ b/packages/junior/tests/unit/runtime/respond-provider-retry.test.ts @@ -340,6 +340,49 @@ describe("generateAssistantReply provider retry", () => { }); }); + it("keeps steered messages when yielding after steering drain", async () => { + agentMode.value = "cooperativeYield"; + + const error = await generateAssistantReply("help me", { + requester: { userId: "U123" }, + correlation: { + conversationId: "conversation-yield-steering", + turnId: "turn-yield-steering", + channelId: "C123", + threadTs: "1712345.0005", + }, + drainSteeringMessages: async (inject) => { + const messages = [ + { text: "actually do the other thing", timestampMs: 2_000 }, + ]; + await inject(messages); + return messages; + }, + shouldYield: () => true, + }).then( + () => undefined, + (caught: unknown) => caught, + ); + + expect(isCooperativeTurnYieldError(error)).toBe(true); + const sessionRecord = await turnSessionState.getAgentTurnSessionRecord( + "conversation-yield-steering", + "turn-yield-steering", + ); + expect(sessionRecord).toMatchObject({ + state: "awaiting_resume", + resumeReason: "yield", + sliceId: 1, + }); + expect(sessionRecord?.piMessages.map((message) => message.role)).toEqual([ + "user", + "user", + ]); + const serializedMessages = JSON.stringify(sessionRecord?.piMessages); + expect(serializedMessages).toContain("help me"); + expect(serializedMessages).toContain("actually do the other thing"); + }); + it("throws when a cooperative yield cannot persist its resumable boundary", async () => { agentMode.value = "cooperativeYield"; const upsertSpy = vi From 81d97f38b35962e81609f65f0064e6f5527bb073 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 06:51:15 +0200 Subject: [PATCH 36/45] fix(runtime): Preserve parked continuations Bound signed conversation-work callbacks to a small timestamp window so old queue payloads cannot be replayed indefinitely. When a Slack follow-up arrives during an active parked turn, keep auth pauses parked and fail malformed awaiting continuations before accepting new work. This prevents a fresh turn from replacing the durable session state. Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/runtime/reply-executor.ts | 32 +++- .../src/chat/task-execution/queue-signing.ts | 10 +- .../task-execution/conversation-work.test.ts | 34 ++-- .../integration/slack/bot-handlers.test.ts | 150 +++++++++++++++++- 4 files changed, 212 insertions(+), 14 deletions(-) diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 7198d7dbb..b117caf89 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -397,7 +397,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { ); } }; - const activeTurnId = preparedState.conversation.processing.activeTurnId; + let activeTurnId = preparedState.conversation.processing.activeTurnId; if (preparedState.userMessageAlreadyReplied) { await persistThreadState(thread, { conversation: preparedState.conversation, @@ -435,6 +435,36 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }); return; } + + const sessionRecord = await getAgentTurnSessionRecord( + conversationId, + activeTurnId, + ); + if (sessionRecord?.state === "awaiting_resume") { + if (sessionRecord.resumeReason === "auth") { + await persistThreadState(thread, { + conversation: preparedState.conversation, + }); + await options.onTurnStatePersisted?.(); + return; + } + + await failAgentTurnSessionRecord({ + conversationId, + expectedVersion: sessionRecord.version, + sessionId: activeTurnId, + errorMessage: + "Awaiting turn continuation metadata could not be materialized", + }); + markTurnFailed({ + conversation: preparedState.conversation, + nowMs: Date.now(), + sessionId: activeTurnId, + markConversationMessage, + updateConversationStats, + }); + activeTurnId = undefined; + } } const configReply = await maybeApplyProviderDefaultConfigRequest({ channelConfiguration: preparedState.channelConfiguration, diff --git a/packages/junior/src/chat/task-execution/queue-signing.ts b/packages/junior/src/chat/task-execution/queue-signing.ts index bd367bd97..f886812dd 100644 --- a/packages/junior/src/chat/task-execution/queue-signing.ts +++ b/packages/junior/src/chat/task-execution/queue-signing.ts @@ -4,6 +4,7 @@ import type { ConversationQueueMessage } from "./queue"; const CONVERSATION_WORK_QUEUE_SIGNATURE_CONTEXT = "junior.conversation_work_queue.v1"; const CONVERSATION_WORK_QUEUE_SIGNATURE_VERSION = "v1"; +const CONVERSATION_WORK_QUEUE_SIGNATURE_MAX_SKEW_MS = 5 * 60 * 1000; interface SignedConversationQueueMessage extends ConversationQueueMessage { signature: string; @@ -94,10 +95,17 @@ export function signConversationQueueMessage( /** Verify a signed conversation queue payload from the Vercel Queue callback. */ export function verifySignedConversationQueueMessage( value: unknown, + nowMs = Date.now(), ): ConversationQueueMessage | undefined { const message = parseSignedConversationQueueMessage(value); const secret = getConversationWorkQueueSecret(); - if (!message || !secret) { + if ( + !message || + !secret || + !Number.isFinite(nowMs) || + Math.abs(nowMs - message.signedAtMs) > + CONVERSATION_WORK_QUEUE_SIGNATURE_MAX_SKEW_MS + ) { return undefined; } diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index 86332d4e1..a9f514e99 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -674,25 +674,39 @@ describe("conversation work execution", () => { it("verifies signed Vercel Queue callback payloads", () => { process.env.JUNIOR_SECRET = "conversation-work-secret"; + const signedAtMs = 12_345; + const maxSkewMs = 5 * 60 * 1000; const signed = signConversationQueueMessage( { conversationId: CONVERSATION_ID }, - 12_345, + signedAtMs, ); - expect(verifySignedConversationQueueMessage(signed)).toEqual({ + expect(verifySignedConversationQueueMessage(signed, signedAtMs)).toEqual({ conversationId: CONVERSATION_ID, }); expect( - verifySignedConversationQueueMessage({ - ...signed, - conversationId: "slack:C123:forged", - }), + verifySignedConversationQueueMessage( + { + ...signed, + conversationId: "slack:C123:forged", + }, + signedAtMs, + ), ).toBeUndefined(); expect( - verifySignedConversationQueueMessage({ - ...signed, - signature: "deadbeef", - }), + verifySignedConversationQueueMessage( + { + ...signed, + signature: "deadbeef", + }, + signedAtMs, + ), + ).toBeUndefined(); + expect( + verifySignedConversationQueueMessage(signed, signedAtMs + maxSkewMs + 1), + ).toBeUndefined(); + expect( + verifySignedConversationQueueMessage(signed, signedAtMs - maxSkewMs - 1), ).toBeUndefined(); }); diff --git a/packages/junior/tests/integration/slack/bot-handlers.test.ts b/packages/junior/tests/integration/slack/bot-handlers.test.ts index 68be1d6b0..4c5ff5e02 100644 --- a/packages/junior/tests/integration/slack/bot-handlers.test.ts +++ b/packages/junior/tests/integration/slack/bot-handlers.test.ts @@ -1,8 +1,13 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { JuniorRuntimeServiceOverrides } from "@/chat/app/services"; import { makeAssistantStatus } from "@/chat/slack/assistant-thread/status"; import { getSlackInterruptionMarker } from "@/chat/slack/output"; import { RetryableTurnError } from "@/chat/runtime/turn"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; +import { + getAgentTurnSessionRecord, + upsertAgentTurnSessionRecord, +} from "@/chat/state/turn-session"; import { getCapturedSlackApiCalls, resetSlackApiMockState, @@ -96,12 +101,27 @@ function createAwaitingContinuationState(args: { }; } +function turnPiMessages(text: string) { + return [ + { + role: "user" as const, + content: [{ type: "text" as const, text }], + timestamp: 1, + }, + ]; +} + // ── Tests ──────────────────────────────────────────────────────────── describe("bot handlers (integration)", () => { - afterEach(() => { + beforeEach(async () => { + await disconnectStateAdapter(); + }); + + afterEach(async () => { resetSlackApiMockState(); vi.restoreAllMocks(); + await disconnectStateAdapter(); }); it("handleNewMention: posts reply from generateAssistantReply", async () => { @@ -850,6 +870,132 @@ describe("bot handlers (integration)", () => { expect(followUp?.meta?.skippedReason).toBeUndefined(); }); + it("parks auth-paused active turns without starting a new follow-up turn", async () => { + const conversationId = "slack:C_AUTH_PARKED:1700000000.000"; + const activeSessionId = "turn_msg-auth-original"; + const generateAssistantReply = vi.fn(); + const onTurnStatePersisted = vi.fn(); + await upsertAgentTurnSessionRecord({ + conversationId, + sessionId: activeSessionId, + sliceId: 1, + state: "awaiting_resume", + resumeReason: "auth", + piMessages: turnPiMessages("please use notion"), + }); + const { slackRuntime } = createRuntime({ + services: { + replyExecutor: { + generateAssistantReply, + }, + }, + }); + + const thread = createTestThread({ + id: conversationId, + state: createAwaitingContinuationState({ activeSessionId }), + }); + + await slackRuntime.handleNewMention( + thread, + createTestMessage({ + id: "msg-auth-follow-up", + threadId: conversationId, + text: "any update?", + isMention: true, + }), + { onTurnStatePersisted }, + ); + + expect(generateAssistantReply).not.toHaveBeenCalled(); + expect(onTurnStatePersisted).toHaveBeenCalledOnce(); + expect(thread.posts).toEqual([]); + const state = thread.getState(); + const conversation = ( + state as { + conversation?: { + messages?: Array<{ + id?: string; + meta?: { replied?: boolean; skippedReason?: string }; + }>; + processing?: { activeTurnId?: string }; + }; + } + ).conversation; + expect(conversation?.processing?.activeTurnId).toBe(activeSessionId); + const followUp = conversation?.messages?.find( + (message) => message.id === "msg-auth-follow-up", + ); + expect(followUp).toBeDefined(); + expect(followUp?.meta?.replied).toBeUndefined(); + expect(followUp?.meta?.skippedReason).toBeUndefined(); + }); + + it("fails malformed awaiting continuations before handling the follow-up", async () => { + const conversationId = "slack:C_BAD_CONTINUATION:1700000000.000"; + const activeSessionId = "turn_msg-timeout-original"; + const generateAssistantReply = vi.fn().mockResolvedValue({ + text: "Recovered.", + diagnostics: { + assistantMessageCount: 1, + modelId: "test-model", + outcome: "success" as const, + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: true, + }, + }); + await upsertAgentTurnSessionRecord({ + conversationId, + sessionId: activeSessionId, + sliceId: 1, + state: "awaiting_resume", + resumeReason: "timeout", + piMessages: turnPiMessages("please keep working"), + }); + const { slackRuntime } = createRuntime({ + services: { + replyExecutor: { + generateAssistantReply, + }, + }, + }); + + const thread = createTestThread({ + id: conversationId, + state: createAwaitingContinuationState({ activeSessionId }), + }); + + await slackRuntime.handleNewMention( + thread, + createTestMessage({ + id: "msg-timeout-follow-up", + threadId: conversationId, + text: "what happened?", + isMention: true, + }), + ); + + expect(generateAssistantReply).toHaveBeenCalledOnce(); + expect(postIncludes(thread, "Recovered.")).toBe(true); + const failedRecord = await getAgentTurnSessionRecord( + conversationId, + activeSessionId, + ); + expect(failedRecord?.state).toBe("failed"); + expect(failedRecord?.errorMessage).toBe( + "Awaiting turn continuation metadata could not be materialized", + ); + const state = thread.getState(); + const conversation = ( + state as { + conversation?: { processing?: { activeTurnId?: string } }; + } + ).conversation; + expect(conversation?.processing?.activeTurnId).toBeUndefined(); + }); + it("reschedules an awaiting continuation for repeated delivery of the active message", async () => { const conversationId = "slack:C_TIMEOUT_DUPLICATE:1700000000.000"; const activeSessionId = "turn_msg-duplicate"; From 9ecf3ed47d39e97f547954dac0065bbf31789ce8 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 10:17:45 +0200 Subject: [PATCH 37/45] fix(slack): Harden durable turn continuation Preserve rapid Slack follow-ups through durable conversation work and tighten timeout-resume scheduling. Add shared test adapters for queue, Slack webhook, Slack outbox, waitUntil, and signed resume requests so tests exercise real boundaries with less mocking. Document Django-inspired test adapter principles and stabilize slow Slack integration timeouts under clustered runs. Refs GH-470 Co-Authored-By: GPT-5 Codex --- AGENTS.md | 1 + .../junior/src/chat/ingress/slack-webhook.ts | 27 +- packages/junior/src/chat/respond.ts | 30 +- .../junior/src/chat/runtime/reply-executor.ts | 5 + .../junior/src/chat/runtime/slack-runtime.ts | 15 +- .../src/chat/runtime/timeout-resume-runner.ts | 17 +- packages/junior/src/chat/runtime/turn.ts | 17 + .../src/chat/task-execution/queue-signing.ts | 2 +- .../src/chat/task-execution/slack-work.ts | 22 +- .../junior/src/chat/task-execution/store.ts | 23 +- packages/junior/src/handlers/turn-resume.ts | 8 +- .../component/runtime/timeout-resume.test.ts | 76 +-- .../task-execution/conversation-work.test.ts | 153 ++++-- .../slack-conversation-work.test.ts | 484 +++++++++--------- .../tests/fixtures/conversation-work.ts | 203 +++++--- .../junior/tests/fixtures/slack-api-outbox.ts | 40 ++ .../tests/fixtures/slack/webhook-client.ts | 82 +++ packages/junior/tests/fixtures/turn-resume.ts | 92 ++++ packages/junior/tests/fixtures/wait-until.ts | 25 + .../tests/integration/heartbeat.test.ts | 127 ++--- .../integration/oauth-callback-slack.test.ts | 4 +- .../slack/app-home-webhook.test.ts | 254 +++------ .../slack/assistant-thread-contract.test.ts | 105 ++-- .../slack/attachment-media-behavior.test.ts | 2 +- .../slack/bot-image-hydration.test.ts | 4 +- ...onversation-turn-steering-behavior.test.ts | 96 +++- .../slack/message-changed-behavior.test.ts | 154 ++---- .../message-changed-reply-contract.test.ts | 54 +- .../message-im-attachment-contract.test.ts | 50 +- .../processing-reaction-behavior.test.ts | 42 +- .../slack/webhook-auth-boundary.test.ts | 79 ++- .../integration/turn-resume-slack.test.ts | 174 +++---- .../unit/handlers/mcp-oauth-callback.test.ts | 25 +- .../tests/unit/handlers/turn-resume.test.ts | 93 ++-- .../unit/runtime/respond-error-path.test.ts | 30 +- .../tests/unit/slack/slack-runtime.test.ts | 48 +- policies/README.md | 1 + policies/test-adapters.md | 27 + 38 files changed, 1416 insertions(+), 1275 deletions(-) create mode 100644 packages/junior/tests/fixtures/slack-api-outbox.ts create mode 100644 packages/junior/tests/fixtures/slack/webhook-client.ts create mode 100644 packages/junior/tests/fixtures/turn-resume.ts create mode 100644 packages/junior/tests/fixtures/wait-until.ts create mode 100644 policies/test-adapters.md diff --git a/AGENTS.md b/AGENTS.md index da6865086..603cedaf2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,6 +65,7 @@ Co-Authored-By: (agent model name) - `policies/frontend-components.md` (Tailwind colocation and component-owned frontend styling) - `policies/interface-design.md` (naming, module paths, and minimal interface boundaries) - `policies/policy-template.md` (template for adding new policy docs) +- `policies/test-adapters.md` (Django-inspired shared test adapters, outboxes, and isolation rules) ## Investigation-First Development diff --git a/packages/junior/src/chat/ingress/slack-webhook.ts b/packages/junior/src/chat/ingress/slack-webhook.ts index 6aaf13e2f..d99c01f4e 100644 --- a/packages/junior/src/chat/ingress/slack-webhook.ts +++ b/packages/junior/src/chat/ingress/slack-webhook.ts @@ -180,6 +180,10 @@ function shouldIgnoreMessage(message: Message): boolean { ); } +function shouldPersistBeforeAck(body: SlackEventEnvelope): boolean { + return body.event?.type === "app_mention" || body.event?.type === "message"; +} + async function persistSlackMessage(args: { adapter: SlackAdapter; installation: SlackInstallationContext; @@ -625,15 +629,20 @@ export async function handleSlackWebhook(args: { } if (parsed.type === "event_callback") { - enqueue( - args.waitUntil, - handleSlackEvent({ - body: parsed, - services: args.services, - }).catch((error) => { - logException(error, "slack_event_enqueue_failed"); - }), - ); + const eventTask = handleSlackEvent({ + body: parsed, + services: args.services, + }); + if (shouldPersistBeforeAck(parsed)) { + await eventTask; + } else { + enqueue( + args.waitUntil, + eventTask.catch((error) => { + logException(error, "slack_event_enqueue_failed"); + }), + ); + } } return new Response("ok", { status: 200 }); diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index 095c24aa1..bb42857e5 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -85,6 +85,7 @@ import { mergeArtifactsState } from "@/chat/runtime/thread-state"; import { CooperativeTurnYieldError, RetryableTurnError, + isTurnInputCommitLostError, isRetryableTurnError, } from "@/chat/runtime/turn"; import { @@ -228,6 +229,7 @@ export interface ReplyRequestContext { onArtifactStateUpdated?: ( artifactState: ThreadArtifactsState, ) => void | Promise; + onInputCommitted?: () => void | Promise; toolOverrides?: { imageGenerate?: ImageGenerateToolDeps; webFetch?: WebFetchToolDeps; @@ -469,6 +471,7 @@ export async function generateAssistantReply( let sandboxExecutor: SandboxExecutor | undefined; let timedOut = false; let yielded = false; + let inputCommitted = false; let turnUsage: AgentTurnUsage | undefined; let thinkingSelection: TurnThinkingSelection | undefined; const requester = requesterFromContext( @@ -1105,15 +1108,22 @@ export async function generateAssistantReply( // ── Agent execution ────────────────────────────────────────────── let hasEmittedText = false; let needsSeparator = false; + const commitInput = async (): Promise => { + if (inputCommitted) { + return; + } + await context.onInputCommitted?.(); + inputCommitted = true; + }; const persistSafeBoundary = async ( messages: PiMessage[], - ): Promise => { + ): Promise => { if ( !turnSessionState.canUseTurnSession || !sessionConversationId || !sessionId ) { - return; + return false; } await persistRunningSessionRecord({ @@ -1127,6 +1137,7 @@ export async function generateAssistantReply( requester, }); latestSafeBoundaryMessages = [...messages]; + return true; }; const drainSteeringMessages = async (): Promise => { if ( @@ -1206,7 +1217,9 @@ export async function generateAssistantReply( const unsubscribe = agent.subscribe((event) => { if (event.type === "turn_end" && event.toolResults.length > 0) { - return persistSafeBoundary([...agent!.state.messages]); + return persistSafeBoundary([...agent!.state.messages]).then( + () => undefined, + ); } if (event.type === "message_start") { Promise.resolve(context.onAssistantMessageStart?.()).catch((error) => { @@ -1274,10 +1287,13 @@ export async function generateAssistantReply( timestamp: Date.now(), } as PiMessage; if (!resumedFromSessionRecord) { - await persistSafeBoundary([ + const promptPersisted = await persistSafeBoundary([ ...agent.state.messages, freshPromptMessage, ]); + if (promptPersisted) { + await commitInput(); + } } const runAgentStep = async ( @@ -1607,9 +1623,15 @@ export async function generateAssistantReply( if (isRetryableTurnError(error)) { throw error; } + if (isTurnInputCommitLostError(error)) { + throw error; + } if (error instanceof AuthorizationFlowDisabledError) { throw error; } + if (context.onInputCommitted && !inputCommitted) { + throw error; + } logException( error, diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index b117caf89..3967f4c88 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -270,6 +270,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { options: { beforeFirstResponsePost?: () => Promise; explicitMention?: boolean; + onInputCommitted?: () => Promise; onToolInvocation?: (invocation: TurnToolInvocation) => void; onTurnStatePersisted?: () => Promise; preparedState?: PreparedTurnState; @@ -403,6 +404,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { conversation: preparedState.conversation, }); await options.onTurnStatePersisted?.(); + await options.onInputCommitted?.(); return; } if (conversationId && activeTurnId) { @@ -446,6 +448,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { conversation: preparedState.conversation, }); await options.onTurnStatePersisted?.(); + await options.onInputCommitted?.(); return; } @@ -499,6 +502,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { conversation: preparedState.conversation, }); await options.onTurnStatePersisted?.(); + await options.onInputCommitted?.(); return; } startActiveTurn({ @@ -776,6 +780,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }, onStatus: (nextStatus) => status.update(nextStatus), onToolInvocation: options.onToolInvocation, + onInputCommitted: options.onInputCommitted, drainSteeringMessages, shouldYield: options.shouldYield, }, diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index 774c8c2f0..c9451c0c7 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -10,6 +10,7 @@ import type { Message, MessageContext, Thread } from "chat"; import { getSubscribedReplyPreflightDecision } from "@/chat/services/subscribed-decision"; import { isCooperativeTurnYieldError, + isTurnInputCommitLostError, isRetryableTurnError, } from "@/chat/runtime/turn"; import { buildTurnFailureResponse } from "@/chat/logging"; @@ -52,6 +53,7 @@ export interface ReplyHooks { inject: (messages: Message[]) => Promise, ) => Promise; messageContext?: MessageContext; + onInputCommitted?: () => Promise; onToolInvocation?: (invocation: TurnToolInvocation) => void; onTurnStatePersisted?: () => Promise; shouldYield?: () => boolean; @@ -146,6 +148,7 @@ export interface SlackTurnRuntimeDependencies { options?: { beforeFirstResponsePost?: () => Promise; explicitMention?: boolean; + onInputCommitted?: () => Promise; onToolInvocation?: (invocation: TurnToolInvocation) => void; onTurnStatePersisted?: () => Promise; preparedState?: TPreparedState; @@ -413,6 +416,7 @@ export function createSlackTurnRuntime< explicitMention: true, beforeFirstResponsePost: hooks?.beforeFirstResponsePost, queuedMessages, + onInputCommitted: hooks?.onInputCommitted, onToolInvocation: toolInvocationHook, drainSteeringMessages, onTurnStatePersisted: hooks?.onTurnStatePersisted, @@ -420,7 +424,10 @@ export function createSlackTurnRuntime< }); }); } catch (error) { - if (isCooperativeTurnYieldError(error)) { + if ( + isCooperativeTurnYieldError(error) || + isTurnInputCommitLostError(error) + ) { throw error; } const errorContext = logContext({ @@ -617,6 +624,7 @@ export function createSlackTurnRuntime< preparedState, beforeFirstResponsePost: hooks?.beforeFirstResponsePost, queuedMessages, + onInputCommitted: hooks?.onInputCommitted, onToolInvocation: toolInvocationHook, drainSteeringMessages, onTurnStatePersisted: hooks?.onTurnStatePersisted, @@ -624,7 +632,10 @@ export function createSlackTurnRuntime< }); }); } catch (error) { - if (isCooperativeTurnYieldError(error)) { + if ( + isCooperativeTurnYieldError(error) || + isTurnInputCommitLostError(error) + ) { throw error; } const errorContext = logContext({ diff --git a/packages/junior/src/chat/runtime/timeout-resume-runner.ts b/packages/junior/src/chat/runtime/timeout-resume-runner.ts index d21b61b31..c459449f1 100644 --- a/packages/junior/src/chat/runtime/timeout-resume-runner.ts +++ b/packages/junior/src/chat/runtime/timeout-resume-runner.ts @@ -29,7 +29,7 @@ import { import { coerceThreadArtifactsState } from "@/chat/state/artifacts"; import { isRetryableTurnError, markTurnFailed } from "@/chat/runtime/turn"; import { - scheduleTurnTimeoutResume, + scheduleTurnTimeoutResume as defaultScheduleTurnTimeoutResume, type TurnContinuationRequest, } from "@/chat/services/timeout-resume"; import { parseSlackThreadId } from "@/chat/slack/context"; @@ -42,6 +42,13 @@ import { const TIMEOUT_RESUME_LOCK_RETRY_DELAYS_MS = [250, 1_000, 2_000] as const; +/** Runtime ports for timeout continuation scheduling. */ +export interface TimeoutResumeRunnerOptions { + scheduleTurnTimeoutResume?: ( + request: TurnContinuationRequest, + ) => Promise; +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -132,6 +139,7 @@ async function persistFailedReplyState( */ export async function resumeTimedOutTurn( payload: TurnContinuationRequest, + options: TimeoutResumeRunnerOptions = {}, ): Promise { const thread = parseSlackThreadId(payload.conversationId); if (!thread) { @@ -139,6 +147,8 @@ export async function resumeTimedOutTurn( `Timeout resume requires a Slack thread conversation id, got "${payload.conversationId}"`, ); } + const scheduleTurnTimeoutResume = + options.scheduleTurnTimeoutResume ?? defaultScheduleTurnTimeoutResume; return await resumeSlackTurn({ messageText: "", @@ -280,13 +290,16 @@ export async function resumeTimedOutTurn( */ export async function resumeTimedOutTurnWithLockRetry( payload: TurnContinuationRequest, + options: TimeoutResumeRunnerOptions = {}, ): Promise { + const scheduleTurnTimeoutResume = + options.scheduleTurnTimeoutResume ?? defaultScheduleTurnTimeoutResume; for (const [attempt, delayMs] of [ ...TIMEOUT_RESUME_LOCK_RETRY_DELAYS_MS, undefined, ].entries()) { try { - return await resumeTimedOutTurn(payload); + return await resumeTimedOutTurn(payload, options); } catch (error) { if (!(error instanceof ResumeTurnBusyError)) { throw error; diff --git a/packages/junior/src/chat/runtime/turn.ts b/packages/junior/src/chat/runtime/turn.ts index 6ee27e7c7..ae3d7b2ca 100644 --- a/packages/junior/src/chat/runtime/turn.ts +++ b/packages/junior/src/chat/runtime/turn.ts @@ -77,6 +77,23 @@ export function isCooperativeTurnYieldError( return error instanceof CooperativeTurnYieldError; } +/** Error indicating durable turn input could not be committed by the worker owner. */ +export class TurnInputCommitLostError extends Error { + readonly code = "turn_input_commit_lost"; + + constructor(message = "Turn input commit lost its durable owner") { + super(message); + this.name = "TurnInputCommitLostError"; + } +} + +/** Return whether an error means the durable worker lost input ownership. */ +export function isTurnInputCommitLostError( + error: unknown, +): error is TurnInputCommitLostError { + return error instanceof TurnInputCommitLostError; +} + // --------------------------------------------------------------------------- // Turn lifecycle mutations // --------------------------------------------------------------------------- diff --git a/packages/junior/src/chat/task-execution/queue-signing.ts b/packages/junior/src/chat/task-execution/queue-signing.ts index f886812dd..6755c5e80 100644 --- a/packages/junior/src/chat/task-execution/queue-signing.ts +++ b/packages/junior/src/chat/task-execution/queue-signing.ts @@ -4,7 +4,7 @@ import type { ConversationQueueMessage } from "./queue"; const CONVERSATION_WORK_QUEUE_SIGNATURE_CONTEXT = "junior.conversation_work_queue.v1"; const CONVERSATION_WORK_QUEUE_SIGNATURE_VERSION = "v1"; -const CONVERSATION_WORK_QUEUE_SIGNATURE_MAX_SKEW_MS = 5 * 60 * 1000; +const CONVERSATION_WORK_QUEUE_SIGNATURE_MAX_SKEW_MS = 60 * 60 * 1000; interface SignedConversationQueueMessage extends ConversationQueueMessage { signature: string; diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 7c2b249b2..a5cfb86b8 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -8,7 +8,11 @@ import { type StateAdapter, } from "chat"; import type { SlackTurnRuntime } from "@/chat/runtime/slack-runtime"; -import { isCooperativeTurnYieldError } from "@/chat/runtime/turn"; +import { + isCooperativeTurnYieldError, + isTurnInputCommitLostError, + TurnInputCommitLostError, +} from "@/chat/runtime/turn"; import { normalizeIncomingSlackThreadId } from "@/chat/ingress/message-router"; import { rehydrateAttachmentFetchers } from "@/chat/queue/thread-message-dispatcher"; import { getAwaitingTurnContinuationRequest } from "@/chat/services/timeout-resume"; @@ -242,7 +246,7 @@ export function createSlackConversationWorker( }), ); if (records.length === 0) { - await resumeAwaitingContinuation(context.conversationId); + await resumeContinuation(context.conversationId); return { status: "completed" }; } @@ -291,7 +295,6 @@ export function createSlackConversationWorker( (record) => record.inboundMessageId, ); let initialMessagesPersisted = false; - let leaseLostDuringTurnHandoff = false; const markInitialMessagesInjected = async (): Promise => { if (initialMessagesPersisted) { return true; @@ -305,11 +308,10 @@ export function createSlackConversationWorker( initialMessagesPersisted = marked; return marked; }; - const onTurnStatePersisted = async (): Promise => { + const onInputCommitted = async (): Promise => { if (!(await markInitialMessagesInjected())) { - leaseLostDuringTurnHandoff = true; - throw new Error( - `Conversation work lease lost before Slack turn handoff for ${context.conversationId}`, + throw new TurnInputCommitLostError( + `Conversation work lease lost before Slack input commit for ${context.conversationId}`, ); } }; @@ -335,7 +337,7 @@ export function createSlackConversationWorker( await options.runtime.handleNewMention(thread, latestMessage, { messageContext, drainSteeringMessages, - onTurnStatePersisted, + onInputCommitted, shouldYield: context.shouldYield, }); return; @@ -344,14 +346,14 @@ export function createSlackConversationWorker( await options.runtime.handleSubscribedMessage(thread, latestMessage, { messageContext, drainSteeringMessages, - onTurnStatePersisted, + onInputCommitted, shouldYield: context.shouldYield, }); } catch (error) { if (isCooperativeTurnYieldError(error)) { return { status: "yielded" } satisfies ConversationWorkerResult; } - if (leaseLostDuringTurnHandoff) { + if (isTurnInputCommitLostError(error)) { return { status: "lost_lease" } satisfies ConversationWorkerResult; } throw error; diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index 45642fb86..d3017a67c 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -642,19 +642,28 @@ export async function drainConversationMailbox(args: { state?: StateAdapter; }): Promise { const nowMs = args.nowMs ?? now(); - return await withConversationMutation(args, async (state) => { + const pending = await withConversationMutation(args, async (state) => { const current = await readWorkState(state, args.conversationId); if (!current || current.lease?.leaseToken !== args.leaseToken) { throw new Error( `Conversation work lease is not held for ${args.conversationId}`, ); } - const pending = pendingMessages(current); - if (pending.length === 0) { - return []; - } + return pendingMessages(current); + }); + if (pending.length === 0) { + return []; + } + + await args.inject(pending); - await args.inject(pending); + await withConversationMutation(args, async (state) => { + const current = await readWorkState(state, args.conversationId); + if (!current || current.lease?.leaseToken !== args.leaseToken) { + throw new Error( + `Conversation work lease is not held for ${args.conversationId}`, + ); + } const drainedIds = new Set( pending.map((message) => message.inboundMessageId), ); @@ -672,8 +681,8 @@ export async function drainConversationMailbox(args: { needsRun: hasPending, updatedAtMs: nowMs, }); - return pending; }); + return pending; } /** Mark selected leased mailbox entries after their session-log injection succeeds. */ diff --git a/packages/junior/src/handlers/turn-resume.ts b/packages/junior/src/handlers/turn-resume.ts index 53608de11..bae7fb726 100644 --- a/packages/junior/src/handlers/turn-resume.ts +++ b/packages/junior/src/handlers/turn-resume.ts @@ -7,7 +7,10 @@ */ import { logException } from "@/chat/logging"; import { runWithTurnRequestDeadline } from "@/chat/runtime/request-deadline"; -import { resumeTimedOutTurnWithLockRetry } from "@/chat/runtime/timeout-resume-runner"; +import { + resumeTimedOutTurnWithLockRetry, + type TimeoutResumeRunnerOptions, +} from "@/chat/runtime/timeout-resume-runner"; import { verifyTurnTimeoutResumeRequest } from "@/chat/services/timeout-resume"; import type { WaitUntilFn } from "@/handlers/types"; @@ -15,6 +18,7 @@ import type { WaitUntilFn } from "@/handlers/types"; export async function POST( request: Request, waitUntil: WaitUntilFn, + options: TimeoutResumeRunnerOptions = {}, ): Promise { const payload = await verifyTurnTimeoutResumeRequest(request); if (!payload) { @@ -23,7 +27,7 @@ export async function POST( waitUntil(() => runWithTurnRequestDeadline(() => - resumeTimedOutTurnWithLockRetry(payload).catch((error) => { + resumeTimedOutTurnWithLockRetry(payload, options).catch((error) => { logException( error, "timeout_resume_handler_failed", diff --git a/packages/junior/tests/component/runtime/timeout-resume.test.ts b/packages/junior/tests/component/runtime/timeout-resume.test.ts index 04c94f4c4..4caa6a0eb 100644 --- a/packages/junior/tests/component/runtime/timeout-resume.test.ts +++ b/packages/junior/tests/component/runtime/timeout-resume.test.ts @@ -1,12 +1,12 @@ -import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { scheduleTurnTimeoutResume, verifyTurnTimeoutResumeRequest, } from "@/chat/services/timeout-resume"; -import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { getConversationWorkState } from "@/chat/task-execution/store"; import { disconnectStateAdapter } from "@/chat/state/adapter"; +import { createConversationWorkQueueTestAdapter } from "../../fixtures/conversation-work"; +import { createTurnResumeTestClient } from "../../fixtures/turn-resume"; const ORIGINAL_ENV = vi.hoisted(() => { const original = { @@ -19,43 +19,6 @@ const ORIGINAL_ENV = vi.hoisted(() => { return original; }); -class FakeQueue implements ConversationWorkQueue { - sent: Array<{ - conversationId: string; - delayMs?: number; - idempotencyKey?: string; - }> = []; - - async send( - message: { conversationId: string }, - options?: { delayMs?: number; idempotencyKey?: string }, - ): Promise<{ messageId: string }> { - this.sent.push({ - conversationId: message.conversationId, - delayMs: options?.delayMs, - idempotencyKey: options?.idempotencyKey, - }); - return { messageId: `queue-${this.sent.length}` }; - } -} - -function makeSignedResumeRequest(body: Record): Request { - const timestamp = Date.now().toString(); - const serializedBody = JSON.stringify(body); - const signature = createHmac("sha256", "resume-secret") - .update(`junior.turn_timeout_resume.v1:${timestamp}:${serializedBody}`) - .digest("hex"); - return new Request("https://junior.example.com/api/internal/turn-resume", { - method: "POST", - headers: { - "content-type": "application/json", - "x-junior-resume-timestamp": timestamp, - "x-junior-resume-signature": `v1=${signature}`, - }, - body: serializedBody, - }); -} - function restoreEnv(name: string, value: string | undefined): void { if (value === undefined) { delete process.env[name]; @@ -80,7 +43,7 @@ describe("timeout resume callback signing", () => { }); it("marks timeout continuations runnable and wakes the durable queue", async () => { - const queue = new FakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const conversationId = "slack:C123:1712345.0001"; await scheduleTurnTimeoutResume( @@ -92,7 +55,7 @@ describe("timeout resume callback signing", () => { { queue, nowMs: 1_000 }, ); - expect(queue.sent).toEqual([ + expect(queue.sentRecords()).toEqual([ { conversationId, idempotencyKey: `timeout:${conversationId}:turn_msg_1:3`, @@ -108,7 +71,10 @@ describe("timeout resume callback signing", () => { }); it("still verifies signed callbacks that were already in flight", async () => { - const request = makeSignedResumeRequest({ + const client = createTurnResumeTestClient({ + juniorSecret: "resume-secret", + }); + const request = client.request({ conversationId: "slack:C123:1712345.0001", sessionId: "turn_msg_1", expectedVersion: 3, @@ -122,10 +88,13 @@ describe("timeout resume callback signing", () => { }); it("accepts the previous expected checkpoint version field", async () => { - const request = makeSignedResumeRequest({ + const client = createTurnResumeTestClient({ + juniorSecret: "resume-secret", + }); + const request = client.legacyRequest({ conversationId: "slack:C123:1712345.0001", sessionId: "turn_msg_1", - expectedCheckpointVersion: 3, + expectedVersion: 3, }); await expect(verifyTurnTimeoutResumeRequest(request)).resolves.toEqual({ @@ -136,26 +105,25 @@ describe("timeout resume callback signing", () => { }); it("rejects requests whose signature does not match the body", async () => { - const request = makeSignedResumeRequest({ + const client = createTurnResumeTestClient({ + juniorSecret: "resume-secret", + }); + const request = client.invalidSignature({ conversationId: "slack:C123:1712345.0001", sessionId: "turn_msg_1", expectedVersion: 3, }); - const headers = new Headers(request.headers); - headers.set("x-junior-resume-signature", "v1=deadbeef"); - const tampered = new Request(request.url, { - method: request.method, - headers, - body: await request.text(), - }); await expect( - verifyTurnTimeoutResumeRequest(tampered), + verifyTurnTimeoutResumeRequest(request), ).resolves.toBeUndefined(); }); it("requires the Junior secret to verify legacy callbacks", async () => { - const request = makeSignedResumeRequest({ + const client = createTurnResumeTestClient({ + juniorSecret: "resume-secret", + }); + const request = client.request({ conversationId: "slack:C123:1712345.0001", sessionId: "turn_msg_1", expectedVersion: 3, diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index a9f514e99..b46d04859 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -28,11 +28,12 @@ import { import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { CONVERSATION_ID, - createFakeQueue, + createConversationWorkQueueTestAdapter, deferred, delayIndexLockOnce, delayMutationLockUntil, inboundMessage, + observeConversationMutationLock, } from "../../fixtures/conversation-work"; describe("conversation work execution", () => { @@ -53,7 +54,7 @@ describe("conversation work execution", () => { }); it("stores inbound mailbox messages idempotently", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await expect( appendAndEnqueueInboundMessage({ message: inboundMessage("m1"), @@ -77,11 +78,11 @@ describe("conversation work execution", () => { }); expect(state?.messages).toHaveLength(1); expect(state ? countPendingConversationMessages(state) : 0).toBe(1); - expect(queue.sent).toHaveLength(2); + expect(queue.sentRecords()).toHaveLength(2); }); it("retries transient conversation work index lock contention", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = delayIndexLockOnce(getStateAdapter()); await expect( @@ -98,12 +99,12 @@ describe("conversation work execution", () => { state, }); expect(work?.messages).toHaveLength(1); - expect(queue.sent).toHaveLength(1); + expect(queue.sentRecords()).toHaveLength(1); }); it("waits through same-conversation mutation lock contention", async () => { vi.useFakeTimers({ now: 1_000 }); - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = delayMutationLockUntil({ conversationId: CONVERSATION_ID, readyAtMs: 3_500, @@ -122,12 +123,12 @@ describe("conversation work execution", () => { status: "appended", queueMessageId: "queue-1", }); - expect(queue.sent).toHaveLength(1); + expect(queue.sentRecords()).toHaveLength(1); }); it("repairs pending mailbox work when the initial queue send fails", async () => { - const queue = createFakeQueue(); - queue.fail = true; + const queue = createConversationWorkQueueTestAdapter(); + queue.rejectSends(); await expect( appendAndEnqueueInboundMessage({ message: inboundMessage("m1"), @@ -136,14 +137,14 @@ describe("conversation work execution", () => { }), ).rejects.toThrow("queue unavailable"); - queue.fail = false; + queue.allowSends(); await expect( recoverConversationWork({ nowMs: 62_000, queue, }), ).resolves.toEqual({ expiredLeaseCount: 0, pendingCount: 1 }); - expect(queue.sent).toEqual([ + expect(queue.sentRecords()).toEqual([ { conversationId: CONVERSATION_ID, idempotencyKey: `heartbeat:pending:${CONVERSATION_ID}:62000`, @@ -152,7 +153,7 @@ describe("conversation work execution", () => { }); it("defers duplicate queue nudges while a conversation lease is active", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const entered = deferred(); const finish = deferred(); @@ -180,7 +181,7 @@ describe("conversation work execution", () => { }), ).resolves.toEqual({ status: "active" }); expect(runs).toBe(1); - expect(queue.sent).toMatchObject([ + expect(queue.sentRecords()).toMatchObject([ { conversationId: CONVERSATION_ID, delayMs: CONVERSATION_WORK_DEFER_DELAY_MS, @@ -192,7 +193,7 @@ describe("conversation work execution", () => { }); it("requeues work requested while a lease is running", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); let currentNowMs = 1_000; await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -218,7 +219,7 @@ describe("conversation work execution", () => { expect(state?.lease).toBeUndefined(); expect(state?.needsRun).toBe(true); expect(state ? countPendingConversationMessages(state) : 0).toBe(0); - expect(queue.sent).toMatchObject([ + expect(queue.sentRecords()).toMatchObject([ { conversationId: CONVERSATION_ID, idempotencyKey: `pending:${CONVERSATION_ID}:2000`, @@ -227,7 +228,7 @@ describe("conversation work execution", () => { }); it("uses fresh queue idempotency keys for repeated worker requeues", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); let currentNowMs = 1_000; await requestConversationWork({ conversationId: CONVERSATION_ID, @@ -254,14 +255,14 @@ describe("conversation work execution", () => { await runSlice(2_000); await runSlice(63_000); - expect(queue.sent.map((send) => send.idempotencyKey)).toEqual([ + expect(queue.sentRecords().map((send) => send.idempotencyKey)).toEqual([ `pending:${CONVERSATION_ID}:2000`, `pending:${CONVERSATION_ID}:63000`, ]); }); it("nudges failed worker runs before releasing runnable work", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); let currentNowMs = 1_000; await requestConversationWork({ conversationId: CONVERSATION_ID, @@ -285,7 +286,7 @@ describe("conversation work execution", () => { expect(state?.lease).toBeUndefined(); expect(state?.needsRun).toBe(true); expect(state?.lastEnqueuedAtMs).toBe(2_000); - expect(queue.sent).toMatchObject([ + expect(queue.sentRecords()).toMatchObject([ { conversationId: CONVERSATION_ID, idempotencyKey: `error:${CONVERSATION_ID}:2000`, @@ -294,7 +295,7 @@ describe("conversation work execution", () => { }); it("drains pending messages and completes the leased conversation", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const injected: InboundMessageRecord[][] = []; @@ -319,9 +320,68 @@ describe("conversation work execution", () => { expect(state ? countPendingConversationMessages(state) : 0).toBe(0); }); + it("does not block new mailbox appends while injection is in progress", async () => { + const queue = createConversationWorkQueueTestAdapter(); + const observed = observeConversationMutationLock({ + conversationId: CONVERSATION_ID, + state: getStateAdapter(), + }); + await appendInboundMessage({ + message: inboundMessage("m1"), + nowMs: 1_000, + state: observed.state, + }); + const injectionStarted = deferred(); + const finishInjection = deferred(); + + await expect( + processConversationWork(CONVERSATION_ID, { + queue, + state: observed.state, + run: async (context) => { + const drain = context.drainMailbox(async () => { + expect(observed.isHeld()).toBe(false); + injectionStarted.resolve(); + await finishInjection.promise; + }); + await injectionStarted.promise; + + const append = appendInboundMessage({ + message: inboundMessage("m2", { + createdAtMs: 2_000, + receivedAtMs: 2_100, + }), + nowMs: 2_100, + state: observed.state, + }); + + finishInjection.resolve(); + await drain; + await append; + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "pending_requeued" }); + + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + state: observed.state, + }); + expect(state?.needsRun).toBe(true); + expect(state ? countPendingConversationMessages(state) : 0).toBe(1); + expect(state?.messages.map((message) => message.inboundMessageId)).toEqual([ + "m1", + "m2", + ]); + expect(state?.messages.map((message) => message.injectedAtMs)).toEqual([ + expect.any(Number), + undefined, + ]); + }); + it("extends the lease with worker check-ins during long execution", async () => { vi.useFakeTimers({ now: 1_000 }); - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const entered = deferred(); const finish = deferred(); @@ -358,7 +418,7 @@ describe("conversation work execution", () => { }); it("requeues an expired conversation lease from heartbeat", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( startConversationWork({ conversationId: CONVERSATION_ID, nowMs: 2_000 }), @@ -375,7 +435,7 @@ describe("conversation work execution", () => { }); expect(state?.lease).toBeUndefined(); expect(state?.needsRun).toBe(true); - expect(queue.sent).toMatchObject([ + expect(queue.sentRecords()).toMatchObject([ { conversationId: CONVERSATION_ID, idempotencyKey: `heartbeat:lease:${CONVERSATION_ID}:92000`, @@ -384,7 +444,7 @@ describe("conversation work execution", () => { }); it("keeps an expired injected-message lease runnable for continuation recovery", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const lease = await startConversationWork({ conversationId: CONVERSATION_ID, @@ -416,7 +476,7 @@ describe("conversation work execution", () => { }); it("requeues pending mailbox work with no recent queue marker", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( @@ -425,11 +485,11 @@ describe("conversation work execution", () => { queue, }), ).resolves.toEqual({ expiredLeaseCount: 0, pendingCount: 1 }); - expect(queue.sent).toHaveLength(1); + expect(queue.sentRecords()).toHaveLength(1); }); it("uses fresh queue idempotency keys for repeated heartbeat recovery", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( @@ -445,14 +505,14 @@ describe("conversation work execution", () => { }), ).resolves.toEqual({ expiredLeaseCount: 0, pendingCount: 1 }); - expect(queue.sent.map((send) => send.idempotencyKey)).toEqual([ + expect(queue.sentRecords().map((send) => send.idempotencyKey)).toEqual([ `heartbeat:pending:${CONVERSATION_ID}:62000`, `heartbeat:pending:${CONVERSATION_ID}:122001`, ]); }); it("runs conversation work recovery from the core heartbeat", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await runHeartbeat({ @@ -460,7 +520,7 @@ describe("conversation work execution", () => { conversationWorkQueue: queue, }); - expect(queue.sent).toEqual([ + expect(queue.sentRecords()).toEqual([ { conversationId: CONVERSATION_ID, idempotencyKey: `heartbeat:pending:${CONVERSATION_ID}:62000`, @@ -469,7 +529,7 @@ describe("conversation work execution", () => { }); it("injects messages that arrive during active execution at a safe boundary", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const injected: string[][] = []; @@ -497,7 +557,7 @@ describe("conversation work execution", () => { }); it("clears the run marker after draining messages that arrived during active execution", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); await expect( @@ -526,7 +586,7 @@ describe("conversation work execution", () => { }); it("requeues instead of completing when final mailbox work remains", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); let currentNowMs = 1_000; await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -548,7 +608,7 @@ describe("conversation work execution", () => { }, }), ).resolves.toEqual({ status: "pending_requeued" }); - expect(queue.sent).toMatchObject([ + expect(queue.sentRecords()).toMatchObject([ { conversationId: CONVERSATION_ID, idempotencyKey: `pending:${CONVERSATION_ID}:2100`, @@ -557,7 +617,7 @@ describe("conversation work execution", () => { }); it("yields cooperatively and leaves the conversation resumable", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); let currentNowMs = 1_000; await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -579,7 +639,7 @@ describe("conversation work execution", () => { }); expect(state?.lease).toBeUndefined(); expect(state?.needsRun).toBe(true); - expect(queue.sent).toMatchObject([ + expect(queue.sentRecords()).toMatchObject([ { conversationId: CONVERSATION_ID, idempotencyKey: `yield:${CONVERSATION_ID}:242000`, @@ -675,7 +735,7 @@ describe("conversation work execution", () => { it("verifies signed Vercel Queue callback payloads", () => { process.env.JUNIOR_SECRET = "conversation-work-secret"; const signedAtMs = 12_345; - const maxSkewMs = 5 * 60 * 1000; + const maxSkewMs = 60 * 60 * 1000; const signed = signConversationQueueMessage( { conversationId: CONVERSATION_ID }, signedAtMs, @@ -710,8 +770,23 @@ describe("conversation work execution", () => { ).toBeUndefined(); }); + it("keeps queue signatures valid across default visibility redelivery", () => { + process.env.JUNIOR_SECRET = "conversation-work-secret"; + const signedAtMs = 12_345; + const signed = signConversationQueueMessage( + { conversationId: CONVERSATION_ID }, + signedAtMs, + ); + + expect( + verifySignedConversationQueueMessage(signed, signedAtMs + 330_000), + ).toEqual({ + conversationId: CONVERSATION_ID, + }); + }); + it("processes Vercel Queue payloads through the leased worker", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); const injected: string[] = []; @@ -735,7 +810,7 @@ describe("conversation work execution", () => { }); it("rejects malformed Vercel Queue payloads", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); await expect( processConversationQueueMessage( diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index 06e17d861..bdcccfdb6 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { Message, Thread } from "chat"; +import type { Message, StateAdapter, Thread } from "chat"; import { CooperativeTurnYieldError } from "@/chat/runtime/turn"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; import { @@ -11,6 +11,7 @@ import { startConversationWork, } from "@/chat/task-execution/store"; import { processConversationWork } from "@/chat/task-execution/worker"; +import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { @@ -20,15 +21,52 @@ import { import { persistThreadStateById } from "@/chat/runtime/thread-state"; import { CONVERSATION_ID, - createFakeQueue, + createConversationWorkQueueTestAdapter, SLACK_BOT_USER_ID, createNoopSlackWebhookRuntime, createSlackAdapterFixture, + type ConversationWorkQueueTestAdapter, handleSlackWebhookAndFlush, slackEnvelope, slackWebhookRequest, } from "../../fixtures/conversation-work"; +type SlackWorkerOptions = Parameters[0]; + +interface ProcessQueuedSlackWorkArgs { + getSlackAdapter: SlackWorkerOptions["getSlackAdapter"]; + nowMs?: () => number; + queue: ConversationWorkQueueTestAdapter; + resumeAwaitingContinuation?: SlackWorkerOptions["resumeAwaitingContinuation"]; + runtime: SlackWorkerOptions["runtime"]; + state: StateAdapter; +} + +function processNextQueuedSlackWork(args: ProcessQueuedSlackWorkArgs) { + return processConversationQueueMessage(args.queue.takeMessage(), { + nowMs: args.nowMs, + queue: args.queue, + run: createSlackConversationWorker({ + getSlackAdapter: args.getSlackAdapter, + resumeAwaitingContinuation: args.resumeAwaitingContinuation, + runtime: args.runtime, + state: args.state, + }), + state: args.state, + }); +} + +/** Prove redundant queue deliveries do not replay already-drained Slack work. */ +async function expectRemainingQueuedSlackWorkIsNoop( + args: ProcessQueuedSlackWorkArgs, +): Promise { + while (args.queue.hasQueuedMessages()) { + await expect(processNextQueuedSlackWork(args)).resolves.toEqual({ + status: "no_work", + }); + } +} + describe("Slack conversation work execution", () => { beforeEach(async () => { await disconnectStateAdapter(); @@ -39,7 +77,7 @@ describe("Slack conversation work execution", () => { }); it("persists Slack mentions into the durable mailbox and wakes the queue", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -59,11 +97,14 @@ describe("Slack conversation work execution", () => { }); expect(response.status).toBe(200); - expect(queue.sent).toEqual([ + expect(queue.sentRecords()).toEqual([ expect.objectContaining({ conversationId: CONVERSATION_ID, }), ]); + expect(queue.queuedMessages()).toEqual([ + { conversationId: CONVERSATION_ID }, + ]); const work = await getConversationWorkState({ conversationId: CONVERSATION_ID, state, @@ -85,7 +126,7 @@ describe("Slack conversation work execution", () => { }); it("routes edited Slack mentions through the durable mailbox", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -127,7 +168,7 @@ describe("Slack conversation work execution", () => { }); expect(response.status).toBe(200); - expect(queue.sent).toEqual([ + expect(queue.sentRecords()).toEqual([ expect.objectContaining({ conversationId: `slack:C123:${editedTs}`, idempotencyKey: `slack:T123:slack:C123:${editedTs}:${editedTs}:message_changed_mention`, @@ -136,22 +177,19 @@ describe("Slack conversation work execution", () => { const calls: Array<{ message: Message; thread: Thread }> = []; await expect( - processConversationWork(`slack:C123:${editedTs}`, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (thread, message, hooks) => { - await hooks?.onTurnStatePersisted?.(); - calls.push({ thread, message }); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, + runtime: { + handleNewMention: async (thread, message, hooks) => { + await hooks?.onInputCommitted?.(); + calls.push({ thread, message }); }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, }), ).resolves.toEqual({ status: "completed" }); @@ -163,7 +201,7 @@ describe("Slack conversation work execution", () => { }); it("runs queued Slack mailbox work through the Slack runtime", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -203,27 +241,25 @@ describe("Slack conversation work execution", () => { }, }); + const runtime: SlackWorkerOptions["runtime"] = { + handleNewMention: async (thread, message, hooks) => { + await hooks?.onInputCommitted?.(); + calls.push({ + thread, + message, + skipped: hooks?.messageContext?.skipped ?? [], + }); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }; await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, queue, + runtime, state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (thread, message, hooks) => { - await hooks?.onTurnStatePersisted?.(); - calls.push({ - thread, - message, - skipped: hooks?.messageContext?.skipped ?? [], - }); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }), }), ).resolves.toEqual({ status: "completed" }); @@ -239,10 +275,16 @@ describe("Slack conversation work execution", () => { state, }); expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + await expectRemainingQueuedSlackWorkIsNoop({ + getSlackAdapter: () => slackAdapter, + queue, + runtime, + state, + }); }); it("keeps restored thread context aligned with promoted mention routing", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -292,30 +334,26 @@ describe("Slack conversation work execution", () => { ]); await state.unsubscribe(CONVERSATION_ID); + const runtime: SlackWorkerOptions["runtime"] = { + handleNewMention: async (thread, message, hooks) => { + await hooks?.onInputCommitted?.(); + subscribedValues.push(await thread.isSubscribed()); + calls.push({ + thread, + message, + skipped: hooks?.messageContext?.skipped ?? [], + }); + }, + handleSubscribedMessage: async () => { + throw new Error("mixed mention batches should promote to mention"); + }, + }; await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, queue, + runtime, state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (thread, message, hooks) => { - await hooks?.onTurnStatePersisted?.(); - subscribedValues.push(await thread.isSubscribed()); - calls.push({ - thread, - message, - skipped: hooks?.messageContext?.skipped ?? [], - }); - }, - handleSubscribedMessage: async () => { - throw new Error( - "mixed mention batches should promote to mention", - ); - }, - }, - state, - }), }), ).resolves.toEqual({ status: "completed" }); @@ -325,10 +363,16 @@ describe("Slack conversation work execution", () => { "1712345.0001", ]); expect(subscribedValues).toEqual([false]); + await expectRemainingQueuedSlackWorkIsNoop({ + getSlackAdapter: () => slackAdapter, + queue, + runtime, + state, + }); }); it("processes pending Slack follow-ups when no continuation starts", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -352,23 +396,20 @@ describe("Slack conversation work execution", () => { const calls: string[] = []; await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - resumeAwaitingContinuation, - runtime: { - handleNewMention: async (_thread, message, hooks) => { - await hooks?.onTurnStatePersisted?.(); - calls.push(message.text); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, + resumeAwaitingContinuation, + runtime: { + handleNewMention: async (_thread, message, hooks) => { + await hooks?.onInputCommitted?.(); + calls.push(message.text); }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, }), ).resolves.toEqual({ status: "completed" }); @@ -382,7 +423,7 @@ describe("Slack conversation work execution", () => { }); it("resumes awaiting continuations before routing pending Slack follow-ups", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -403,31 +444,28 @@ describe("Slack conversation work execution", () => { state, }, }); - queue.sent = []; + queue.clearSentRecords(); await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, nowMs: () => 3_500, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - resumeAwaitingContinuation, - runtime: { - handleNewMention: async () => { - throw new Error("pending follow-up should wait for resume"); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, + resumeAwaitingContinuation, + runtime: { + handleNewMention: async () => { + throw new Error("pending follow-up should wait for resume"); }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, }), ).resolves.toEqual({ status: "pending_requeued" }); expect(resumeAwaitingContinuation).toHaveBeenCalledWith(CONVERSATION_ID); - expect(queue.sent).toEqual([ + expect(queue.sentRecords()).toEqual([ expect.objectContaining({ conversationId: CONVERSATION_ID, idempotencyKey: `pending:${CONVERSATION_ID}:3500`, @@ -444,7 +482,7 @@ describe("Slack conversation work execution", () => { }); it("drains Slack messages that arrive during an active turn into steering", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -466,37 +504,35 @@ describe("Slack conversation work execution", () => { const injected: string[][] = []; const drained: string[][] = []; + const runtime: SlackWorkerOptions["runtime"] = { + handleNewMention: async (_thread, _message, hooks) => { + await hooks?.onInputCommitted?.(); + await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ + text: `<@${SLACK_BOT_USER_ID}> steer this`, + ts: "1712345.0002", + threadTs: "1712345.0001", + }), + ), + services: ingressServices, + }); + const messages = + (await hooks?.drainSteeringMessages?.(async (steering) => { + injected.push(steering.map((message) => message.id)); + })) ?? []; + drained.push(messages.map((message) => message.id)); + }, + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }; await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, queue, + runtime, state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (_thread, _message, hooks) => { - await hooks?.onTurnStatePersisted?.(); - await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ - text: `<@${SLACK_BOT_USER_ID}> steer this`, - ts: "1712345.0002", - threadTs: "1712345.0001", - }), - ), - services: ingressServices, - }); - const messages = - (await hooks?.drainSteeringMessages?.(async (steering) => { - injected.push(steering.map((message) => message.id)); - })) ?? []; - drained.push(messages.map((message) => message.id)); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, - }, - state, - }), }), ).resolves.toEqual({ status: "completed" }); @@ -511,10 +547,16 @@ describe("Slack conversation work execution", () => { expect.any(Number), ]); expect(work ? countPendingConversationMessages(work) : 0).toBe(0); + await expectRemainingQueuedSlackWorkIsNoop({ + getSlackAdapter: () => slackAdapter, + queue, + runtime, + state, + }); }); it("does not replay injected Slack mailbox records after lease recovery", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -561,21 +603,18 @@ describe("Slack conversation work execution", () => { }); await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async () => { - throw new Error("injected messages should not replay"); - }, - handleSubscribedMessage: async () => { - throw new Error("injected messages should not replay"); - }, + runtime: { + handleNewMention: async () => { + throw new Error("injected messages should not replay"); }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("injected messages should not replay"); + }, + }, + state, }), ).resolves.toEqual({ status: "completed" }); @@ -588,7 +627,7 @@ describe("Slack conversation work execution", () => { }); it("terminalizes invalid idle continuation metadata", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -643,7 +682,7 @@ describe("Slack conversation work execution", () => { }); it("terminalizes stale idle continuations skipped by resume startup", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -737,8 +776,8 @@ describe("Slack conversation work execution", () => { }); }); - it("keeps Slack mailbox records pending when the runtime handoff fails", async () => { - const queue = createFakeQueue(); + it("keeps Slack mailbox records pending when input commit fails", async () => { + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -758,23 +797,20 @@ describe("Slack conversation work execution", () => { }); await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async () => { - throw new Error("runtime failed before durable handoff"); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, + runtime: { + handleNewMention: async () => { + throw new Error("runtime failed before input commit"); }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, }), - ).rejects.toThrow("runtime failed before durable handoff"); + ).rejects.toThrow("runtime failed before input commit"); const work = await getConversationWorkState({ conversationId: CONVERSATION_ID, @@ -785,8 +821,8 @@ describe("Slack conversation work execution", () => { expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); }); - it("requeues Slack mailbox records when the runtime returns without durable handoff", async () => { - const queue = createFakeQueue(); + it("requeues Slack mailbox records when the runtime returns without input commit", async () => { + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -804,31 +840,28 @@ describe("Slack conversation work execution", () => { state, }, }); - queue.sent = []; + queue.clearSentRecords(); let handled = 0; await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, nowMs: () => 3_000, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async () => { - handled += 1; - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, + runtime: { + handleNewMention: async () => { + handled += 1; }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, }), ).resolves.toEqual({ status: "pending_requeued" }); expect(handled).toBe(1); - expect(queue.sent).toEqual([ + expect(queue.sentRecords()).toEqual([ expect.objectContaining({ conversationId: CONVERSATION_ID, idempotencyKey: `pending:${CONVERSATION_ID}:3000`, @@ -844,8 +877,8 @@ describe("Slack conversation work execution", () => { expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); }); - it("reports lost lease when Slack turn handoff loses the mailbox lease", async () => { - const queue = createFakeQueue(); + it("reports lost lease when input commit loses the mailbox lease", async () => { + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -864,35 +897,32 @@ describe("Slack conversation work execution", () => { state, }, }); - queue.sent = []; + queue.clearSentRecords(); await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, nowMs: () => currentNowMs, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (_thread, _message, hooks) => { - currentNowMs = 1_000 + CONVERSATION_WORK_LEASE_TTL_MS + 1; - await recoverConversationWork({ - nowMs: currentNowMs, - queue, - state, - }); - await hooks?.onTurnStatePersisted?.(); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, + runtime: { + handleNewMention: async (_thread, _message, hooks) => { + currentNowMs = 1_000 + CONVERSATION_WORK_LEASE_TTL_MS + 1; + await recoverConversationWork({ + nowMs: currentNowMs, + queue, + state, + }); + await hooks?.onInputCommitted?.(); }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, }), ).resolves.toEqual({ status: "lost_lease" }); - expect(queue.sent).toEqual([ + expect(queue.sentRecords()).toEqual([ expect.objectContaining({ conversationId: CONVERSATION_ID, idempotencyKey: `heartbeat:lease:${CONVERSATION_ID}:${currentNowMs}`, @@ -909,7 +939,7 @@ describe("Slack conversation work execution", () => { }); it("completes Slack mailbox work when the handler finishes after the soft deadline", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -928,30 +958,27 @@ describe("Slack conversation work execution", () => { state, }, }); - queue.sent = []; + queue.clearSentRecords(); await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, nowMs: () => currentNowMs, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (_thread, _message, hooks) => { - currentNowMs = 242_000; - await hooks?.onTurnStatePersisted?.(); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, + runtime: { + handleNewMention: async (_thread, _message, hooks) => { + currentNowMs = 242_000; + await hooks?.onInputCommitted?.(); }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, }), ).resolves.toEqual({ status: "completed" }); - expect(queue.sent).toEqual([]); + expect(queue.sentRecords()).toEqual([]); const work = await getConversationWorkState({ conversationId: CONVERSATION_ID, state, @@ -961,7 +988,7 @@ describe("Slack conversation work execution", () => { }); it("yields Slack mailbox work after a persisted safe boundary", async () => { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const state = getStateAdapter(); await state.connect(); const slackAdapter = createSlackAdapterFixture(); @@ -980,31 +1007,28 @@ describe("Slack conversation work execution", () => { state, }, }); - queue.sent = []; + queue.clearSentRecords(); await expect( - processConversationWork(CONVERSATION_ID, { + processNextQueuedSlackWork({ + getSlackAdapter: () => slackAdapter, nowMs: () => currentNowMs, queue, - state, - run: createSlackConversationWorker({ - getSlackAdapter: () => slackAdapter, - runtime: { - handleNewMention: async (_thread, _message, hooks) => { - await hooks?.onTurnStatePersisted?.(); - currentNowMs = 242_000; - throw new CooperativeTurnYieldError(); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed route"); - }, + runtime: { + handleNewMention: async (_thread, _message, hooks) => { + await hooks?.onInputCommitted?.(); + currentNowMs = 242_000; + throw new CooperativeTurnYieldError(); }, - state, - }), + handleSubscribedMessage: async () => { + throw new Error("unexpected subscribed route"); + }, + }, + state, }), ).resolves.toEqual({ status: "yielded" }); - expect(queue.sent).toMatchObject([ + expect(queue.sentRecords()).toMatchObject([ { conversationId: CONVERSATION_ID, idempotencyKey: `yield:${CONVERSATION_ID}:242000`, diff --git a/packages/junior/tests/fixtures/conversation-work.ts b/packages/junior/tests/fixtures/conversation-work.ts index 62939bd95..c90b5e13e 100644 --- a/packages/junior/tests/fixtures/conversation-work.ts +++ b/packages/junior/tests/fixtures/conversation-work.ts @@ -1,5 +1,4 @@ -import { createHmac } from "node:crypto"; -import type { StateAdapter } from "chat"; +import type { Lock, StateAdapter } from "chat"; import type { ConversationQueueMessage, ConversationQueueSendOptions, @@ -8,43 +7,145 @@ import type { import type { InboundMessageRecord } from "@/chat/task-execution/store"; import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; -import type { WaitUntilFn } from "@/handlers/types"; +import { createSlackWebhookTestClient } from "./slack/webhook-client"; +import { createWaitUntilCollector } from "./wait-until"; export const CONVERSATION_ID = "slack:C123:1712345.0001"; export const SLACK_BOT_USER_ID = "U_BOT"; export const SLACK_SIGNING_SECRET = "slack-signature-fixture"; -/** Queue fixture state exposed for recovery and idempotency assertions. */ -export interface FakeConversationWorkQueue extends ConversationWorkQueue { - fail: boolean; - sent: Array<{ - conversationId: string; - delayMs?: number; - idempotencyKey?: string; - }>; +export interface ConversationQueueSendRecord { + conversationId: string; + delayMs?: number; + idempotencyKey?: string; } -/** Create a queue port fixture that records durable wake-up sends. */ -export function createFakeQueue(): FakeConversationWorkQueue { - const queue: FakeConversationWorkQueue = { - fail: false, - sent: [], - async send( - message: ConversationQueueMessage, - options?: ConversationQueueSendOptions, - ): Promise<{ messageId: string }> { - if (queue.fail) { - throw new Error("queue unavailable"); - } - queue.sent.push({ - conversationId: message.conversationId, - delayMs: options?.delayMs, - idempotencyKey: options?.idempotencyKey, - }); - return { messageId: `queue-${queue.sent.length}` }; - }, +interface QueueSendHold { + entered: () => void; + release: Promise; +} + +/** + * In-memory queue adapter for tests that need queue delivery plus send introspection. + * + * `send` behaves like the production queue handoff: it records send metadata and + * makes the payload available for callback-style delivery through `takeMessage`. + */ +export class ConversationWorkQueueTestAdapter implements ConversationWorkQueue { + #queuedMessages: ConversationQueueMessage[] = []; + #rejectSends = false; + #sendHolds: QueueSendHold[] = []; + #sentRecords: ConversationQueueSendRecord[] = []; + + allowSends(): void { + this.#rejectSends = false; + } + + clearSentRecords(): void { + this.#sentRecords = []; + } + + hasQueuedMessages(): boolean { + return this.#queuedMessages.length > 0; + } + + queuedMessages(): ConversationQueueMessage[] { + return this.#queuedMessages.map((message) => ({ ...message })); + } + + rejectSends(): void { + this.#rejectSends = true; + } + + sentRecords(): ConversationQueueSendRecord[] { + return this.#sentRecords.map((record) => ({ ...record })); + } + + /** Hold the next send open after it records the queued payload. */ + holdNextSendUntil(release: Promise): Promise { + return new Promise((entered) => { + this.#sendHolds.push({ entered, release }); + }); + } + + async send( + message: ConversationQueueMessage, + options?: ConversationQueueSendOptions, + ): Promise<{ messageId: string }> { + if (this.#rejectSends) { + throw new Error("queue unavailable"); + } + this.#queuedMessages.push({ ...message }); + const record: ConversationQueueSendRecord = { + conversationId: message.conversationId, + }; + if (options?.delayMs !== undefined) { + record.delayMs = options.delayMs; + } + if (options?.idempotencyKey !== undefined) { + record.idempotencyKey = options.idempotencyKey; + } + this.#sentRecords.push(record); + const hold = this.#sendHolds.shift(); + if (hold) { + hold.entered(); + await hold.release; + } + return { messageId: `queue-${this.#sentRecords.length}` }; + } + + takeMessage(): ConversationQueueMessage { + const message = this.#queuedMessages.shift(); + if (!message) { + throw new Error("Expected queued conversation work payload"); + } + return message; + } +} + +/** Create a durable queue adapter for component and integration tests. */ +export function createConversationWorkQueueTestAdapter(): ConversationWorkQueueTestAdapter { + return new ConversationWorkQueueTestAdapter(); +} + +/** Observe whether one conversation's mutation lock is currently held. */ +export function observeConversationMutationLock(args: { + conversationId: string; + state: StateAdapter; +}): { isHeld: () => boolean; state: StateAdapter } { + const mutationLockKey = `junior:conversation-work:mutation:${args.conversationId}`; + const locks = new WeakSet(); + let held = false; + return { + isHeld: () => held, + state: new Proxy(args.state, { + get(target, prop, receiver) { + if (prop === "acquireLock") { + return async (key: string, ttlMs: number) => { + const lock = await target.acquireLock(key, ttlMs); + if (lock && key === mutationLockKey) { + locks.add(lock); + held = true; + } + return lock; + }; + } + if (prop === "releaseLock") { + return async (lock: Lock) => { + try { + return await target.releaseLock(lock); + } finally { + if (locks.delete(lock)) { + held = false; + } + } + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as StateAdapter, }; - return queue; } /** Build a durable mailbox record for component-level conversation work tests. */ @@ -109,25 +210,11 @@ export function delayMutationLockUntil(args: { }) as StateAdapter; } -function signSlackBody(body: string, timestamp: string): string { - return `v0=${createHmac("sha256", SLACK_SIGNING_SECRET) - .update(`v0:${timestamp}:${body}`) - .digest("hex")}`; -} - /** Build a signed Slack JSON webhook request for Slack ingress tests. */ export function slackWebhookRequest(body: unknown): Request { - const serialized = JSON.stringify(body); - const timestamp = String(Math.floor(Date.now() / 1000)); - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(serialized, timestamp), - }, - body: serialized, - }); + return createSlackWebhookTestClient({ + signingSecret: SLACK_SIGNING_SECRET, + }).event(body); } /** Build the minimal Slack Events API envelope used by durable ingress tests. */ @@ -168,30 +255,16 @@ export function deferred(): { return { promise, resolve }; } -type WaitUntilTask = () => Promise; - -function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task : () => task); - }; -} - -async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { - for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]?.(); - } -} - /** Run Slack webhook ingress and flush every scheduled waitUntil task. */ export async function handleSlackWebhookAndFlush( args: Omit[0], "waitUntil">, ): Promise { - const waitUntilTasks: WaitUntilTask[] = []; + const waitUntil = createWaitUntilCollector(); const response = await handleSlackWebhook({ ...args, - waitUntil: collectWaitUntil(waitUntilTasks), + waitUntil: waitUntil.fn, }); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); return response; } diff --git a/packages/junior/tests/fixtures/slack-api-outbox.ts b/packages/junior/tests/fixtures/slack-api-outbox.ts new file mode 100644 index 000000000..a50e84380 --- /dev/null +++ b/packages/junior/tests/fixtures/slack-api-outbox.ts @@ -0,0 +1,40 @@ +import { + getCapturedSlackApiCalls, + getCapturedSlackFileUploadCalls, + type CapturedSlackApiCall, + type CapturedSlackFileUploadCall, + type SlackApiMethod, +} from "../msw/handlers/slack-api"; + +/** Read-only outbox for Slack MSW calls captured during a test. */ +export class SlackApiOutbox { + calls(method?: SlackApiMethod): CapturedSlackApiCall[] { + return getCapturedSlackApiCalls(method); + } + + fileUploads(): CapturedSlackFileUploadCall[] { + return getCapturedSlackFileUploadCalls(); + } + + homeViews(): CapturedSlackApiCall[] { + return this.calls("views.publish"); + } + + messages(): CapturedSlackApiCall[] { + return this.calls("chat.postMessage"); + } + + reactionAdds(): CapturedSlackApiCall[] { + return this.calls("reactions.add"); + } + + reactionRemovals(): CapturedSlackApiCall[] { + return this.calls("reactions.remove"); + } + + reactions(): CapturedSlackApiCall[] { + return this.reactionAdds(); + } +} + +export const slackApiOutbox = new SlackApiOutbox(); diff --git a/packages/junior/tests/fixtures/slack/webhook-client.ts b/packages/junior/tests/fixtures/slack/webhook-client.ts new file mode 100644 index 000000000..bb707182f --- /dev/null +++ b/packages/junior/tests/fixtures/slack/webhook-client.ts @@ -0,0 +1,82 @@ +import { createHmac } from "node:crypto"; +import { + createWaitUntilCollector, + type WaitUntilCollector, +} from "../wait-until"; + +export interface SlackWebhookClientOptions { + signingSecret: string; +} + +/** Build signed Slack webhook requests with deterministic test credentials. */ +export class SlackWebhookTestClient { + readonly signingSecret: string; + + constructor(options: SlackWebhookClientOptions) { + this.signingSecret = options.signingSecret; + } + + event(payload: unknown): Request { + return this.json(payload); + } + + form(params: URLSearchParams): Request { + const body = params.toString(); + const timestamp = String(Math.floor(Date.now() / 1000)); + return this.request({ + body, + contentType: "application/x-www-form-urlencoded", + signature: this.sign(body, timestamp), + timestamp, + }); + } + + invalidSignature(payload: unknown): Request { + return this.json(payload, "v0=invalid"); + } + + json(payload: unknown, signature?: string): Request { + const body = JSON.stringify(payload); + const timestamp = String(Math.floor(Date.now() / 1000)); + return this.request({ + body, + contentType: "application/json", + signature: signature ?? this.sign(body, timestamp), + timestamp, + }); + } + + waitUntil(): WaitUntilCollector { + return createWaitUntilCollector(); + } + + private request(args: { + body: string; + contentType: string; + signature: string; + timestamp: string; + }): Request { + return new Request("https://example.test/api/webhooks/slack", { + method: "POST", + headers: { + "content-type": args.contentType, + "x-slack-request-timestamp": args.timestamp, + "x-slack-signature": args.signature, + }, + body: args.body, + }); + } + + private sign(body: string, timestamp: string): string { + return `v0=${createHmac("sha256", this.signingSecret) + .update(`v0:${timestamp}:${body}`) + .digest("hex")}`; + } +} + +/** Create a Slack webhook client for signed Events API and interaction tests. */ +export function createSlackWebhookTestClient( + options: SlackWebhookClientOptions, +): SlackWebhookTestClient { + return new SlackWebhookTestClient(options); +} diff --git a/packages/junior/tests/fixtures/turn-resume.ts b/packages/junior/tests/fixtures/turn-resume.ts new file mode 100644 index 000000000..9b241db69 --- /dev/null +++ b/packages/junior/tests/fixtures/turn-resume.ts @@ -0,0 +1,92 @@ +import { createHmac } from "node:crypto"; +import { + createWaitUntilCollector, + type WaitUntilCollector, +} from "./wait-until"; + +export interface TurnResumeTestClientOptions { + juniorSecret: string; +} + +export interface TurnResumeTestRequest { + conversationId: string; + expectedVersion: number; + sessionId: string; +} + +/** + * Build signed internal timeout-resume requests with deterministic test secrets. + */ +export class TurnResumeTestClient { + readonly juniorSecret: string; + + constructor(options: TurnResumeTestClientOptions) { + this.juniorSecret = options.juniorSecret; + } + + invalidSignature(payload: TurnResumeTestRequest): Request { + return this.buildRequest({ + body: JSON.stringify(payload), + signature: "v1=invalid", + timestamp: Date.now().toString(), + }); + } + + legacyRequest(payload: TurnResumeTestRequest): Request { + const body = JSON.stringify({ + conversationId: payload.conversationId, + expectedCheckpointVersion: payload.expectedVersion, + sessionId: payload.sessionId, + }); + const timestamp = Date.now().toString(); + return this.buildRequest({ + body, + signature: this.sign(body, timestamp), + timestamp, + }); + } + + request(payload: TurnResumeTestRequest): Request { + const body = JSON.stringify(payload); + const timestamp = Date.now().toString(); + return this.buildRequest({ + body, + signature: this.sign(body, timestamp), + timestamp, + }); + } + + waitUntil(): WaitUntilCollector { + return createWaitUntilCollector(); + } + + private buildRequest(args: { + body: string; + signature: string; + timestamp: string; + }): Request { + return new Request("https://junior.example.com/api/internal/turn-resume", { + method: "POST", + headers: { + "content-type": "application/json", + "x-junior-resume-signature": args.signature, + "x-junior-resume-timestamp": args.timestamp, + }, + body: args.body, + }); + } + + private sign(body: string, timestamp: string): string { + const digest = createHmac("sha256", this.juniorSecret) + .update(`junior.turn_timeout_resume.v1:${timestamp}:${body}`) + .digest("hex"); + return `v1=${digest}`; + } +} + +/** Create a signed timeout-resume request client for handler tests. */ +export function createTurnResumeTestClient( + options: TurnResumeTestClientOptions, +): TurnResumeTestClient { + return new TurnResumeTestClient(options); +} diff --git a/packages/junior/tests/fixtures/wait-until.ts b/packages/junior/tests/fixtures/wait-until.ts new file mode 100644 index 000000000..60765e76c --- /dev/null +++ b/packages/junior/tests/fixtures/wait-until.ts @@ -0,0 +1,25 @@ +import type { WaitUntilFn } from "@/handlers/types"; + +/** Collect waitUntil tasks so tests can assert and flush background work explicitly. */ +export class WaitUntilCollector { + #tasks: Promise[] = []; + + readonly fn: WaitUntilFn = (task) => { + this.#tasks.push(typeof task === "function" ? task() : task); + }; + + pendingCount(): number { + return this.#tasks.length; + } + + async flush(): Promise { + while (this.#tasks.length > 0) { + await this.#tasks.shift(); + } + } +} + +/** Create a waitUntil collector for handler and webhook tests. */ +export function createWaitUntilCollector(): WaitUntilCollector { + return new WaitUntilCollector(); +} diff --git a/packages/junior/tests/integration/heartbeat.test.ts b/packages/junior/tests/integration/heartbeat.test.ts index 26e2d6514..5c248a394 100644 --- a/packages/junior/tests/integration/heartbeat.test.ts +++ b/packages/junior/tests/integration/heartbeat.test.ts @@ -24,10 +24,14 @@ import { getConversationWorkState } from "@/chat/task-execution/store"; import type { PiMessage } from "@/chat/pi/messages"; import { setAgentPlugins } from "@/chat/plugins/agent-hooks"; import { GET as heartbeat } from "@/handlers/heartbeat"; -import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; -import type { WaitUntilFn } from "@/handlers/types"; import { createSlackDirectCredentialSubject } from "@/chat/credentials/subject"; -import { getCapturedSlackApiCalls } from "../msw/handlers/slack-api"; +import { createConversationWorkQueueTestAdapter } from "../fixtures/conversation-work"; +import { conversationsInfoOk } from "../fixtures/slack/factories/api"; +import { createWaitUntilCollector } from "../fixtures/wait-until"; +import { + getCapturedSlackApiCalls, + queueSlackApiResponse, +} from "../msw/handlers/slack-api"; vi.hoisted(() => { process.env.JUNIOR_STATE_ADAPTER = "memory"; @@ -36,27 +40,6 @@ vi.hoisted(() => { const TEST_NOW_MS = Date.parse("2026-05-26T12:05:00.000Z"); const TEST_RUN_AT_MS = Date.parse("2026-05-26T12:00:00.000Z"); -function collectWaitUntil(tasks: Promise[]): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task() : task); - }; -} - -class FakeConversationWorkQueue implements ConversationWorkQueue { - sent: Array<{ conversationId: string; idempotencyKey?: string }> = []; - - async send( - message: { conversationId: string }, - options?: { idempotencyKey?: string }, - ): Promise<{ messageId: string }> { - this.sent.push({ - conversationId: message.conversationId, - idempotencyKey: options?.idempotencyKey, - }); - return { messageId: `queue-${this.sent.length}` }; - } -} - function schedulerStore() { return createSchedulerStore(createPluginState("scheduler")); } @@ -203,14 +186,14 @@ describe("trusted plugin heartbeat", () => { }); it("rejects unauthenticated heartbeat requests", async () => { - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat"), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, ); expect(response.status).toBe(401); - expect(waitUntilTasks).toHaveLength(0); + expect(waitUntil.pendingCount()).toBe(0); }); it("runs trusted plugin heartbeat hooks", async () => { @@ -228,21 +211,21 @@ describe("trusted plugin heartbeat", () => { }, }), ]); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); + await waitUntil.flush(); expect(seen).toHaveLength(1); }); it("reschedules stale timeout resume records", async () => { - const queue = new FakeConversationWorkQueue(); + const queue = createConversationWorkQueueTestAdapter(); const conversationId = "slack:C123:1712345.0001"; const sessionId = "turn-timeout"; const staleNowMs = TEST_NOW_MS - 3 * 60 * 1000; @@ -264,18 +247,18 @@ describe("trusted plugin heartbeat", () => { await persistActiveTurn(conversationId, sessionId); vi.setSystemTime(TEST_NOW_MS); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, { conversationWorkQueue: queue }, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); - expect(queue.sent).toEqual([ + await waitUntil.flush(); + expect(queue.sentRecords()).toEqual([ { conversationId, idempotencyKey: expect.stringContaining( @@ -292,7 +275,7 @@ describe("trusted plugin heartbeat", () => { }); it("reschedules stale cooperative yield resume records", async () => { - const queue = new FakeConversationWorkQueue(); + const queue = createConversationWorkQueueTestAdapter(); const conversationId = "slack:C123:1712345.0008"; const sessionId = "turn-yield"; const staleNowMs = TEST_NOW_MS - 3 * 60 * 1000; @@ -314,18 +297,18 @@ describe("trusted plugin heartbeat", () => { await persistActiveTurn(conversationId, sessionId); vi.setSystemTime(TEST_NOW_MS); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, { conversationWorkQueue: queue }, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); - expect(queue.sent).toEqual([ + await waitUntil.flush(); + expect(queue.sentRecords()).toEqual([ { conversationId, idempotencyKey: expect.stringContaining( @@ -342,7 +325,7 @@ describe("trusted plugin heartbeat", () => { }); it("skips stale timeout resume records for inactive turns", async () => { - const queue = new FakeConversationWorkQueue(); + const queue = createConversationWorkQueueTestAdapter(); const conversationId = "slack:C123:1712345.0007"; const sessionId = "turn-timeout-inactive"; const staleNowMs = TEST_NOW_MS - 3 * 60 * 1000; @@ -364,18 +347,18 @@ describe("trusted plugin heartbeat", () => { await persistActiveTurn(conversationId, "turn-newer"); vi.setSystemTime(TEST_NOW_MS); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, { conversationWorkQueue: queue }, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); - expect(queue.sent).toEqual([]); + await waitUntil.flush(); + expect(queue.sentRecords()).toEqual([]); await expect(getConversationWorkState({ conversationId })).resolves.toBe( undefined, ); @@ -727,15 +710,15 @@ describe("trusted plugin heartbeat", () => { const store = schedulerStore(); await store.saveTask(createTask()); - const firstWaitUntilTasks: Promise[] = []; + const firstWaitUntil = createWaitUntilCollector(); const firstResponse = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(firstWaitUntilTasks), + firstWaitUntil.fn, ); expect(firstResponse.status).toBe(202); - await Promise.all(firstWaitUntilTasks); + await firstWaitUntil.flush(); const running = await store.getRun(`sched_plugin_1:${TEST_RUN_AT_MS}`); expect(running).toMatchObject({ @@ -758,15 +741,15 @@ describe("trusted plugin heartbeat", () => { }); }); - const secondWaitUntilTasks: Promise[] = []; + const secondWaitUntil = createWaitUntilCollector(); const secondResponse = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(secondWaitUntilTasks), + secondWaitUntil.fn, ); expect(secondResponse.status).toBe(202); - await Promise.all(secondWaitUntilTasks); + await secondWaitUntil.flush(); await expect(store.getRun(running!.id)).resolves.toMatchObject({ status: "completed", @@ -797,15 +780,15 @@ describe("trusted plugin heartbeat", () => { }), ); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); + await waitUntil.flush(); const running = await store.getRun(`sched_plugin_1:${TEST_RUN_AT_MS}`); expect(running?.dispatchId).toEqual(expect.any(String)); @@ -836,15 +819,15 @@ describe("trusted plugin heartbeat", () => { const store = schedulerStore(); await store.saveTask(createTask()); - const firstWaitUntilTasks: Promise[] = []; + const firstWaitUntil = createWaitUntilCollector(); const firstResponse = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(firstWaitUntilTasks), + firstWaitUntil.fn, ); expect(firstResponse.status).toBe(202); - await Promise.all(firstWaitUntilTasks); + await firstWaitUntil.flush(); const running = await store.getRun(`sched_plugin_1:${TEST_RUN_AT_MS}`); expect(running).toMatchObject({ @@ -855,15 +838,15 @@ describe("trusted plugin heartbeat", () => { await state.connect(); await state.delete(getDispatchStorageKey(running!.dispatchId!)); - const secondWaitUntilTasks: Promise[] = []; + const secondWaitUntil = createWaitUntilCollector(); const secondResponse = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(secondWaitUntilTasks), + secondWaitUntil.fn, ); expect(secondResponse.status).toBe(202); - await Promise.all(secondWaitUntilTasks); + await secondWaitUntil.flush(); await expect(store.getRun(running!.id)).resolves.toMatchObject({ status: "failed", @@ -889,15 +872,15 @@ describe("trusted plugin heartbeat", () => { } as unknown as ScheduledTask["task"], }); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); + await waitUntil.flush(); await expect( store.getRun(`sched_plugin_malformed:${TEST_RUN_AT_MS}`), @@ -935,15 +918,15 @@ describe("trusted plugin heartbeat", () => { }, }); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); + await waitUntil.flush(); await expect( store.getRun(`sched_plugin_bad_destination:${TEST_RUN_AT_MS}`), @@ -974,15 +957,15 @@ describe("trusted plugin heartbeat", () => { const task = createDailyTask(); await store.saveTask(task); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); + await waitUntil.flush(); await expect( store.getRun(`${task.id}:${task.nextRunAtMs}`), @@ -1015,15 +998,15 @@ describe("trusted plugin heartbeat", () => { await store.saveTask(first); await store.saveTask(duplicate); - const waitUntilTasks: Promise[] = []; + const waitUntil = createWaitUntilCollector(); const response = await heartbeat( new Request("https://example.invalid/api/internal/heartbeat", { headers: { authorization: "Bearer heartbeat-secret" }, }), - collectWaitUntil(waitUntilTasks), + waitUntil.fn, ); expect(response.status).toBe(202); - await Promise.all(waitUntilTasks); + await waitUntil.flush(); await expect( store.getRun(`${duplicate.id}:${duplicate.nextRunAtMs}`), diff --git a/packages/junior/tests/integration/oauth-callback-slack.test.ts b/packages/junior/tests/integration/oauth-callback-slack.test.ts index 70c92a6df..246052520 100644 --- a/packages/junior/tests/integration/oauth-callback-slack.test.ts +++ b/packages/junior/tests/integration/oauth-callback-slack.test.ts @@ -91,7 +91,7 @@ describe("oauth callback slack integration", () => { }), }), ]); - }); + }, 20_000); it("resumes a pending OAuth request with persisted thread context", async () => { await stateAdapterModule @@ -165,7 +165,7 @@ describe("oauth callback slack integration", () => { }), ]), ); - }); + }, 20_000); it("resumes a session-recorded OAuth turn with persisted thread state", async () => { const conversationId = "slack:C123:1700000000.009"; diff --git a/packages/junior/tests/integration/slack/app-home-webhook.test.ts b/packages/junior/tests/integration/slack/app-home-webhook.test.ts index 07f6c125b..529b358ee 100644 --- a/packages/junior/tests/integration/slack/app-home-webhook.test.ts +++ b/packages/junior/tests/integration/slack/app-home-webhook.test.ts @@ -1,4 +1,3 @@ -import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createMemoryState } from "@chat-adapter/state-memory"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; @@ -6,48 +5,27 @@ import type { UserTokenStore } from "@/chat/credentials/user-token-store"; import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; import { getWorkspaceTeamId } from "@/chat/slack/workspace-context"; import { disconnectStateAdapter } from "@/chat/state/adapter"; -import type { WaitUntilFn } from "@/handlers/types"; import { - getCapturedSlackApiCalls, queueSlackApiError, resetSlackApiMockState, } from "../../msw/handlers/slack-api"; +import { + createConversationWorkQueueTestAdapter, + createNoopSlackWebhookRuntime, + deferred, +} from "../../fixtures/conversation-work"; +import { slackApiOutbox } from "../../fixtures/slack-api-outbox"; +import { createSlackWebhookTestClient } from "../../fixtures/slack/webhook-client"; const SIGNING_SECRET = "test-signing-secret"; const BOT_USER_ID = "U_BOT"; const ORIGINAL_ENV = { ...process.env }; -function signSlackBody(body: string, timestamp: string): string { - return `v0=${createHmac("sha256", SIGNING_SECRET) - .update(`v0:${timestamp}:${body}`) - .digest("hex")}`; -} - -function slackWebhookRequest(body: unknown): Request { - const serialized = JSON.stringify(body); - const timestamp = String(Math.floor(Date.now() / 1000)); - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(serialized, timestamp), - }, - body: serialized, - }); -} - -function slackFormRequest(params: URLSearchParams): Request { - const serialized = params.toString(); - const timestamp = String(Math.floor(Date.now() / 1000)); - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/x-www-form-urlencoded", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(serialized, timestamp), - }, - body: serialized, +function createSlackAdapter() { + return createJuniorSlackAdapter({ + botToken: "xoxb-test-token", + botUserId: BOT_USER_ID, + signingSecret: SIGNING_SECRET, }); } @@ -80,31 +58,6 @@ function createTokenStore( }; } -type WaitUntilTask = Promise; - -function collectWaitUntil(tasks: WaitUntilTask[]): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task() : task); - }; -} - -async function flushWaitUntil(tasks: WaitUntilTask[]): Promise { - for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]; - } -} - -function deferred(): { - promise: Promise; - resolve(value: T): void; -} { - let resolve!: (value: T) => void; - const promise = new Promise((resolvePromise) => { - resolve = resolvePromise; - }); - return { promise, resolve }; -} - describe("Slack webhook: App Home events", () => { beforeEach(() => { process.env = { @@ -127,15 +80,15 @@ describe("Slack webhook: App Home events", () => { }); const state = createMemoryState(); - const waitUntilTasks: WaitUntilTask[] = []; - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test-token", - botUserId: BOT_USER_ID, + const client = createSlackWebhookTestClient({ signingSecret: SIGNING_SECRET, }); + const queue = createConversationWorkQueueTestAdapter(); + const slackAdapter = createSlackAdapter(); + const waitUntil = client.waitUntil(); const response = await handleSlackWebhook({ - request: slackWebhookRequest({ + request: client.event({ team_id: "T123", type: "event_callback", event: { @@ -144,52 +97,36 @@ describe("Slack webhook: App Home events", () => { event_ts: "1712345.0001", }, }), - waitUntil: collectWaitUntil(waitUntilTasks), + waitUntil: waitUntil.fn, services: { getSlackAdapter: () => slackAdapter, - queue: { - send: async () => { - throw new Error("app_home_opened should not enqueue work"); - }, - }, - runtime: { - handleAssistantContextChanged: async () => { - throw new Error("unexpected assistant context callback"); - }, - handleAssistantThreadStarted: async () => { - throw new Error("unexpected assistant thread callback"); - }, - handleNewMention: async () => { - throw new Error("unexpected mention callback"); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed message callback"); - }, - }, + queue, + runtime: createNoopSlackWebhookRuntime(), state, }, }); expect(response.status).toBe(200); - expect(waitUntilTasks).toHaveLength(1); - await flushWaitUntil(waitUntilTasks); - expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(1); + expect(queue.sentRecords()).toEqual([]); + expect(waitUntil.pendingCount()).toBe(1); + await waitUntil.flush(); + expect(slackApiOutbox.homeViews()).toHaveLength(1); }); - it("acknowledges message events before durable handoff work finishes", async () => { + it("acknowledges message events after durable handoff finishes", async () => { const state = createMemoryState(); - const waitUntilTasks: WaitUntilTask[] = []; - const queueMessages: Array<{ conversationId: string }> = []; - const queueSendEntered = deferred(); - const finishQueueSend = deferred(); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test-token", - botUserId: BOT_USER_ID, + const client = createSlackWebhookTestClient({ signingSecret: SIGNING_SECRET, }); + const waitUntil = client.waitUntil(); + const queue = createConversationWorkQueueTestAdapter(); + const finishQueueSend = deferred(); + let responseSettled = false; + const queueSendEntered = queue.holdNextSendUntil(finishQueueSend.promise); + const slackAdapter = createSlackAdapter(); - const response = await handleSlackWebhook({ - request: slackWebhookRequest({ + const responsePromise = handleSlackWebhook({ + request: client.event({ team_id: "T123", type: "event_callback", event: { @@ -202,58 +139,40 @@ describe("Slack webhook: App Home events", () => { channel_type: "channel", }, }), - waitUntil: collectWaitUntil(waitUntilTasks), + waitUntil: waitUntil.fn, services: { getSlackAdapter: () => slackAdapter, - queue: { - send: async (message) => { - queueMessages.push(message); - queueSendEntered.resolve(); - await finishQueueSend.promise; - return { messageId: "queue-1" }; - }, - }, - runtime: { - handleAssistantContextChanged: async () => { - throw new Error("unexpected assistant context callback"); - }, - handleAssistantThreadStarted: async () => { - throw new Error("unexpected assistant thread callback"); - }, - handleNewMention: async () => { - throw new Error("unexpected mention callback"); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed message callback"); - }, - }, + queue, + runtime: createNoopSlackWebhookRuntime(), state, }, + }).then((response) => { + responseSettled = true; + return response; }); - expect(response.status).toBe(200); - expect(waitUntilTasks).toHaveLength(1); - await queueSendEntered.promise; - expect(queueMessages).toEqual([ + await queueSendEntered; + expect(responseSettled).toBe(false); + expect(queue.queuedMessages()).toEqual([ { conversationId: "slack:C123:1712345.0001" }, ]); + expect(waitUntil.pendingCount()).toBe(0); finishQueueSend.resolve(); - await flushWaitUntil(waitUntilTasks); + await expect(responsePromise).resolves.toMatchObject({ status: 200 }); }); it("routes explicit mentions from other Slack bots", async () => { const state = createMemoryState(); - const waitUntilTasks: WaitUntilTask[] = []; - const queueMessages: Array<{ conversationId: string }> = []; - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test-token", - botUserId: BOT_USER_ID, + const client = createSlackWebhookTestClient({ signingSecret: SIGNING_SECRET, }); + const waitUntil = client.waitUntil(); + const queue = createConversationWorkQueueTestAdapter(); + const slackAdapter = createSlackAdapter(); const response = await handleSlackWebhook({ - request: slackWebhookRequest({ + request: client.event({ team_id: "T123", type: "event_callback", event: { @@ -268,93 +187,58 @@ describe("Slack webhook: App Home events", () => { channel_type: "channel", }, }), - waitUntil: collectWaitUntil(waitUntilTasks), + waitUntil: waitUntil.fn, services: { getSlackAdapter: () => slackAdapter, - queue: { - send: async (message) => { - queueMessages.push(message); - return { messageId: "queue-1" }; - }, - }, - runtime: { - handleAssistantContextChanged: async () => { - throw new Error("unexpected assistant context callback"); - }, - handleAssistantThreadStarted: async () => { - throw new Error("unexpected assistant thread callback"); - }, - handleNewMention: async () => { - throw new Error("unexpected mention callback"); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed message callback"); - }, - }, + queue, + runtime: createNoopSlackWebhookRuntime(), state, }, }); expect(response.status).toBe(200); - expect(waitUntilTasks).toHaveLength(1); - await flushWaitUntil(waitUntilTasks); - expect(queueMessages).toEqual([ + expect(waitUntil.pendingCount()).toBe(0); + expect(queue.queuedMessages()).toEqual([ { conversationId: "slack:C123:1712345.0002" }, ]); }); it("refreshes App Home after disconnect unlink failure", async () => { const state = createMemoryState(); - const waitUntilTasks: WaitUntilTask[] = []; + const client = createSlackWebhookTestClient({ + signingSecret: SIGNING_SECRET, + }); + const waitUntil = client.waitUntil(); + const queue = createConversationWorkQueueTestAdapter(); const workspaceTeamIds: Array = []; const deleteToken = vi.fn(async () => { workspaceTeamIds.push(getWorkspaceTeamId()); throw new Error("token store unavailable"); }); const userTokenStore = createTokenStore({ delete: deleteToken }); - const slackAdapter = createJuniorSlackAdapter({ - botToken: "xoxb-test-token", - botUserId: BOT_USER_ID, - signingSecret: SIGNING_SECRET, - }); + const slackAdapter = createSlackAdapter(); const params = new URLSearchParams({ payload: JSON.stringify(interactiveDisconnectPayload()), }); const response = await handleSlackWebhook({ - request: slackFormRequest(params), - waitUntil: collectWaitUntil(waitUntilTasks), + request: client.form(params), + waitUntil: waitUntil.fn, services: { getSlackAdapter: () => slackAdapter, - queue: { - send: async () => { - throw new Error("interactive disconnect should not enqueue work"); - }, - }, - runtime: { - handleAssistantContextChanged: async () => { - throw new Error("unexpected assistant context callback"); - }, - handleAssistantThreadStarted: async () => { - throw new Error("unexpected assistant thread callback"); - }, - handleNewMention: async () => { - throw new Error("unexpected mention callback"); - }, - handleSubscribedMessage: async () => { - throw new Error("unexpected subscribed message callback"); - }, - }, + queue, + runtime: createNoopSlackWebhookRuntime(), state, getUserTokenStore: () => userTokenStore, }, }); expect(response.status).toBe(200); - expect(waitUntilTasks).toHaveLength(1); - await flushWaitUntil(waitUntilTasks); + expect(queue.sentRecords()).toEqual([]); + expect(waitUntil.pendingCount()).toBe(1); + await waitUntil.flush(); expect(deleteToken).toHaveBeenCalledWith("U123", "notion"); expect(workspaceTeamIds).toEqual(["T123"]); - expect(getCapturedSlackApiCalls("views.publish")).toHaveLength(1); + expect(slackApiOutbox.homeViews()).toHaveLength(1); }); }); diff --git a/packages/junior/tests/integration/slack/assistant-thread-contract.test.ts b/packages/junior/tests/integration/slack/assistant-thread-contract.test.ts index 71870db78..26868a1a6 100644 --- a/packages/junior/tests/integration/slack/assistant-thread-contract.test.ts +++ b/packages/junior/tests/integration/slack/assistant-thread-contract.test.ts @@ -1,19 +1,16 @@ -import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createMemoryState } from "@chat-adapter/state-memory"; import type { SlackAdapter } from "@chat-adapter/slack"; import { slackEventsApiEnvelope } from "../../fixtures/slack/factories/events"; -import { - getCapturedSlackApiCalls, - resetSlackApiMockState, -} from "../../msw/handlers/slack-api"; +import { resetSlackApiMockState } from "../../msw/handlers/slack-api"; +import { slackApiOutbox } from "../../fixtures/slack-api-outbox"; +import { createSlackWebhookTestClient } from "../../fixtures/slack/webhook-client"; import { createSlackRuntime } from "@/chat/app/factory"; import { JuniorChat } from "@/chat/ingress/junior-chat"; import { makeAssistantStatus } from "@/chat/slack/assistant-thread/status"; import type { ReplyExecutorServices } from "@/chat/runtime/reply-executor"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import type { ConversationMemoryDeps } from "@/chat/services/conversation-memory"; -import type { WaitUntilFn } from "@/handlers/types"; import { handlePlatformWebhook } from "@/handlers/webhooks"; const SIGNING_SECRET = "test-signing-secret"; @@ -22,17 +19,15 @@ const DM_CHANNEL_ID = "D12345"; const DM_THREAD_TS = "1700000000.000001"; const CHANNEL_ID = "C12345"; const CHANNEL_ROOT_TS = "1700000200.000200"; - -function signSlackBody(body: string, timestamp: string): string { - const base = `v0:${timestamp}:${body}`; - return `v0=${createHmac("sha256", SIGNING_SECRET).update(base).digest("hex")}`; -} +const slackWebhookClient = createSlackWebhookTestClient({ + signingSecret: SIGNING_SECRET, +}); function createDirectMessageRequest( text: string, options?: { threadTs?: string }, ): Request { - const body = JSON.stringify( + return slackWebhookClient.event( slackEventsApiEnvelope({ eventType: "message", channel: DM_CHANNEL_ID, @@ -41,24 +36,13 @@ function createDirectMessageRequest( ...(options?.threadTs ? { threadTs: options.threadTs } : {}), }), ); - const timestamp = String(Math.floor(Date.now() / 1000)); - - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(body, timestamp), - }, - body, - }); } function createChannelMentionRequest( text: string, options?: { threadTs?: string; ts?: string }, ): Request { - const body = JSON.stringify( + return slackWebhookClient.event( slackEventsApiEnvelope({ eventType: "app_mention", channel: CHANNEL_ID, @@ -67,29 +51,6 @@ function createChannelMentionRequest( ...(options?.threadTs ? { threadTs: options.threadTs } : {}), }), ); - const timestamp = String(Math.floor(Date.now() / 1000)); - - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(body, timestamp), - }, - body, - }); -} - -async function flushWaitUntil(tasks: Array>): Promise { - for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]; - } -} - -function collectWaitUntil(tasks: Array>): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task() : task); - }; } function makeDiagnostics() { @@ -192,19 +153,19 @@ describe("Slack contract: assistant-thread delivery", () => { }; }, }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const response = await handlePlatformWebhook( createDirectMessageRequest("run a command"), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); expect(response.status).toBe(200); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); - expect(getCapturedSlackApiCalls("assistant.threads.setStatus")).toEqual([]); + expect(slackApiOutbox.calls("assistant.threads.setStatus")).toEqual([]); }); it("posts assistant status with a raw DM channel id when thread_ts is present", async () => { @@ -217,21 +178,21 @@ describe("Slack contract: assistant-thread delivery", () => { }; }, }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const response = await handlePlatformWebhook( createDirectMessageRequest("run a command", { threadTs: DM_THREAD_TS, }), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); expect(response.status).toBe(200); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); - expect(getCapturedSlackApiCalls("assistant.threads.setStatus")).toEqual( + expect(slackApiOutbox.calls("assistant.threads.setStatus")).toEqual( expect.arrayContaining([ expect.objectContaining({ params: expect.objectContaining({ @@ -261,19 +222,19 @@ describe("Slack contract: assistant-thread delivery", () => { }; }, }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const response = await handlePlatformWebhook( createChannelMentionRequest("<@U_BOT> run a command"), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); expect(response.status).toBe(200); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); - expect(getCapturedSlackApiCalls("assistant.threads.setStatus")).toEqual( + expect(slackApiOutbox.calls("assistant.threads.setStatus")).toEqual( expect.arrayContaining([ expect.objectContaining({ params: expect.objectContaining({ @@ -305,22 +266,21 @@ describe("Slack contract: assistant-thread delivery", () => { diagnostics: makeDiagnostics(), }), }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const response = await handlePlatformWebhook( createDirectMessageRequest("How do I debug memory leaks in Node?", { threadTs: DM_THREAD_TS, }), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); expect(response.status).toBe(200); - await flushWaitUntil(waitUntilTasks); - await new Promise((resolve) => setTimeout(resolve, 0)); + await waitUntil.flush(); - expect(getCapturedSlackApiCalls("assistant.threads.setTitle")).toEqual([ + expect(slackApiOutbox.calls("assistant.threads.setTitle")).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel_id: DM_CHANNEL_ID, @@ -349,21 +309,21 @@ describe("Slack contract: assistant-thread delivery", () => { diagnostics: makeDiagnostics(), }), }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const response = await handlePlatformWebhook( createDirectMessageRequest("How do I debug memory leaks in Node?", { threadTs: DM_THREAD_TS, }), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); expect(response.status).toBe(200); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); - expect(getCapturedSlackApiCalls("assistant.threads.setTitle")).toEqual([ + expect(slackApiOutbox.calls("assistant.threads.setTitle")).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel_id: DM_CHANNEL_ID, @@ -386,19 +346,18 @@ describe("Slack contract: assistant-thread delivery", () => { diagnostics: makeDiagnostics(), }), }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const response = await handlePlatformWebhook( createDirectMessageRequest("How do I debug memory leaks in Node?"), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); expect(response.status).toBe(200); - await flushWaitUntil(waitUntilTasks); - await new Promise((resolve) => setTimeout(resolve, 0)); + await waitUntil.flush(); - expect(getCapturedSlackApiCalls("assistant.threads.setTitle")).toEqual([]); + expect(slackApiOutbox.calls("assistant.threads.setTitle")).toEqual([]); }); }); diff --git a/packages/junior/tests/integration/slack/attachment-media-behavior.test.ts b/packages/junior/tests/integration/slack/attachment-media-behavior.test.ts index ab1f7e76d..6060dc26a 100644 --- a/packages/junior/tests/integration/slack/attachment-media-behavior.test.ts +++ b/packages/junior/tests/integration/slack/attachment-media-behavior.test.ts @@ -147,7 +147,7 @@ describe("Slack behavior: mixed attachment media", () => { ["image/png", "application/pdf"], ]); expect(capturedAttachmentNames).toEqual([["chart.png", "incident.pdf"]]); - }, 10_000); + }, 20_000); it("drops image attachments when AI_VISION_MODEL is unset", async () => { const imageFetch = vi.fn(async () => Buffer.from("image-bytes")); diff --git a/packages/junior/tests/integration/slack/bot-image-hydration.test.ts b/packages/junior/tests/integration/slack/bot-image-hydration.test.ts index acb44d0aa..f2cf2fd0d 100644 --- a/packages/junior/tests/integration/slack/bot-image-hydration.test.ts +++ b/packages/junior/tests/integration/slack/bot-image-hydration.test.ts @@ -165,7 +165,7 @@ describe("bot image hydration", () => { ); expect(listThreadRepliesMock).toHaveBeenCalledTimes(1); - }, 10_000); + }, 20_000); it("does not hydrate thread images when AI_VISION_MODEL is unset", async () => { const { slackRuntime } = await createRuntime({ @@ -266,7 +266,7 @@ describe("bot image hydration", () => { slackTs: "1700000001.200", }, }); - }); + }, 20_000); it("backfills older image messages after vision is enabled later", async () => { const firstRuntime = await createRuntime({ diff --git a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts index 9e4b16ee3..bd526039b 100644 --- a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts +++ b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts @@ -3,24 +3,25 @@ import type { StateAdapter } from "chat"; import { SLACK_BOT_USER_ID, SLACK_SIGNING_SECRET, - createFakeQueue, + createConversationWorkQueueTestAdapter, deferred, handleSlackWebhookAndFlush, slackEnvelope, slackWebhookRequest, } from "../../fixtures/conversation-work"; -import { - getCapturedSlackApiCalls, - resetSlackApiMockState, -} from "../../msw/handlers/slack-api"; +import { slackApiOutbox } from "../../fixtures/slack-api-outbox"; +import { resetSlackApiMockState } from "../../msw/handlers/slack-api"; import { createSlackRuntime } from "@/chat/app/factory"; import type { ReplyExecutorServices } from "@/chat/runtime/reply-executor"; import type { ReplySteeringMessage } from "@/chat/respond"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { createSlackConversationWorker } from "@/chat/task-execution/slack-work"; -import { getConversationWorkState } from "@/chat/task-execution/store"; -import { processConversationWork } from "@/chat/task-execution/worker"; +import { + countPendingConversationMessages, + getConversationWorkState, +} from "@/chat/task-execution/store"; +import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; const CHANNEL_ID = "C_STEER"; const THREAD_TS = "1712345.000100"; @@ -55,7 +56,7 @@ function createTurnHarness(args: { generateAssistantReply: ReplyExecutorServices["generateAssistantReply"]; state: StateAdapter; }) { - const queue = createFakeQueue(); + const queue = createConversationWorkQueueTestAdapter(); const adapter = createJuniorSlackAdapter({ botToken: "slack-bot-fixture", botUserId: SLACK_BOT_USER_ID, @@ -79,8 +80,9 @@ function createTurnHarness(args: { channel: CHANNEL_ID, threadTs: THREAD_TS, }); - const runWorker = () => - processConversationWork(conversationId, { + const runNextQueuedWork = () => { + const message = queue.takeMessage(); + return processConversationQueueMessage(message, { queue, run: createSlackConversationWorker({ getSlackAdapter: () => adapter, @@ -89,11 +91,12 @@ function createTurnHarness(args: { }), state: args.state, }); + }; return { conversationId, queue, - runWorker, + runNextQueuedWork, services, }; } @@ -119,6 +122,7 @@ describe("Slack behavior: durable turn steering", () => { const state = getStateAdapter(); const generateAssistantReply: ReplyExecutorServices["generateAssistantReply"] = async (prompt, context) => { + await context?.onInputCommitted?.(); agentEntered.resolve(); await releaseAgent.promise; @@ -142,10 +146,11 @@ describe("Slack behavior: durable turn steering", () => { diagnostics: makeDiagnostics(), }; }; - const { conversationId, queue, runWorker, services } = createTurnHarness({ - generateAssistantReply, - state, - }); + const { conversationId, queue, runNextQueuedWork, services } = + createTurnHarness({ + generateAssistantReply, + state, + }); const firstResponse = await handleSlackWebhookAndFlush({ request: slackWebhookRequest( @@ -158,9 +163,9 @@ describe("Slack behavior: durable turn steering", () => { services, }); expect(firstResponse.status).toBe(200); - expect(queue.sent).toHaveLength(1); + expect(queue.sentRecords()).toHaveLength(1); - const activeTurn = runWorker(); + const activeTurn = runNextQueuedWork(); await agentEntered.promise; for (const followUp of [ @@ -183,7 +188,7 @@ describe("Slack behavior: durable turn steering", () => { releaseAgent.resolve(); await expect(activeTurn).resolves.toEqual({ status: "completed" }); - expect(queue.sent).toHaveLength(4); + expect(queue.sentRecords()).toHaveLength(4); expect(agentCalls).toEqual([ { @@ -196,7 +201,7 @@ describe("Slack behavior: durable turn steering", () => { }, ]); - const postCalls = getCapturedSlackApiCalls("chat.postMessage"); + const postCalls = slackApiOutbox.messages(); expect(postCalls).toHaveLength(1); expect(postCalls[0]?.params).toEqual( expect.objectContaining({ @@ -206,13 +211,12 @@ describe("Slack behavior: durable turn steering", () => { }), ); - const queuedWakeups = queue.sent.length; - for (let index = 1; index < queuedWakeups; index += 1) { - await expect(runWorker()).resolves.toEqual({ status: "no_work" }); + while (queue.hasQueuedMessages()) { + await expect(runNextQueuedWork()).resolves.toEqual({ status: "no_work" }); } expect(agentCalls).toHaveLength(1); - expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(1); + expect(slackApiOutbox.messages()).toHaveLength(1); const work = await getConversationWorkState({ conversationId, state, @@ -223,4 +227,50 @@ describe("Slack behavior: durable turn steering", () => { ).toBe(true); expect(work?.needsRun).toBe(false); }); + + it("keeps the mailbox pending when the agent fails before input commit", async () => { + const state = getStateAdapter(); + const generateAssistantReply: ReplyExecutorServices["generateAssistantReply"] = + async (_prompt, context) => { + expect(context?.onInputCommitted).toEqual(expect.any(Function)); + throw new Error("agent crashed before input commit"); + }; + const { conversationId, queue, runNextQueuedWork, services } = + createTurnHarness({ + generateAssistantReply, + state, + }); + + await expect( + handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + makeMessageEvent({ + eventType: "app_mention", + text: `<@${SLACK_BOT_USER_ID}> start the incident summary`, + ts: THREAD_TS, + }), + ), + services, + }), + ).resolves.toMatchObject({ status: 200 }); + + await expect(runNextQueuedWork()).resolves.toEqual({ + status: "pending_requeued", + }); + + const work = await getConversationWorkState({ + conversationId, + state, + }); + expect(work?.needsRun).toBe(true); + expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + expect(work?.messages[0]?.injectedAtMs).toBeUndefined(); + expect(queue.sentRecords()).toEqual([ + expect.objectContaining({ conversationId }), + expect.objectContaining({ + conversationId, + idempotencyKey: expect.stringContaining(`pending:${conversationId}:`), + }), + ]); + }); }); diff --git a/packages/junior/tests/integration/slack/message-changed-behavior.test.ts b/packages/junior/tests/integration/slack/message-changed-behavior.test.ts index 760f6469f..32ea3b81e 100644 --- a/packages/junior/tests/integration/slack/message-changed-behavior.test.ts +++ b/packages/junior/tests/integration/slack/message-changed-behavior.test.ts @@ -1,51 +1,23 @@ -import { createHmac } from "node:crypto"; import { http, HttpResponse } from "msw"; import { afterEach, describe, expect, it } from "vitest"; import { createMemoryState } from "@chat-adapter/state-memory"; import type { SlackAdapter } from "@chat-adapter/slack"; import type { Message } from "chat"; import { slackEventsApiEnvelope } from "../../fixtures/slack/factories/events"; -import { getCapturedSlackApiCalls } from "../../msw/handlers/slack-api"; +import { slackApiOutbox } from "../../fixtures/slack-api-outbox"; +import { createSlackWebhookTestClient } from "../../fixtures/slack/webhook-client"; import { mswServer } from "../../msw/server"; import { createSlackRuntime } from "@/chat/app/factory"; import { JuniorChat } from "@/chat/ingress/junior-chat"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; -import type { WaitUntilFn } from "@/handlers/types"; import { handlePlatformWebhook } from "@/handlers/webhooks"; const SIGNING_SECRET = "test-signing-secret"; const BOT_USER_ID = "U_BOT"; const ORIGINAL_ENV = { ...process.env }; - -function signSlackBody(body: string, timestamp: string): string { - const base = `v0:${timestamp}:${body}`; - return `v0=${createHmac("sha256", SIGNING_SECRET).update(base).digest("hex")}`; -} - -function createSlackRequest(body: string, signature?: string): Request { - const timestamp = String(Math.floor(Date.now() / 1000)); - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - ...(signature ? { "x-slack-signature": signature } : {}), - }, - body, - }); -} - -async function flushWaitUntil(tasks: Array>): Promise { - for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]; - } -} - -function collectWaitUntil(tasks: Array>): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task() : task); - }; -} +const slackWebhookClient = createSlackWebhookTestClient({ + signingSecret: SIGNING_SECRET, +}); function makeDiagnostics() { return { @@ -79,7 +51,7 @@ describe("Slack behavior: message_changed webhook ingress", () => { const handledMessages: Array< Pick > = []; - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); bot.onDirectMessage(async (_thread, message) => { handledMessages.push({ @@ -90,35 +62,20 @@ describe("Slack behavior: message_changed webhook ingress", () => { }); }); - const originalBody = JSON.stringify( - slackEventsApiEnvelope({ - eventType: "message", - channel: "D12345", - ts: "1700000100.000100", - text: "hello there", - }), - ); - const originalTimestamp = String(Math.floor(Date.now() / 1000)); - const originalRequest = new Request( - "https://example.test/api/webhooks/slack", - { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": originalTimestamp, - "x-slack-signature": signSlackBody(originalBody, originalTimestamp), - }, - body: originalBody, - }, - ); - const originalResponse = await handlePlatformWebhook( - originalRequest, + slackWebhookClient.event( + slackEventsApiEnvelope({ + eventType: "message", + channel: "D12345", + ts: "1700000100.000100", + text: "hello there", + }), + ), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); const editedPayload = { ...slackEventsApiEnvelope({ @@ -146,28 +103,14 @@ describe("Slack behavior: message_changed webhook ingress", () => { }, }, }; - const editedBody = JSON.stringify(editedPayload); - const editedTimestamp = String(Math.floor(Date.now() / 1000)); - const editedRequest = new Request( - "https://example.test/api/webhooks/slack", - { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": editedTimestamp, - "x-slack-signature": signSlackBody(editedBody, editedTimestamp), - }, - body: editedBody, - }, - ); const editedResponse = await handlePlatformWebhook( - editedRequest, + slackWebhookClient.event(editedPayload), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); expect(originalResponse.status).toBe(200); expect(editedResponse.status).toBe(200); @@ -214,7 +157,6 @@ describe("Slack behavior: message_changed webhook ingress", () => { }, state, }); - const waitUntilTasks: Array> = []; const handledMessages: Array< Pick > = []; @@ -228,7 +170,8 @@ describe("Slack behavior: message_changed webhook ingress", () => { }); }); - const editedBody = JSON.stringify({ + const waitUntil = slackWebhookClient.waitUntil(); + const editedPayload = { ...slackEventsApiEnvelope({ eventType: "message", channel: "D12345", @@ -262,28 +205,15 @@ describe("Slack behavior: message_changed webhook ingress", () => { ts: "1700000100.000102", }, }, - }); - const editedTimestamp = String(Math.floor(Date.now() / 1000)); - const editedRequest = new Request( - "https://example.test/api/webhooks/slack", - { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": editedTimestamp, - "x-slack-signature": signSlackBody(editedBody, editedTimestamp), - }, - body: editedBody, - }, - ); + }; const response = await handlePlatformWebhook( - editedRequest, + slackWebhookClient.event(editedPayload), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); expect(response.status).toBe(200); expect(handledMessages).toHaveLength(1); @@ -333,13 +263,13 @@ describe("Slack behavior: message_changed webhook ingress", () => { }, }, }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); bot.onDirectMessage((thread, message) => slackRuntime.handleNewMention(thread, message), ); - const editedBody = JSON.stringify({ + const editedPayload = { ...slackEventsApiEnvelope({ eventType: "message", channel: "D12345", @@ -364,31 +294,18 @@ describe("Slack behavior: message_changed webhook ingress", () => { ts: "1700000100.000100", }, }, - }); - const editedTimestamp = String(Math.floor(Date.now() / 1000)); - const editedRequest = new Request( - "https://example.test/api/webhooks/slack", - { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": editedTimestamp, - "x-slack-signature": signSlackBody(editedBody, editedTimestamp), - }, - body: editedBody, - }, - ); + }; const response = await handlePlatformWebhook( - editedRequest, + slackWebhookClient.event(editedPayload), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); expect(response.status).toBe(200); - const postCalls = getCapturedSlackApiCalls("chat.postMessage"); + const postCalls = slackApiOutbox.messages(); expect(postCalls).toHaveLength(1); expect(postCalls[0]).toEqual( @@ -420,7 +337,7 @@ describe("Slack behavior: message_changed webhook ingress", () => { handledMessages.push(message); }); - const body = JSON.stringify({ + const payload = { ...slackEventsApiEnvelope({ eventType: "message", channel: "D12345", @@ -440,11 +357,10 @@ describe("Slack behavior: message_changed webhook ingress", () => { text: "hello there", }, }, - }); - const request = createSlackRequest(body, "v0=forged"); + }; const response = await handlePlatformWebhook( - request, + slackWebhookClient.invalidSignature(payload), "slack", () => undefined, bot, diff --git a/packages/junior/tests/integration/slack/message-changed-reply-contract.test.ts b/packages/junior/tests/integration/slack/message-changed-reply-contract.test.ts index 8c141a20d..e2bffd319 100644 --- a/packages/junior/tests/integration/slack/message-changed-reply-contract.test.ts +++ b/packages/junior/tests/integration/slack/message-changed-reply-contract.test.ts @@ -1,35 +1,20 @@ -import { createHmac } from "node:crypto"; import { describe, expect, it } from "vitest"; import { createMemoryState } from "@chat-adapter/state-memory"; import type { SlackAdapter } from "@chat-adapter/slack"; import { slackEventsApiEnvelope } from "../../fixtures/slack/factories/events"; -import { getCapturedSlackApiCalls } from "../../msw/handlers/slack-api"; +import { slackApiOutbox } from "../../fixtures/slack-api-outbox"; +import { createSlackWebhookTestClient } from "../../fixtures/slack/webhook-client"; import { createSlackRuntime } from "@/chat/app/factory"; import { JuniorChat } from "@/chat/ingress/junior-chat"; import type { ReplyExecutorServices } from "@/chat/runtime/reply-executor"; import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; -import type { WaitUntilFn } from "@/handlers/types"; import { handlePlatformWebhook } from "@/handlers/webhooks"; const SIGNING_SECRET = "test-signing-secret"; const BOT_USER_ID = "U_BOT"; - -function signSlackBody(body: string, timestamp: string): string { - const base = `v0:${timestamp}:${body}`; - return `v0=${createHmac("sha256", SIGNING_SECRET).update(base).digest("hex")}`; -} - -async function flushWaitUntil(tasks: Array>): Promise { - for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]; - } -} - -function collectWaitUntil(tasks: Array>): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task() : task); - }; -} +const slackWebhookClient = createSlackWebhookTestClient({ + signingSecret: SIGNING_SECRET, +}); function makeDiagnostics() { return { @@ -48,7 +33,7 @@ function createEditedMentionRequest(args: { newText: string; prevText: string; }): Request { - const body = JSON.stringify({ + return slackWebhookClient.event({ ...slackEventsApiEnvelope({ eventType: "message", channel: "D12345", @@ -74,17 +59,6 @@ function createEditedMentionRequest(args: { }, }, }); - const timestamp = String(Math.floor(Date.now() / 1000)); - - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(body, timestamp), - }, - body, - }); } async function createEditedDmBot(args: { @@ -130,7 +104,7 @@ describe("Slack contract: edited-message reply delivery", () => { }; }, }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const response = await handlePlatformWebhook( createEditedMentionRequest({ @@ -139,13 +113,13 @@ describe("Slack contract: edited-message reply delivery", () => { prevText: "hello there", }), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); expect(response.status).toBe(200); - expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ + expect(slackApiOutbox.messages()).toEqual([ expect.objectContaining({ params: expect.objectContaining({ blocks: [ @@ -184,7 +158,7 @@ describe("Slack contract: edited-message reply delivery", () => { diagnostics: makeDiagnostics(), }), }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const response = await handlePlatformWebhook( createEditedMentionRequest({ @@ -193,13 +167,13 @@ describe("Slack contract: edited-message reply delivery", () => { prevText: "hello there", }), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); expect(response.status).toBe(200); - const postCalls = getCapturedSlackApiCalls("chat.postMessage"); + const postCalls = slackApiOutbox.messages(); expect(postCalls.length).toBeGreaterThan(1); expect(postCalls[0]).toEqual( expect.objectContaining({ diff --git a/packages/junior/tests/integration/slack/message-im-attachment-contract.test.ts b/packages/junior/tests/integration/slack/message-im-attachment-contract.test.ts index a0da09d5e..57cb877d1 100644 --- a/packages/junior/tests/integration/slack/message-im-attachment-contract.test.ts +++ b/packages/junior/tests/integration/slack/message-im-attachment-contract.test.ts @@ -1,10 +1,9 @@ -import { createHmac } from "node:crypto"; import { http, HttpResponse } from "msw"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createMemoryState } from "@chat-adapter/state-memory"; import { slackEventsApiEnvelope } from "../../fixtures/slack/factories/events"; +import { createSlackWebhookTestClient } from "../../fixtures/slack/webhook-client"; import { mswServer } from "../../msw/server"; -import type { WaitUntilFn } from "@/handlers/types"; import type { ReplyExecutorServices } from "@/chat/runtime/reply-executor"; const SIGNING_SECRET = "test-signing-secret"; @@ -12,36 +11,9 @@ const BOT_USER_ID = "U_BOT"; const DM_CHANNEL_ID = "D12345"; const DM_THREAD_TS = "1700000000.000001"; const ORIGINAL_ENV = { ...process.env }; - -function signSlackBody(body: string, timestamp: string): string { - const base = `v0:${timestamp}:${body}`; - return `v0=${createHmac("sha256", SIGNING_SECRET).update(base).digest("hex")}`; -} - -function createSlackRequest(body: string): Request { - const timestamp = String(Math.floor(Date.now() / 1000)); - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": timestamp, - "x-slack-signature": signSlackBody(body, timestamp), - }, - body, - }); -} - -function collectWaitUntil(tasks: Array>): WaitUntilFn { - return (task) => { - tasks.push(typeof task === "function" ? task() : task); - }; -} - -async function flushWaitUntil(tasks: Array>): Promise { - for (let index = 0; index < tasks.length; index += 1) { - await tasks[index]; - } -} +const slackWebhookClient = createSlackWebhookTestClient({ + signingSecret: SIGNING_SECRET, +}); function makeDiagnostics() { return { @@ -141,7 +113,7 @@ describe("Slack contract: message.im attachment ingress", () => { }; }, }); - const waitUntilTasks: Array> = []; + const waitUntil = slackWebhookClient.waitUntil(); const baseEnvelope = slackEventsApiEnvelope({ eventType: "message", @@ -150,7 +122,7 @@ describe("Slack contract: message.im attachment ingress", () => { threadTs: DM_THREAD_TS, text: "what is in this screenshot?", }); - const body = JSON.stringify({ + const payload = { ...baseEnvelope, event: { ...baseEnvelope.event, @@ -165,20 +137,20 @@ describe("Slack contract: message.im attachment ingress", () => { }, ], }, - }); + }; const { handlePlatformWebhook } = await import("@/handlers/webhooks"); const response = await handlePlatformWebhook( - createSlackRequest(body), + slackWebhookClient.event(payload), "slack", - collectWaitUntil(waitUntilTasks), + waitUntil.fn, bot, ); expect(response.status).toBe(200); - await flushWaitUntil(waitUntilTasks); + await waitUntil.flush(); expect(capturedAttachmentMediaTypes).toEqual([["image/png"]]); expect(capturedAttachmentNames).toEqual([["current.png"]]); - }, 10_000); + }, 20_000); }); diff --git a/packages/junior/tests/integration/slack/processing-reaction-behavior.test.ts b/packages/junior/tests/integration/slack/processing-reaction-behavior.test.ts index e1c9b0623..53f94124f 100644 --- a/packages/junior/tests/integration/slack/processing-reaction-behavior.test.ts +++ b/packages/junior/tests/integration/slack/processing-reaction-behavior.test.ts @@ -4,7 +4,7 @@ import { createTestMessage, createTestThread, } from "../../fixtures/slack-harness"; -import { getCapturedSlackApiCalls } from "../../msw/handlers/slack-api"; +import { slackApiOutbox } from "../../fixtures/slack-api-outbox"; function successDiagnostics(toolCalls: string[] = []) { return { @@ -24,10 +24,8 @@ describe("Slack behavior: processing reaction", () => { services: { replyExecutor: { generateAssistantReply: async () => { - expect(getCapturedSlackApiCalls("reactions.add")).toHaveLength(1); - expect(getCapturedSlackApiCalls("reactions.remove")).toHaveLength( - 0, - ); + expect(slackApiOutbox.reactionAdds()).toHaveLength(1); + expect(slackApiOutbox.reactionRemovals()).toHaveLength(0); return { text: "Done.", diagnostics: successDiagnostics(), @@ -55,7 +53,7 @@ describe("Slack behavior: processing reaction", () => { }), ); - expect(getCapturedSlackApiCalls("reactions.add")).toEqual([ + expect(slackApiOutbox.reactionAdds()).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel: "C_PROCESSING", @@ -64,7 +62,7 @@ describe("Slack behavior: processing reaction", () => { }), }), ]); - expect(getCapturedSlackApiCalls("reactions.remove")).toEqual([ + expect(slackApiOutbox.reactionRemovals()).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel: "C_PROCESSING", @@ -80,10 +78,8 @@ describe("Slack behavior: processing reaction", () => { services: { subscribedReplyPolicy: { completeObject: async () => { - expect(getCapturedSlackApiCalls("reactions.add")).toHaveLength(0); - expect(getCapturedSlackApiCalls("reactions.remove")).toHaveLength( - 0, - ); + expect(slackApiOutbox.reactionAdds()).toHaveLength(0); + expect(slackApiOutbox.reactionRemovals()).toHaveLength(0); return { object: { should_reply: false, @@ -121,8 +117,8 @@ describe("Slack behavior: processing reaction", () => { ); expect(thread.posts).toHaveLength(0); - expect(getCapturedSlackApiCalls("reactions.add")).toHaveLength(0); - expect(getCapturedSlackApiCalls("reactions.remove")).toHaveLength(0); + expect(slackApiOutbox.reactionAdds()).toHaveLength(0); + expect(slackApiOutbox.reactionRemovals()).toHaveLength(0); }); it("adds eyes after a subscribed message is approved and removes it after the reply", async () => { @@ -130,10 +126,8 @@ describe("Slack behavior: processing reaction", () => { services: { subscribedReplyPolicy: { completeObject: async () => { - expect(getCapturedSlackApiCalls("reactions.add")).toHaveLength(0); - expect(getCapturedSlackApiCalls("reactions.remove")).toHaveLength( - 0, - ); + expect(slackApiOutbox.reactionAdds()).toHaveLength(0); + expect(slackApiOutbox.reactionRemovals()).toHaveLength(0); return { object: { should_reply: true, @@ -146,10 +140,8 @@ describe("Slack behavior: processing reaction", () => { }, replyExecutor: { generateAssistantReply: async () => { - expect(getCapturedSlackApiCalls("reactions.add")).toHaveLength(1); - expect(getCapturedSlackApiCalls("reactions.remove")).toHaveLength( - 0, - ); + expect(slackApiOutbox.reactionAdds()).toHaveLength(1); + expect(slackApiOutbox.reactionRemovals()).toHaveLength(0); return { text: "Done.", diagnostics: successDiagnostics(), @@ -177,7 +169,7 @@ describe("Slack behavior: processing reaction", () => { }), ); - expect(getCapturedSlackApiCalls("reactions.add")).toEqual([ + expect(slackApiOutbox.reactionAdds()).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel: "C_PROCESSING", @@ -186,7 +178,7 @@ describe("Slack behavior: processing reaction", () => { }), }), ]); - expect(getCapturedSlackApiCalls("reactions.remove")).toEqual([ + expect(slackApiOutbox.reactionRemovals()).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel: "C_PROCESSING", @@ -233,7 +225,7 @@ describe("Slack behavior: processing reaction", () => { }), ); - expect(getCapturedSlackApiCalls("reactions.add")).toHaveLength(1); - expect(getCapturedSlackApiCalls("reactions.remove")).toHaveLength(0); + expect(slackApiOutbox.reactionAdds()).toHaveLength(1); + expect(slackApiOutbox.reactionRemovals()).toHaveLength(0); }); }); diff --git a/packages/junior/tests/integration/slack/webhook-auth-boundary.test.ts b/packages/junior/tests/integration/slack/webhook-auth-boundary.test.ts index 8e8d47ed6..e4191a939 100644 --- a/packages/junior/tests/integration/slack/webhook-auth-boundary.test.ts +++ b/packages/junior/tests/integration/slack/webhook-auth-boundary.test.ts @@ -1,57 +1,40 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; +import { handleSlackWebhook } from "@/chat/ingress/slack-webhook"; +import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; +import { + createConversationWorkQueueTestAdapter, + createNoopSlackWebhookRuntime, +} from "../../fixtures/conversation-work"; +import { createSlackWebhookTestClient } from "../../fixtures/slack/webhook-client"; -const ORIGINAL_ENV = { ...process.env }; - -type WaitUntilTask = Promise; - -function invalidSlackRequest(): Request { - const body = JSON.stringify({ type: "event_callback" }); - return new Request("https://example.test/api/webhooks/slack", { - method: "POST", - headers: { - "content-type": "application/json", - "x-slack-request-timestamp": String(Math.floor(Date.now() / 1000)), - "x-slack-signature": "v0=invalid", - }, - body, - }); -} - -function collectWaitUntil(tasks: WaitUntilTask[]) { - return (task: WaitUntilTask | (() => WaitUntilTask)) => { - tasks.push(typeof task === "function" ? task() : task); - }; -} +const SIGNING_SECRET = "test-signing-secret"; describe("Slack webhook auth boundary", () => { - beforeEach(() => { - vi.resetModules(); - process.env = { - ...ORIGINAL_ENV, - SLACK_BOT_TOKEN: "xoxb-test-token", - SLACK_SIGNING_SECRET: "test-signing-secret", - }; - delete process.env.JUNIOR_STATE_ADAPTER; - delete process.env.REDIS_URL; - }); - - afterEach(() => { - process.env = { ...ORIGINAL_ENV }; - vi.resetModules(); - }); - it("rejects invalid Slack signatures before durable state is required", async () => { - const { handlePlatformWebhook } = await import("@/handlers/webhooks"); - const waitUntilTasks: WaitUntilTask[] = []; - - const response = await handlePlatformWebhook( - invalidSlackRequest(), - "slack", - collectWaitUntil(waitUntilTasks), - ); + const client = createSlackWebhookTestClient({ + signingSecret: SIGNING_SECRET, + }); + const queue = createConversationWorkQueueTestAdapter(); + const waitUntil = client.waitUntil(); + const adapter = createJuniorSlackAdapter({ + botToken: "xoxb-test-token", + botUserId: "U_BOT", + signingSecret: SIGNING_SECRET, + }); + + const response = await handleSlackWebhook({ + request: client.invalidSignature({ type: "event_callback" }), + waitUntil: waitUntil.fn, + services: { + getSlackAdapter: () => adapter, + queue, + runtime: createNoopSlackWebhookRuntime(), + }, + }); expect(response.status).toBe(401); expect(await response.text()).toBe("Invalid signature"); - expect(waitUntilTasks).toHaveLength(0); + expect(queue.sentRecords()).toEqual([]); + expect(waitUntil.pendingCount()).toBe(0); }); }); diff --git a/packages/junior/tests/integration/turn-resume-slack.test.ts b/packages/junior/tests/integration/turn-resume-slack.test.ts index ed9b53ba5..0bc7780ff 100644 --- a/packages/junior/tests/integration/turn-resume-slack.test.ts +++ b/packages/junior/tests/integration/turn-resume-slack.test.ts @@ -1,84 +1,67 @@ import { Buffer } from "node:buffer"; -import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { WaitUntilFn } from "@/handlers/types"; import { - getCapturedSlackApiCalls, - getCapturedSlackFileUploadCalls, - resetSlackApiMockState, -} from "../msw/handlers/slack-api"; + createConversationWorkQueueTestAdapter, + type ConversationWorkQueueTestAdapter, +} from "../fixtures/conversation-work"; +import { slackApiOutbox } from "../fixtures/slack-api-outbox"; +import { + createTurnResumeTestClient, + type TurnResumeTestClient, +} from "../fixtures/turn-resume"; +import type { WaitUntilCollector } from "../fixtures/wait-until"; +import { resetSlackApiMockState } from "../msw/handlers/slack-api"; -const { generateAssistantReplyMock, queueSends } = vi.hoisted(() => ({ +const { generateAssistantReplyMock } = vi.hoisted(() => ({ generateAssistantReplyMock: vi.fn(), - queueSends: [] as Array<{ - conversationId: string; - idempotencyKey?: string; - }>, })); vi.mock("@/chat/respond", () => ({ generateAssistantReply: generateAssistantReplyMock, })); -vi.mock("@/chat/task-execution/vercel-queue", () => ({ - DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC: "junior_conversation_work", - getVercelConversationWorkQueue: () => ({ - send: async ( - message: { conversationId: string }, - options?: { idempotencyKey?: string }, - ) => { - queueSends.push({ - conversationId: message.conversationId, - idempotencyKey: options?.idempotencyKey, - }); - return { messageId: `queue-${queueSends.length}` }; - }, - }), -})); - const ORIGINAL_ENV = { ...process.env }; type StateAdapterModule = typeof import("@/chat/state/adapter"); type ThreadStateModule = typeof import("@/chat/runtime/thread-state"); type TurnResumeHandlerModule = typeof import("@/handlers/turn-resume"); type TurnSessionStoreModule = typeof import("@/chat/state/turn-session"); +type TimeoutResumeServiceModule = + typeof import("@/chat/services/timeout-resume"); let stateAdapterModule: StateAdapterModule; let threadStateModule: ThreadStateModule; let turnResumeHandlerModule: TurnResumeHandlerModule; let turnSessionStoreModule: TurnSessionStoreModule; +let timeoutResumeServiceModule: TimeoutResumeServiceModule; +let queue: ConversationWorkQueueTestAdapter; +let turnResumeClient: TurnResumeTestClient; +let waitUntil: WaitUntilCollector; -const waitUntilCallbacks: Array<() => Promise | void> = []; - -const testWaitUntil: WaitUntilFn = (task) => { - waitUntilCallbacks.push(typeof task === "function" ? task : () => task); -}; - -async function buildSignedTurnResumeRequest(args: { +function postResumeRequest(args: { conversationId: string; sessionId: string; expectedVersion: number; -}): Promise { - const timestamp = Date.now().toString(); - const body = JSON.stringify(args); - const digest = createHmac("sha256", "resume-secret") - .update(`junior.turn_timeout_resume.v1:${timestamp}:${body}`) - .digest("hex"); - return new Request("https://junior.example.com/api/internal/turn-resume", { - method: "POST", - headers: { - "content-type": "application/json", - "x-junior-resume-timestamp": timestamp, - "x-junior-resume-signature": `v1=${digest}`, +}): Promise { + return turnResumeHandlerModule.POST( + turnResumeClient.request(args), + waitUntil.fn, + { + scheduleTurnTimeoutResume: (request) => + timeoutResumeServiceModule.scheduleTurnTimeoutResume(request, { + queue, + }), }, - body, - }); + ); } describe("turn resume slack integration", () => { beforeEach(async () => { - waitUntilCallbacks.length = 0; - queueSends.length = 0; + queue = createConversationWorkQueueTestAdapter(); + turnResumeClient = createTurnResumeTestClient({ + juniorSecret: "resume-secret", + }); + waitUntil = turnResumeClient.waitUntil(); generateAssistantReplyMock.mockReset(); generateAssistantReplyMock.mockResolvedValue({ text: "Final resumed answer", @@ -101,6 +84,7 @@ describe("turn resume slack integration", () => { threadStateModule = await import("@/chat/runtime/thread-state"); turnResumeHandlerModule = await import("@/handlers/turn-resume"); turnSessionStoreModule = await import("@/chat/state/turn-session"); + timeoutResumeServiceModule = await import("@/chat/services/timeout-resume"); await stateAdapterModule.disconnectStateAdapter(); await stateAdapterModule.getStateAdapter().connect(); @@ -180,19 +164,16 @@ describe("turn resume slack integration", () => { source: "test", }); - const response = await turnResumeHandlerModule.POST( - await buildSignedTurnResumeRequest({ - conversationId, - sessionId, - expectedVersion: sessionRecord.version, - }), - testWaitUntil, - ); + const response = await postResumeRequest({ + conversationId, + sessionId, + expectedVersion: sessionRecord.version, + }); expect(response.status).toBe(202); - expect(waitUntilCallbacks).toHaveLength(1); + expect(waitUntil.pendingCount()).toBe(1); - await waitUntilCallbacks[0]?.(); + await waitUntil.flush(); expect(generateAssistantReplyMock).toHaveBeenCalledWith( "resume this request", @@ -222,7 +203,7 @@ describe("turn resume slack integration", () => { "acme", ); - expect(getCapturedSlackApiCalls("assistant.threads.setStatus")).toEqual( + expect(slackApiOutbox.calls("assistant.threads.setStatus")).toEqual( expect.arrayContaining([ expect.objectContaining({ params: expect.objectContaining({ @@ -241,7 +222,7 @@ describe("turn resume slack integration", () => { }), ]), ); - expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ + expect(slackApiOutbox.messages()).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel: "C123", @@ -330,22 +311,19 @@ describe("turn resume slack integration", () => { }), ); - const response = await turnResumeHandlerModule.POST( - await buildSignedTurnResumeRequest({ - conversationId, - sessionId, - expectedVersion: sessionRecord.version, - }), - testWaitUntil, - ); + const response = await postResumeRequest({ + conversationId, + sessionId, + expectedVersion: sessionRecord.version, + }); expect(response.status).toBe(202); - expect(waitUntilCallbacks).toHaveLength(1); + expect(waitUntil.pendingCount()).toBe(1); - await waitUntilCallbacks[0]?.(); + await waitUntil.flush(); - expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([]); - expect(queueSends).toEqual([ + expect(slackApiOutbox.messages()).toEqual([]); + expect(queue.sentRecords()).toEqual([ { conversationId, idempotencyKey: expect.stringContaining( @@ -428,23 +406,20 @@ describe("turn resume slack integration", () => { }), ); - const response = await turnResumeHandlerModule.POST( - await buildSignedTurnResumeRequest({ - conversationId, - sessionId, - expectedVersion: sessionRecord.version, - }), - testWaitUntil, - ); + const response = await postResumeRequest({ + conversationId, + sessionId, + expectedVersion: sessionRecord.version, + }); expect(response.status).toBe(202); - expect(waitUntilCallbacks).toHaveLength(1); + expect(waitUntil.pendingCount()).toBe(1); - await waitUntilCallbacks[0]?.(); + await waitUntil.flush(); - const postCalls = getCapturedSlackApiCalls("chat.postMessage"); + const postCalls = slackApiOutbox.messages(); expect(postCalls).toEqual([]); - expect(queueSends).toEqual([ + expect(queue.sentRecords()).toEqual([ { conversationId, idempotencyKey: expect.stringContaining( @@ -526,21 +501,18 @@ describe("turn resume slack integration", () => { }, }); - const response = await turnResumeHandlerModule.POST( - await buildSignedTurnResumeRequest({ - conversationId, - sessionId, - expectedVersion: sessionRecord.version, - }), - testWaitUntil, - ); + const response = await postResumeRequest({ + conversationId, + sessionId, + expectedVersion: sessionRecord.version, + }); expect(response.status).toBe(202); - expect(waitUntilCallbacks).toHaveLength(1); + expect(waitUntil.pendingCount()).toBe(1); - await waitUntilCallbacks[0]?.(); + await waitUntil.flush(); - expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ + expect(slackApiOutbox.messages()).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel: "C123", @@ -549,10 +521,8 @@ describe("turn resume slack integration", () => { }), }), ]); - expect(getCapturedSlackApiCalls("files.getUploadURLExternal")).toHaveLength( - 1, - ); - expect(getCapturedSlackApiCalls("files.completeUploadExternal")).toEqual([ + expect(slackApiOutbox.calls("files.getUploadURLExternal")).toHaveLength(1); + expect(slackApiOutbox.calls("files.completeUploadExternal")).toEqual([ expect.objectContaining({ params: expect.objectContaining({ channel_id: "C123", @@ -560,7 +530,7 @@ describe("turn resume slack integration", () => { }), }), ]); - expect(getCapturedSlackFileUploadCalls()).toHaveLength(1); + expect(slackApiOutbox.fileUploads()).toHaveLength(1); const persisted = await threadStateModule.getPersistedThreadState(conversationId); diff --git a/packages/junior/tests/unit/handlers/mcp-oauth-callback.test.ts b/packages/junior/tests/unit/handlers/mcp-oauth-callback.test.ts index 98cff0465..6fdcff99d 100644 --- a/packages/junior/tests/unit/handlers/mcp-oauth-callback.test.ts +++ b/packages/junior/tests/unit/handlers/mcp-oauth-callback.test.ts @@ -1,5 +1,4 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { WaitUntilFn } from "@/handlers/types"; const { finalizeMcpAuthorizationMock } = vi.hoisted(() => ({ finalizeMcpAuthorizationMock: vi.fn(), @@ -10,21 +9,21 @@ vi.mock("@/chat/mcp/oauth", () => ({ })); import { GET } from "@/handlers/mcp-oauth-callback"; +import { + createWaitUntilCollector, + type WaitUntilCollector, +} from "../../fixtures/wait-until"; -const waitUntilCallbacks: Array<() => Promise | void> = []; +let waitUntil: WaitUntilCollector; function makeRequest(url: string): Request { return new Request(url, { method: "GET" }); } -const testWaitUntil: WaitUntilFn = (task) => { - waitUntilCallbacks.push(typeof task === "function" ? task : () => task); -}; - describe("mcp oauth callback handler", () => { beforeEach(() => { finalizeMcpAuthorizationMock.mockReset(); - waitUntilCallbacks.length = 0; + waitUntil = createWaitUntilCollector(); }); afterEach(() => { @@ -35,13 +34,13 @@ describe("mcp oauth callback handler", () => { const response = await GET( makeRequest("https://example.com/api/oauth/callback/mcp/demo?code=abc"), "demo", - testWaitUntil, + waitUntil.fn, ); expect(response.status).toBe(400); expect(await response.text()).toContain("Missing state parameter"); expect(finalizeMcpAuthorizationMock).not.toHaveBeenCalled(); - expect(waitUntilCallbacks).toHaveLength(0); + expect(waitUntil.pendingCount()).toBe(0); }); it("does not reflect provider error text in the HTML response", async () => { @@ -50,14 +49,14 @@ describe("mcp oauth callback handler", () => { "https://example.com/api/oauth/callback/mcp/demo?state=state-123&error=%3Cscript%3Ealert(1)%3C%2Fscript%3E", ), "demo", - testWaitUntil, + waitUntil.fn, ); expect(response.status).toBe(400); const body = await response.text(); expect(body).toContain("The provider returned an authorization error."); expect(body).not.toContain(""); - expect(waitUntilCallbacks).toHaveLength(0); + expect(waitUntil.pendingCount()).toBe(0); }); it("does not reflect callback exception text in the HTML response", async () => { @@ -70,7 +69,7 @@ describe("mcp oauth callback handler", () => { "https://example.com/api/oauth/callback/mcp/demo?code=auth-code&state=state-123", ), "demo", - testWaitUntil, + waitUntil.fn, ); expect(response.status).toBe(500); @@ -79,6 +78,6 @@ describe("mcp oauth callback handler", () => { "Junior could not finish the authorization callback. Return to Slack and retry the original request.", ); expect(body).not.toContain(""); - expect(waitUntilCallbacks).toHaveLength(0); + expect(waitUntil.pendingCount()).toBe(0); }); }); diff --git a/packages/junior/tests/unit/handlers/turn-resume.test.ts b/packages/junior/tests/unit/handlers/turn-resume.test.ts index 457a3cef6..3269d2c88 100644 --- a/packages/junior/tests/unit/handlers/turn-resume.test.ts +++ b/packages/junior/tests/unit/handlers/turn-resume.test.ts @@ -4,12 +4,10 @@ const { resumeSlackTurnMock, scheduleTurnTimeoutResumeMock, verifyTurnTimeoutResumeRequestMock, - waitUntilCallbacks, } = vi.hoisted(() => ({ resumeSlackTurnMock: vi.fn(), scheduleTurnTimeoutResumeMock: vi.fn(), verifyTurnTimeoutResumeRequestMock: vi.fn(), - waitUntilCallbacks: [] as Array<() => Promise | void>, })); vi.mock("@/chat/config", async (importOriginal) => { @@ -27,7 +25,6 @@ vi.mock("@/chat/config", async (importOriginal) => { vi.mock("@/chat/services/timeout-resume", async (importOriginal) => ({ ...(await importOriginal()), - scheduleTurnTimeoutResume: scheduleTurnTimeoutResumeMock, verifyTurnTimeoutResumeRequest: verifyTurnTimeoutResumeRequestMock, })); @@ -46,15 +43,26 @@ import { import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; import { POST } from "@/handlers/turn-resume"; -import type { WaitUntilFn } from "@/handlers/types"; - -const testWaitUntil: WaitUntilFn = (task) => { - waitUntilCallbacks.push(typeof task === "function" ? task : () => task); -}; +import { + createWaitUntilCollector, + type WaitUntilCollector, +} from "../../fixtures/wait-until"; + +let waitUntil: WaitUntilCollector; + +function postTurnResumeRequest(): Promise { + return POST( + new Request("https://example.com/api/internal/turn-resume", { + method: "POST", + }), + waitUntil.fn, + { scheduleTurnTimeoutResume: scheduleTurnTimeoutResumeMock }, + ); +} describe("turn resume handler", () => { beforeEach(async () => { - waitUntilCallbacks.length = 0; + waitUntil = createWaitUntilCollector(); resumeSlackTurnMock.mockReset(); scheduleTurnTimeoutResumeMock.mockReset(); verifyTurnTimeoutResumeRequestMock.mockReset(); @@ -75,15 +83,10 @@ describe("turn resume handler", () => { it("rejects unauthenticated internal resume callbacks", async () => { verifyTurnTimeoutResumeRequestMock.mockResolvedValue(undefined); - const response = await POST( - new Request("https://example.com/api/internal/turn-resume", { - method: "POST", - }), - testWaitUntil, - ); + const response = await postTurnResumeRequest(); expect(response.status).toBe(401); - expect(waitUntilCallbacks).toHaveLength(0); + expect(waitUntil.pendingCount()).toBe(0); }); it("drops stale callbacks after the resume lock is acquired", async () => { @@ -158,15 +161,10 @@ describe("turn resume handler", () => { expect(await args.beforeStart?.()).toBe(false); }); - const response = await POST( - new Request("https://example.com/api/internal/turn-resume", { - method: "POST", - }), - testWaitUntil, - ); + const response = await postTurnResumeRequest(); expect(response.status).toBe(202); - await waitUntilCallbacks[0]?.(); + await waitUntil.flush(); expect(scheduleTurnTimeoutResumeMock).not.toHaveBeenCalled(); }); @@ -246,15 +244,10 @@ describe("turn resume handler", () => { ); }); - const response = await POST( - new Request("https://example.com/api/internal/turn-resume", { - method: "POST", - }), - testWaitUntil, - ); + const response = await postTurnResumeRequest(); expect(response.status).toBe(202); - await waitUntilCallbacks[0]?.(); + await waitUntil.flush(); expect(scheduleTurnTimeoutResumeMock).toHaveBeenCalledWith({ conversationId, @@ -328,17 +321,12 @@ describe("turn resume handler", () => { .mockRejectedValueOnce(new ResumeTurnBusyError(conversationId)) .mockResolvedValueOnce(undefined); - const response = await POST( - new Request("https://example.com/api/internal/turn-resume", { - method: "POST", - }), - testWaitUntil, - ); + const response = await postTurnResumeRequest(); expect(response.status).toBe(202); - const task = waitUntilCallbacks[0]?.(); + const flush = waitUntil.flush(); await vi.runOnlyPendingTimersAsync(); - await task; + await flush; expect(resumeSlackTurnMock).toHaveBeenCalledTimes(2); }); @@ -408,17 +396,12 @@ describe("turn resume handler", () => { new ResumeTurnBusyError(conversationId), ); - const response = await POST( - new Request("https://example.com/api/internal/turn-resume", { - method: "POST", - }), - testWaitUntil, - ); + const response = await postTurnResumeRequest(); expect(response.status).toBe(202); - const task = waitUntilCallbacks[0]?.(); + const flush = waitUntil.flush(); await vi.runAllTimersAsync(); - await task; + await flush; expect(resumeSlackTurnMock).toHaveBeenCalledTimes(4); expect(scheduleTurnTimeoutResumeMock).toHaveBeenCalledWith({ @@ -512,15 +495,10 @@ describe("turn resume handler", () => { await runArgs.onSuccess?.(reply); }); - const response = await POST( - new Request("https://example.com/api/internal/turn-resume", { - method: "POST", - }), - testWaitUntil, - ); + const response = await postTurnResumeRequest(); expect(response.status).toBe(202); - await waitUntilCallbacks[0]?.(); + await waitUntil.flush(); expect(scheduleTurnTimeoutResumeMock).not.toHaveBeenCalled(); @@ -623,15 +601,10 @@ describe("turn resume handler", () => { } }); - const response = await POST( - new Request("https://example.com/api/internal/turn-resume", { - method: "POST", - }), - testWaitUntil, - ); + const response = await postTurnResumeRequest(); expect(response.status).toBe(202); - await waitUntilCallbacks[0]?.(); + await waitUntil.flush(); expect(scheduleTurnTimeoutResumeMock).toHaveBeenCalledWith({ conversationId, diff --git a/packages/junior/tests/unit/runtime/respond-error-path.test.ts b/packages/junior/tests/unit/runtime/respond-error-path.test.ts index c74b68991..0d86ea3b3 100644 --- a/packages/junior/tests/unit/runtime/respond-error-path.test.ts +++ b/packages/junior/tests/unit/runtime/respond-error-path.test.ts @@ -1,4 +1,8 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterAll, describe, expect, it, vi } from "vitest"; + +const originalAiModel = process.env.AI_MODEL; + +process.env.AI_MODEL = "openai/gpt-5.4"; vi.mock("@/chat/skills", () => ({ discoverSkills: vi.fn(async () => { @@ -8,16 +12,18 @@ vi.mock("@/chat/skills", () => ({ parseSkillInvocation: vi.fn(), })); +const { generateAssistantReply } = await import("@/chat/respond"); + describe("generateAssistantReply error path", () => { - afterEach(() => { - vi.unstubAllEnvs(); + afterAll(() => { + if (originalAiModel === undefined) { + delete process.env.AI_MODEL; + } else { + process.env.AI_MODEL = originalAiModel; + } }); it("preserves sandbox dependency hash on non-retryable failures", async () => { - vi.resetModules(); - vi.stubEnv("AI_MODEL", "openai/gpt-5.4"); - const { generateAssistantReply } = await import("@/chat/respond"); - const reply = await generateAssistantReply("hello", { sandbox: { sandboxId: "sb-123", @@ -32,4 +38,14 @@ describe("generateAssistantReply error path", () => { expect(reply.diagnostics.modelId).toBe("openai/gpt-5.4"); expect(reply.diagnostics.thinkingLevel).toBeUndefined(); }, 10_000); + + it("propagates pre-commit failures when durable input commit is required", async () => { + await expect( + generateAssistantReply("hello", { + onInputCommitted: async () => { + throw new Error("input should not commit before startup succeeds"); + }, + }), + ).rejects.toThrow("discover failed"); + }, 10_000); }); diff --git a/packages/junior/tests/unit/slack/slack-runtime.test.ts b/packages/junior/tests/unit/slack/slack-runtime.test.ts index deca28ead..8414c98cb 100644 --- a/packages/junior/tests/unit/slack/slack-runtime.test.ts +++ b/packages/junior/tests/unit/slack/slack-runtime.test.ts @@ -56,12 +56,15 @@ describe("createSlackTurnRuntime", () => { await runtime.handleNewMention(thread, message); expect(thread.subscribeCalls).toBe(1); - expect(deps.replyToThread).toHaveBeenCalledWith(thread, message, { - beforeFirstResponsePost: undefined, - explicitMention: true, - onToolInvocation: expect.any(Function), - queuedMessages: [], - }); + expect(deps.replyToThread).toHaveBeenCalledWith( + thread, + message, + expect.objectContaining({ + explicitMention: true, + onToolInvocation: expect.any(Function), + queuedMessages: [], + }), + ); }); it("forwards queued SDK context as ordered turn messages", async () => { @@ -108,39 +111,6 @@ describe("createSlackTurnRuntime", () => { }); describe("handleSubscribedMessage", () => { - it("calls prepareTurnState → persistPreparedState → decideSubscribedReply → replyToThread in order", async () => { - const callOrder: string[] = []; - const deps = createMockDeps({ - prepareTurnState: vi.fn(async () => { - callOrder.push("prepareTurnState"); - return { prepared: true }; - }), - persistPreparedState: vi.fn(async () => { - callOrder.push("persistPreparedState"); - }), - decideSubscribedReply: vi.fn(async () => { - callOrder.push("decideSubscribedReply"); - return { shouldReply: true, reason: "test" }; - }), - replyToThread: vi.fn(async () => { - callOrder.push("replyToThread"); - }), - withSpan: vi.fn(async (_n, _o, _c, cb) => cb()), - }); - const runtime = createSlackTurnRuntime(deps); - const thread = createTestThread({}); - const message = createTestMessage({}); - - await runtime.handleSubscribedMessage(thread, message); - - expect(callOrder).toEqual([ - "prepareTurnState", - "persistPreparedState", - "decideSubscribedReply", - "replyToThread", - ]); - }); - it("passes stripped text via stripLeadingBotMention to prepareTurnState", async () => { const deps = createMockDeps({ stripLeadingBotMention: vi.fn(() => "stripped text"), diff --git a/policies/README.md b/policies/README.md index fcc064dac..6030f1cb3 100644 --- a/policies/README.md +++ b/policies/README.md @@ -10,6 +10,7 @@ Good policy topics: - code comments and docstrings - frontend component styling - testing expectations +- test adapters and harnesses - naming conventions - interface design - migration hygiene diff --git a/policies/test-adapters.md b/policies/test-adapters.md new file mode 100644 index 000000000..16d2fb8f4 --- /dev/null +++ b/policies/test-adapters.md @@ -0,0 +1,27 @@ +# Test Adapters + +## Intent + +Tests should be easy to write because the repo provides faithful test adapters for common boundaries, not because each test invents its own mocks. Django's test suite is a useful model: it gives tests a client, isolated state, explicit environment overrides, observable outboxes, and runner tools for finding leakage. + +## Policy + +- Start from `specs/testing.md` for layer selection; use this policy for the fixture and adapter shape inside that layer. +- Prefer shared test adapters over one-off mocks when a boundary recurs across tests. +- A test adapter should implement the production-facing contract closely enough that tests can inject real payloads and observe resulting effects. +- Give adapters small, role-specific introspection methods such as `queuedMessages()`, `messages()`, or `fileUploads()`. Do not expose broad mutable internals. +- Model external side effects as outboxes or captured deliveries that are reset between tests. +- Model request ingress with signed/request-shaped clients instead of hand-built `Request` objects in every test. +- Model background work with collectors that follow production scheduling semantics and require tests to flush explicitly. +- Centralize temporary environment or configuration overrides in helpers that restore state automatically. +- Make isolation explicit. Tests that use shared resources, fake clocks, singleton state, or process-global configuration must reset them locally or opt into an isolated/serial harness. +- Keep test-only capabilities out of production singletons. Prefer injected ports, local factories, and test adapters over `setForTests` globals or module mocks. +- Add adapter behavior only for a real recurring test need, and keep it named after the user-visible boundary rather than the implementation mechanism. +- When a suite fails only under order, shuffle, reverse, or parallel load, treat that as a test-isolation bug unless proven otherwise. + +## Exceptions + +- A local stub is acceptable for one-off pure unit logic when the boundary is not shared and the behavior is deterministic. +- Module mocks are acceptable at the one explicitly allowed boundary for a test layer, such as the deterministic fake agent boundary in integration tests. +- A route harness may defer `waitUntil` execution when the contract under test is the response/ack boundary before background work; make the deferred flush explicit. +- Very low-level adapter contract tests may inspect raw captured payloads when the payload shape itself is the contract under test. From a60fc6e1fb1b534c10f3078359a8be4a6e485188 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 12:05:04 +0200 Subject: [PATCH 38/45] fix(slack): Track turn processing reactions Replace automatic processing eyes with a completion check when Slack turns finish. Leave parked, skipped, and failed turns without completion so reactions match the lifecycle. Restore requester credential context when timeout resumes rebuild reply context after the rebase. Co-Authored-By: GPT-5 Codex --- .../src/chat/runtime/processing-reaction.ts | 54 +++++-- .../junior/src/chat/runtime/reply-executor.ts | 3 + .../junior/src/chat/runtime/slack-resume.ts | 6 +- .../junior/src/chat/runtime/slack-runtime.ts | 152 ++++++++++++++++-- .../src/chat/runtime/timeout-resume-runner.ts | 6 + .../mcp-auth-runtime-slack.test.ts | 22 ++- .../mcp-oauth-callback-slack.test.ts | 7 + .../integration/oauth-callback-slack.test.ts | 18 ++- ...onversation-turn-steering-behavior.test.ts | 42 +++++ .../processing-reaction-behavior.test.ts | 48 +++--- specs/slack-agent-delivery.md | 14 +- 11 files changed, 301 insertions(+), 71 deletions(-) diff --git a/packages/junior/src/chat/runtime/processing-reaction.ts b/packages/junior/src/chat/runtime/processing-reaction.ts index e9948998e..ad3a4c148 100644 --- a/packages/junior/src/chat/runtime/processing-reaction.ts +++ b/packages/junior/src/chat/runtime/processing-reaction.ts @@ -9,14 +9,17 @@ import { getChannelId, getMessageTs } from "@/chat/runtime/thread-context"; import type { TurnToolInvocation } from "@/chat/runtime/turn-input"; const PROCESSING_REACTION_EMOJI = "eyes"; +const COMPLETED_REACTION_EMOJI = "white_check_mark"; /** Controls the automatic Slack processing reaction lifecycle for one message. */ export interface ProcessingReactionSession { + complete: () => Promise; keep: () => void; stop: () => Promise; } const noProcessingReaction: ProcessingReactionSession = { + complete: async () => undefined, keep: () => undefined, stop: async () => undefined, }; @@ -104,34 +107,65 @@ export async function startSlackProcessingReactionForMessage(args: { } let shouldRemove = true; + const removeProcessingReaction = async (): Promise => { + if (!shouldRemove) { + return false; + } + + try { + await removeReactionFromMessage({ + channelId: args.channelId, + timestamp: args.timestamp, + emoji: PROCESSING_REACTION_EMOJI, + }); + return true; + } catch (error) { + args.logException( + error, + "slack_processing_reaction_remove_failed", + args.logContext, + { + "app.slack.action": "reactions.remove", + "messaging.message.id": args.timestamp, + ...getSlackErrorObservabilityAttributes(error), + }, + "Failed to remove Slack processing reaction", + ); + return false; + } + }; + return { - keep: () => { - shouldRemove = false; - }, - stop: async () => { - if (!shouldRemove) { + complete: async () => { + if (!(await removeProcessingReaction())) { return; } try { - await removeReactionFromMessage({ + await addReactionToMessage({ channelId: args.channelId, timestamp: args.timestamp, - emoji: PROCESSING_REACTION_EMOJI, + emoji: COMPLETED_REACTION_EMOJI, }); } catch (error) { args.logException( error, - "slack_processing_reaction_remove_failed", + "slack_processing_reaction_complete_failed", args.logContext, { - "app.slack.action": "reactions.remove", + "app.slack.action": "reactions.add", "messaging.message.id": args.timestamp, ...getSlackErrorObservabilityAttributes(error), }, - "Failed to remove Slack processing reaction", + "Failed to add Slack completed reaction", ); } }, + keep: () => { + shouldRemove = false; + }, + stop: async () => { + await removeProcessingReaction(); + }, }; } diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index 3967f4c88..d193fd962 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -272,6 +272,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { explicitMention?: boolean; onInputCommitted?: () => Promise; onToolInvocation?: (invocation: TurnToolInvocation) => void; + onTurnCompleted?: () => Promise; onTurnStatePersisted?: () => Promise; preparedState?: PreparedTurnState; queuedMessages?: QueuedTurnMessage[]; @@ -405,6 +406,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }); await options.onTurnStatePersisted?.(); await options.onInputCommitted?.(); + await options.onTurnCompleted?.(); return; } if (conversationId && activeTurnId) { @@ -944,6 +946,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { "Agent turn completed", ); } + await options.onTurnCompleted?.(); } catch (error) { if (isCooperativeTurnYieldError(error)) { shouldPersistFailureState = false; diff --git a/packages/junior/src/chat/runtime/slack-resume.ts b/packages/junior/src/chat/runtime/slack-resume.ts index be70bb2ca..5a0d6572c 100644 --- a/packages/junior/src/chat/runtime/slack-resume.ts +++ b/packages/junior/src/chat/runtime/slack-resume.ts @@ -422,7 +422,11 @@ export async function resumeSlackTurn( }; } } finally { - await processingReaction?.stop(); + if (finalReplyDelivered) { + await processingReaction?.complete(); + } else { + await processingReaction?.stop(); + } await stateAdapter.releaseLock(lock); } diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index c9451c0c7..f31abf56b 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -28,6 +28,7 @@ import { startSlackProcessingReaction, type ProcessingReactionSession, } from "@/chat/runtime/processing-reaction"; +import { getMessageTs } from "@/chat/runtime/thread-context"; import { combineTurnText, type PrepareTurnStateInput, @@ -150,6 +151,7 @@ export interface SlackTurnRuntimeDependencies { explicitMention?: boolean; onInputCommitted?: () => Promise; onToolInvocation?: (invocation: TurnToolInvocation) => void; + onTurnCompleted?: () => Promise; onTurnStatePersisted?: () => Promise; preparedState?: TPreparedState; queuedMessages?: QueuedTurnMessage[]; @@ -217,6 +219,7 @@ function createSteeringMessageDrain( hooks: ReplyHooks | undefined, options: { explicitMention: boolean; + onMessagesAccepted?: (messages: Message[]) => Promise; stripLeadingBotMention: SlackTurnRuntimeDependencies["stripLeadingBotMention"]; }, ): @@ -229,9 +232,15 @@ function createSteeringMessageDrain( } return async (inject) => { + let acceptedMessages: Message[] | undefined; const drained = await hooks.drainSteeringMessages!(async (messages) => { await inject(getQueuedMessagesFromSlackMessages(messages, options)); + acceptedMessages = messages; + await options.onMessagesAccepted?.(messages); }); + if (!acceptedMessages) { + await options.onMessagesAccepted?.(drained); + } return getQueuedMessagesFromSlackMessages(drained, options); }; } @@ -306,6 +315,60 @@ export function createSlackTurnRuntime< }; }; + const stopProcessingReactions = async ( + processingReactions: ProcessingReactionSession[], + ): Promise => { + await Promise.all(processingReactions.map((reaction) => reaction.stop())); + }; + + const completeProcessingReactions = async ( + processingReactions: ProcessingReactionSession[], + ): Promise => { + await Promise.all( + processingReactions.map((reaction) => reaction.complete()), + ); + }; + + const createProcessingReactionTracker = (thread: Thread) => { + const processingReactions: ProcessingReactionSession[] = []; + const processingReactionByMessage = new Map< + string, + ProcessingReactionSession + >(); + + return { + start: async ( + context: RuntimeLogContext, + targetMessage: Message, + ): Promise => { + const channelId = deps.getChannelId(thread, targetMessage); + const messageTs = getMessageTs(targetMessage); + const reactionKey = + channelId && messageTs ? `${channelId}:${messageTs}` : undefined; + if (reactionKey) { + const existing = processingReactionByMessage.get(reactionKey); + if (existing) { + return existing; + } + } + + const started = await startSlackProcessingReaction({ + thread, + message: targetMessage, + logException: deps.logException, + logContext: context, + }); + processingReactions.push(started); + if (reactionKey) { + processingReactionByMessage.set(reactionKey, started); + } + return started; + }, + completeAll: () => completeProcessingReactions(processingReactions), + stopAll: () => stopProcessingReactions(processingReactions), + }; + }; + const postFallbackErrorReplyWithLogging = async (args: { thread: Thread; errorContext: RuntimeLogContext; @@ -379,7 +442,12 @@ export function createSlackTurnRuntime< message: Message, hooks?: ReplyHooks, ): Promise { + const processingReactions = createProcessingReactionTracker(thread); let processingReaction: ProcessingReactionSession | undefined; + let completed = false; + const onTurnCompleted = async (): Promise => { + completed = true; + }; try { const threadId = deps.getThreadId(thread, message); const channelId = deps.getChannelId(thread, message); @@ -391,12 +459,7 @@ export function createSlackTurnRuntime< requesterUserName: message.author.userName, runId, }); - processingReaction = await startSlackProcessingReaction({ - thread, - message, - logException: deps.logException, - logContext: context, - }); + processingReaction = await processingReactions.start(context, message); const toolInvocationHook = createToolInvocationHook( processingReaction, hooks, @@ -408,16 +471,40 @@ export function createSlackTurnRuntime< explicitMention: true, stripLeadingBotMention: deps.stripLeadingBotMention, }); + let queuedProcessingReactionsStarted = false; + const startQueuedProcessingReactions = async (): Promise => { + if (queuedProcessingReactionsStarted) { + return; + } + queuedProcessingReactionsStarted = true; + await Promise.all( + queuedMessages.map((queued) => + processingReactions.start(context, queued.message), + ), + ); + }; + const onInputCommitted = async (): Promise => { + await hooks?.onInputCommitted?.(); + await startQueuedProcessingReactions(); + }; const drainSteeringMessages = createSteeringMessageDrain(hooks, { explicitMention: true, + onMessagesAccepted: async (messages) => { + await Promise.all( + messages.map((drainedMessage) => + processingReactions.start(context, drainedMessage), + ), + ); + }, stripLeadingBotMention: deps.stripLeadingBotMention, }); await deps.replyToThread(thread, message, { explicitMention: true, beforeFirstResponsePost: hooks?.beforeFirstResponsePost, queuedMessages, - onInputCommitted: hooks?.onInputCommitted, + onInputCommitted, onToolInvocation: toolInvocationHook, + onTurnCompleted, drainSteeringMessages, onTurnStatePersisted: hooks?.onTurnStatePersisted, shouldYield: hooks?.shouldYield, @@ -472,7 +559,11 @@ export function createSlackTurnRuntime< "Failed to post fallback error reply for mention handler", }); } finally { - await processingReaction?.stop(); + if (completed) { + await processingReactions.completeAll(); + } else { + await processingReactions.stopAll(); + } } }, @@ -481,7 +572,12 @@ export function createSlackTurnRuntime< message: Message, hooks?: ReplyHooks, ): Promise { + const processingReactions = createProcessingReactionTracker(thread); let processingReaction: ProcessingReactionSession | undefined; + let completed = false; + const onTurnCompleted = async (): Promise => { + completed = true; + }; try { const threadId = deps.getThreadId(thread, message); const channelId = deps.getChannelId(thread, message); @@ -521,6 +617,13 @@ export function createSlackTurnRuntime< }); const drainSteeringMessages = createSteeringMessageDrain(hooks, { explicitMention: Boolean(message.isMention), + onMessagesAccepted: async (messages) => { + await Promise.all( + messages.map((drainedMessage) => + processingReactions.start(turnContext, drainedMessage), + ), + ); + }, stripLeadingBotMention: deps.stripLeadingBotMention, }); const combinedText = combineTurnText(queuedMessages, currentText); @@ -608,12 +711,26 @@ export function createSlackTurnRuntime< return; } - processingReaction = await startSlackProcessingReaction({ - thread, + processingReaction = await processingReactions.start( + turnContext, message, - logException: deps.logException, - logContext: turnContext, - }); + ); + let queuedProcessingReactionsStarted = false; + const startQueuedProcessingReactions = async (): Promise => { + if (queuedProcessingReactionsStarted) { + return; + } + queuedProcessingReactionsStarted = true; + await Promise.all( + queuedMessages.map((queued) => + processingReactions.start(turnContext, queued.message), + ), + ); + }; + const onInputCommitted = async (): Promise => { + await hooks?.onInputCommitted?.(); + await startQueuedProcessingReactions(); + }; const toolInvocationHook = createToolInvocationHook( processingReaction, hooks, @@ -624,8 +741,9 @@ export function createSlackTurnRuntime< preparedState, beforeFirstResponsePost: hooks?.beforeFirstResponsePost, queuedMessages, - onInputCommitted: hooks?.onInputCommitted, + onInputCommitted, onToolInvocation: toolInvocationHook, + onTurnCompleted, drainSteeringMessages, onTurnStatePersisted: hooks?.onTurnStatePersisted, shouldYield: hooks?.shouldYield, @@ -681,7 +799,11 @@ export function createSlackTurnRuntime< "Failed to post fallback error reply for subscribed message handler", }); } finally { - await processingReaction?.stop(); + if (completed) { + await processingReactions.completeAll(); + } else { + await processingReactions.stopAll(); + } } }, diff --git a/packages/junior/src/chat/runtime/timeout-resume-runner.ts b/packages/junior/src/chat/runtime/timeout-resume-runner.ts index c459449f1..ca639867d 100644 --- a/packages/junior/src/chat/runtime/timeout-resume-runner.ts +++ b/packages/junior/src/chat/runtime/timeout-resume-runner.ts @@ -197,6 +197,12 @@ export async function resumeTimedOutTurn( messageText: userMessage.text, messageTs: getTurnUserSlackMessageTs(userMessage), replyContext: { + credentialContext: { + actor: { + type: "user", + userId: userMessage.author.userId, + }, + }, requester: { userId: userMessage.author.userId, userName: userMessage.author.userName, diff --git a/packages/junior/tests/integration/mcp-auth-runtime-slack.test.ts b/packages/junior/tests/integration/mcp-auth-runtime-slack.test.ts index 601c169e6..c5b66f67f 100644 --- a/packages/junior/tests/integration/mcp-auth-runtime-slack.test.ts +++ b/packages/junior/tests/integration/mcp-auth-runtime-slack.test.ts @@ -269,21 +269,28 @@ async function mirrorThreadStateToAdapter(thread: TestThread): Promise { function expectProcessingReactionLifecycles(args: { channel: string; + completedCount?: number; count: number; timestamp: string; }): void { - const call = () => + const call = (name: string) => expect.objectContaining({ params: expect.objectContaining({ channel: args.channel, timestamp: args.timestamp, - name: "eyes", + name, }), }); - const expected = Array.from({ length: args.count }, call); + const eyes = Array.from({ length: args.count }, () => call("eyes")); + const completed = Array.from({ length: args.completedCount ?? 0 }, () => + call("white_check_mark"), + ); - expect(getCapturedSlackApiCalls("reactions.add")).toEqual(expected); - expect(getCapturedSlackApiCalls("reactions.remove")).toEqual(expected); + expect(getCapturedSlackApiCalls("reactions.add")).toEqual([ + ...eyes, + ...completed, + ]); + expect(getCapturedSlackApiCalls("reactions.remove")).toEqual(eyes); } describe("mcp auth runtime slack integration", () => { @@ -311,14 +318,14 @@ describe("mcp auth runtime slack integration", () => { await stateAdapterModule.disconnectStateAdapter(); await stateAdapterModule.getStateAdapter().connect(); - }); + }, 45_000); afterEach(async () => { await stateAdapterModule?.disconnectStateAdapter(); await pluginApp?.cleanup(); pluginApp = undefined; process.env = { ...ORIGINAL_ENV }; - }); + }, 45_000); it("parks an MCP auth challenge from the real Slack runtime and resumes after OAuth callback", async () => { const threadId = "slack:C123:1700000000.001"; @@ -546,6 +553,7 @@ describe("mcp auth runtime slack integration", () => { channel: "C123", timestamp: "1700000000.002", count: 2, + completedCount: 1, }); }); diff --git a/packages/junior/tests/integration/mcp-oauth-callback-slack.test.ts b/packages/junior/tests/integration/mcp-oauth-callback-slack.test.ts index cf79f2171..38a2467c3 100644 --- a/packages/junior/tests/integration/mcp-oauth-callback-slack.test.ts +++ b/packages/junior/tests/integration/mcp-oauth-callback-slack.test.ts @@ -530,6 +530,13 @@ describe("mcp oauth callback slack integration", () => { expect.objectContaining({ params: expect.objectContaining({ timestamp: "1700000000.0052", + name: "eyes", + }), + }), + expect.objectContaining({ + params: expect.objectContaining({ + timestamp: "1700000000.0052", + name: "white_check_mark", }), }), ]); diff --git a/packages/junior/tests/integration/oauth-callback-slack.test.ts b/packages/junior/tests/integration/oauth-callback-slack.test.ts index 246052520..e3f41fc7c 100644 --- a/packages/junior/tests/integration/oauth-callback-slack.test.ts +++ b/packages/junior/tests/integration/oauth-callback-slack.test.ts @@ -57,14 +57,14 @@ describe("oauth callback slack integration", () => { turnSessionStoreModule = await import("@/chat/state/turn-session"); await stateAdapterModule.disconnectStateAdapter(); await stateAdapterModule.getStateAdapter().connect(); - }); + }, 45_000); afterEach(async () => { await stateAdapterModule?.disconnectStateAdapter(); await pluginApp?.cleanup(); pluginApp = undefined; process.env = { ...ORIGINAL_ENV }; - }); + }, 45_000); it("publishes app home through the Slack MSW harness after generic OAuth callback", async () => { await stateAdapterModule @@ -324,6 +324,13 @@ describe("oauth callback slack integration", () => { name: "eyes", }), }), + expect.objectContaining({ + params: expect.objectContaining({ + channel: "C123", + timestamp: "1700000000.010", + name: "white_check_mark", + }), + }), ]); expect(getCapturedSlackApiCalls("reactions.remove")).toEqual([ expect.objectContaining({ @@ -490,6 +497,13 @@ describe("oauth callback slack integration", () => { expect.objectContaining({ params: expect.objectContaining({ timestamp: "1700000000.0112", + name: "eyes", + }), + }), + expect.objectContaining({ + params: expect.objectContaining({ + timestamp: "1700000000.0112", + name: "white_check_mark", }), }), ]); diff --git a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts index bd526039b..778a36286 100644 --- a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts +++ b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts @@ -52,6 +52,28 @@ function makeDiagnostics() { }; } +function reactionTargets( + calls: ReturnType, +) { + return calls + .map((call) => ({ + channel: call.params.channel, + name: call.params.name, + timestamp: call.params.timestamp, + })) + .sort((left, right) => + `${left.channel}:${left.timestamp}:${left.name}`.localeCompare( + `${right.channel}:${right.timestamp}:${right.name}`, + ), + ); +} + +function reactionTargetsByName(name: string) { + return reactionTargets( + slackApiOutbox.reactionAdds().filter((call) => call.params.name === name), + ); +} + function createTurnHarness(args: { generateAssistantReply: ReplyExecutorServices["generateAssistantReply"]; state: StateAdapter; @@ -226,6 +248,26 @@ describe("Slack behavior: durable turn steering", () => { work?.messages.every((message) => message.injectedAtMs !== undefined), ).toBe(true); expect(work?.needsRun).toBe(false); + + const expectedReactionTargets = (name: string) => + [THREAD_TS, "1712345.000200", "1712345.000300", "1712345.000400"].map( + (timestamp) => ({ + channel: CHANNEL_ID, + name, + timestamp, + }), + ); + const expectedProcessingReactions = expectedReactionTargets("eyes"); + const expectedCompletedReactions = + expectedReactionTargets("white_check_mark"); + + expect(reactionTargetsByName("eyes")).toEqual(expectedProcessingReactions); + expect(reactionTargets(slackApiOutbox.reactionRemovals())).toEqual( + expectedProcessingReactions, + ); + expect(reactionTargetsByName("white_check_mark")).toEqual( + expectedCompletedReactions, + ); }); it("keeps the mailbox pending when the agent fails before input commit", async () => { diff --git a/packages/junior/tests/integration/slack/processing-reaction-behavior.test.ts b/packages/junior/tests/integration/slack/processing-reaction-behavior.test.ts index 53f94124f..145915142 100644 --- a/packages/junior/tests/integration/slack/processing-reaction-behavior.test.ts +++ b/packages/junior/tests/integration/slack/processing-reaction-behavior.test.ts @@ -18,8 +18,18 @@ function successDiagnostics(toolCalls: string[] = []) { }; } +function reactionCall(name: string, timestamp: string) { + return expect.objectContaining({ + params: expect.objectContaining({ + channel: "C_PROCESSING", + timestamp, + name, + }), + }); +} + describe("Slack behavior: processing reaction", () => { - it("adds eyes before mention work and removes it after the reply", async () => { + it("adds eyes before mention work and marks the message complete after the reply", async () => { const { slackRuntime } = createTestChatRuntime({ services: { replyExecutor: { @@ -54,22 +64,11 @@ describe("Slack behavior: processing reaction", () => { ); expect(slackApiOutbox.reactionAdds()).toEqual([ - expect.objectContaining({ - params: expect.objectContaining({ - channel: "C_PROCESSING", - timestamp: "1700007001.000000", - name: "eyes", - }), - }), + reactionCall("eyes", "1700007001.000000"), + reactionCall("white_check_mark", "1700007001.000000"), ]); expect(slackApiOutbox.reactionRemovals()).toEqual([ - expect.objectContaining({ - params: expect.objectContaining({ - channel: "C_PROCESSING", - timestamp: "1700007001.000000", - name: "eyes", - }), - }), + reactionCall("eyes", "1700007001.000000"), ]); }); @@ -121,7 +120,7 @@ describe("Slack behavior: processing reaction", () => { expect(slackApiOutbox.reactionRemovals()).toHaveLength(0); }); - it("adds eyes after a subscribed message is approved and removes it after the reply", async () => { + it("adds eyes after a subscribed message is approved and marks the message complete after the reply", async () => { const { slackRuntime } = createTestChatRuntime({ services: { subscribedReplyPolicy: { @@ -170,22 +169,11 @@ describe("Slack behavior: processing reaction", () => { ); expect(slackApiOutbox.reactionAdds()).toEqual([ - expect.objectContaining({ - params: expect.objectContaining({ - channel: "C_PROCESSING", - timestamp: "1700007151.000000", - name: "eyes", - }), - }), + reactionCall("eyes", "1700007151.000000"), + reactionCall("white_check_mark", "1700007151.000000"), ]); expect(slackApiOutbox.reactionRemovals()).toEqual([ - expect.objectContaining({ - params: expect.objectContaining({ - channel: "C_PROCESSING", - timestamp: "1700007151.000000", - name: "eyes", - }), - }), + reactionCall("eyes", "1700007151.000000"), ]); }); diff --git a/specs/slack-agent-delivery.md b/specs/slack-agent-delivery.md index ba915706d..8d47aef68 100644 --- a/specs/slack-agent-delivery.md +++ b/specs/slack-agent-delivery.md @@ -130,12 +130,14 @@ Current rules: 1. DM and explicit-mention handlers add `:eyes:` before turn preparation or assistant execution. 2. Subscribed-thread handlers add `:eyes:` only after preflight and passive reply routing return `shouldReply: true`, immediately before assistant execution. -3. Skipped subscribed-thread messages, including passive no-reply and opt-out decisions, do not add or remove the automatic processing reaction. -4. Junior removes an automatic `:eyes:` reaction when the handler completes after the reaction has started, including reply, auth-pause, timeout-continuation, and fallback-error paths. -5. When an OAuth/MCP callback resumes an auth-paused request, Junior re-adds `:eyes:` to the original triggering Slack message while resumed processing runs, then removes it when the resumed handler completes. -6. Processing-reaction add and remove calls are best effort. Failures are observable but must not fail the turn or change reply routing. -7. The automatic processing reaction is runtime-owned. It must not be exposed as model progress, and it must not count as a successful user-requested reaction tool call. -8. If the assistant explicitly uses the Slack reaction tool to add `:eyes:` to the same inbound message, Junior leaves the reaction in place instead of removing the automatic acknowledgement. +3. Batched or steered Slack messages that Junior durably accepts into an active turn also get `:eyes:` so each handled user message has the same visible acknowledgement. +4. When the accepted message completes with a final delivered reply or successful side effect, Junior replaces the automatic `:eyes:` with `:white_check_mark:`. +5. Skipped subscribed-thread messages, including passive no-reply and opt-out decisions, do not add or remove an automatic reaction. +6. Junior removes an automatic `:eyes:` reaction without adding `:white_check_mark:` when the handler stops before completion, including auth-pause, timeout-continuation, cooperative-yield, and fallback-error paths. +7. When an OAuth/MCP callback resumes an auth-paused request, Junior re-adds `:eyes:` to the original triggering Slack message while resumed processing runs, then replaces it with `:white_check_mark:` only after the resumed final reply is delivered. +8. Processing-reaction add, remove, and completion calls are best effort. Failures are observable but must not fail the turn or change reply routing. +9. The automatic processing reaction is runtime-owned. It must not be exposed as model progress, and it must not count as a successful user-requested reaction tool call. +10. If the assistant explicitly uses the Slack reaction tool to add `:eyes:` to the same inbound message, Junior leaves the reaction in place instead of replacing the automatic acknowledgement. ### 6. Primary Reply Contract From 7e43a52efad5c9edc98ff02623c6dace7c8291e8 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 12:13:24 +0200 Subject: [PATCH 39/45] fix(runtime): Fail closed on missing input checkpoints Report whether running turn checkpoints actually persist before committing durable Slack mailbox input. Propagate lost-input ownership errors instead of allowing a successful turn without mailbox commit. Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond.ts | 28 +++++- .../src/chat/services/turn-session-record.ts | 6 +- .../runtime/respond-timeout-resume.test.ts | 16 +++- .../unit/services/turn-session-record.test.ts | 93 +++++++++++++------ 4 files changed, 110 insertions(+), 33 deletions(-) diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index bb42857e5..30f6623a8 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -85,6 +85,7 @@ import { mergeArtifactsState } from "@/chat/runtime/thread-state"; import { CooperativeTurnYieldError, RetryableTurnError, + TurnInputCommitLostError, isTurnInputCommitLostError, isRetryableTurnError, } from "@/chat/runtime/turn"; @@ -1126,7 +1127,7 @@ export async function generateAssistantReply( return false; } - await persistRunningSessionRecord({ + const persisted = await persistRunningSessionRecord({ channelName: context.correlation?.channelName, conversationId: sessionConversationId, sessionId, @@ -1136,9 +1137,24 @@ export async function generateAssistantReply( logContext: sessionRecordLogContext, requester, }); + if (!persisted) { + return false; + } + latestSafeBoundaryMessages = [...messages]; return true; }; + const requireDurableInputCheckpoint = async ( + messages: PiMessage[], + ): Promise => { + const persisted = await persistSafeBoundary(messages); + if (!persisted && context.onInputCommitted) { + throw new TurnInputCommitLostError( + `Durable turn input could not be checkpointed for conversation=${sessionConversationId ?? "unknown"} session=${sessionId ?? "unknown"}`, + ); + } + return persisted; + }; const drainSteeringMessages = async (): Promise => { if ( !context.drainSteeringMessages || @@ -1156,7 +1172,10 @@ export async function generateAssistantReply( if (piMessages.length === 0) { return; } - await persistSafeBoundary([...agent!.state.messages, ...piMessages]); + await requireDurableInputCheckpoint([ + ...agent!.state.messages, + ...piMessages, + ]); for (const message of piMessages) { agent!.steer(message); } @@ -1173,6 +1192,9 @@ export async function generateAssistantReply( ); } } catch (error) { + if (isTurnInputCommitLostError(error)) { + throw error; + } logWarn( "agent_turn_steering_messages_drain_failed", spanContext, @@ -1287,7 +1309,7 @@ export async function generateAssistantReply( timestamp: Date.now(), } as PiMessage; if (!resumedFromSessionRecord) { - const promptPersisted = await persistSafeBoundary([ + const promptPersisted = await requireDurableInputCheckpoint([ ...agent.state.messages, freshPromptMessage, ]); diff --git a/packages/junior/src/chat/services/turn-session-record.ts b/packages/junior/src/chat/services/turn-session-record.ts index 1b35e1bde..b8440e98a 100644 --- a/packages/junior/src/chat/services/turn-session-record.ts +++ b/packages/junior/src/chat/services/turn-session-record.ts @@ -134,9 +134,9 @@ export async function persistRunningSessionRecord(args: { loadedSkillNames?: string[]; logContext: SessionRecordLogContext; requester?: AgentTurnRequester; -}): Promise { +}): Promise { if (args.messages.length === 0 || !isContinuableBoundary(args.messages)) { - return; + return false; } try { @@ -165,6 +165,7 @@ export async function persistRunningSessionRecord(args: { ? { traceId: getActiveTraceId() ?? latestSessionRecord?.traceId } : {}), }); + return true; } catch (recordError) { logSessionRecordError( recordError, @@ -175,6 +176,7 @@ export async function persistRunningSessionRecord(args: { }, "Failed to persist running turn session record", ); + return false; } } diff --git a/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts b/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts index c1f5cbef3..e7c287396 100644 --- a/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts +++ b/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts @@ -187,7 +187,10 @@ vi.mock("@/chat/skills", async (importOriginal) => ({ })); import { generateAssistantReply } from "@/chat/respond"; -import { isRetryableTurnError } from "@/chat/runtime/turn"; +import { + isRetryableTurnError, + isTurnInputCommitLostError, +} from "@/chat/runtime/turn"; import { disconnectStateAdapter } from "@/chat/state/adapter"; import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; @@ -206,6 +209,17 @@ describe("generateAssistantReply timeout resume", () => { delete process.env.JUNIOR_STATE_ADAPTER; }); + it("rejects durable input when no prompt checkpoint can be persisted", async () => { + const onInputCommitted = vi.fn(); + + const error = await generateAssistantReply("help me", { + onInputCommitted, + }).catch((caught) => caught); + + expect(isTurnInputCommitLostError(error)).toBe(true); + expect(onInputCommitted).not.toHaveBeenCalled(); + }); + it("stores the last safe boundary and throws a retryable timeout error", async () => { const replyPromise = generateAssistantReply("help me", { requester: { userId: "U123" }, diff --git a/packages/junior/tests/unit/services/turn-session-record.test.ts b/packages/junior/tests/unit/services/turn-session-record.test.ts index 5a7647541..d8e136711 100644 --- a/packages/junior/tests/unit/services/turn-session-record.test.ts +++ b/packages/junior/tests/unit/services/turn-session-record.test.ts @@ -511,25 +511,29 @@ describe("persistAuthPauseSessionRecord", () => { } as PiMessage, ]; - await persistRunningSessionRecord({ - conversationId: "conversation-1", - sessionId: "turn-1", - sliceId: 1, - messages: userBoundary, - logContext: { - modelId: "test-model", - }, - }); + await expect( + persistRunningSessionRecord({ + conversationId: "conversation-1", + sessionId: "turn-1", + sliceId: 1, + messages: userBoundary, + logContext: { + modelId: "test-model", + }, + }), + ).resolves.toBe(true); - await persistRunningSessionRecord({ - conversationId: "conversation-1", - sessionId: "turn-1", - sliceId: 1, - messages: unsafeAssistantBoundary, - logContext: { - modelId: "test-model", - }, - }); + await expect( + persistRunningSessionRecord({ + conversationId: "conversation-1", + sessionId: "turn-1", + sliceId: 1, + messages: unsafeAssistantBoundary, + logContext: { + modelId: "test-model", + }, + }), + ).resolves.toBe(false); let sessionRecord = await getAgentTurnSessionRecord( "conversation-1", @@ -540,15 +544,17 @@ describe("persistAuthPauseSessionRecord", () => { piMessages: userBoundary, }); - await persistRunningSessionRecord({ - conversationId: "conversation-1", - sessionId: "turn-1", - sliceId: 1, - messages: toolResultBoundary, - logContext: { - modelId: "test-model", - }, - }); + await expect( + persistRunningSessionRecord({ + conversationId: "conversation-1", + sessionId: "turn-1", + sliceId: 1, + messages: toolResultBoundary, + logContext: { + modelId: "test-model", + }, + }), + ).resolves.toBe(true); sessionRecord = await getAgentTurnSessionRecord("conversation-1", "turn-1"); expect(sessionRecord).toMatchObject({ @@ -557,6 +563,39 @@ describe("persistAuthPauseSessionRecord", () => { }); }); + it("reports running record storage failures", async () => { + vi.doMock("@/chat/state/turn-session", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + upsertAgentTurnSessionRecord: vi.fn(async () => { + throw new Error("storage unavailable"); + }), + }; + }); + const { persistRunningSessionRecord } = + await import("@/chat/services/turn-session-record"); + + await expect( + persistRunningSessionRecord({ + conversationId: "conversation-storage-failure", + sessionId: "turn-storage-failure", + sliceId: 1, + messages: [ + { + role: "user", + content: [{ type: "text", text: "help me" }], + timestamp: 1, + }, + ], + logContext: { + modelId: "test-model", + }, + }), + ).resolves.toBe(false); + }); + it("promotes the latest running record when timeout capture has no messages", async () => { const { persistTimeoutSessionRecord, persistRunningSessionRecord } = await import("@/chat/services/turn-session-record"); From 5f05e09ef90d1ef93c65087ddc23b86922dadea0 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 12:21:02 +0200 Subject: [PATCH 40/45] fix(queue): Preserve recoverable work ownership Keep runnable conversation ids in the heartbeat recovery index when pruning overflow entries. Treat failed worker lease check-ins as lost ownership so in-flight work cannot complete after lease loss. Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/task-execution/store.ts | 13 +++- .../junior/src/chat/task-execution/worker.ts | 23 ++++-- .../task-execution/conversation-work.test.ts | 71 +++++++++++++++++++ 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index d3017a67c..2ee309fec 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -337,9 +337,20 @@ async function addToIndex( if (existing.includes(conversationId)) { return; } + const indexed = [...existing, conversationId]; + const remove = new Set(); + for (const id of indexed) { + if (indexed.length - remove.size <= CONVERSATION_WORK_INDEX_MAX_LENGTH) { + break; + } + const work = await readWorkState(state, id); + if (!work || !shouldKeepIndexed(work)) { + remove.add(id); + } + } await state.set( indexKey(), - [...existing, conversationId].slice(-CONVERSATION_WORK_INDEX_MAX_LENGTH), + indexed.filter((id) => !remove.has(id)), JUNIOR_THREAD_STATE_TTL_MS, ); }); diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index 3537474b1..c901b4062 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -87,6 +87,7 @@ async function sendWakeNudge(args: { function startLeaseCheckIn(args: { conversationId: string; leaseToken: string; + onLostLease: () => void; options: ProcessConversationWorkOptions; }): ReturnType { const timer = setInterval(() => { @@ -99,6 +100,7 @@ function startLeaseCheckIn(args: { }).then( (checkedIn) => { if (!checkedIn) { + args.onLostLease(); logWarn( "conversation_work_check_in_failed", { conversationId: args.conversationId }, @@ -172,9 +174,14 @@ export async function processConversationWork( const softYieldDeadlineMs = startedAtMs + (options.softYieldAfterMs ?? CONVERSATION_WORK_SOFT_YIELD_AFTER_MS); + let leaseLost = false; + const markLeaseLost = (): void => { + leaseLost = true; + }; const timer = startLeaseCheckIn({ conversationId, leaseToken: lease.leaseToken, + onLostLease: markLeaseLost, options, }); logInfo( @@ -190,14 +197,19 @@ export async function processConversationWork( const workerContext: ConversationWorkerContext = { conversationId, leaseToken: lease.leaseToken, - shouldYield: () => now(options) >= softYieldDeadlineMs, - checkIn: () => - checkInConversationWork({ + shouldYield: () => leaseLost || now(options) >= softYieldDeadlineMs, + checkIn: async () => { + const checkedIn = await checkInConversationWork({ conversationId, leaseToken: lease.leaseToken, nowMs: now(options), state: options.state, - }), + }); + if (!checkedIn) { + markLeaseLost(); + } + return checkedIn; + }, drainMailbox: (inject) => drainConversationMailbox({ conversationId, @@ -213,6 +225,9 @@ export async function processConversationWork( if (result.status === "lost_lease") { return { status: "lost_lease" }; } + if (leaseLost) { + return { status: "lost_lease" }; + } if (result.status === "yielded") { const yieldNowMs = now(options); const continuationMarked = await requestConversationContinuation({ diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index b46d04859..3211aa3fb 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -10,8 +10,10 @@ import { countPendingConversationMessages, drainConversationMailbox, getConversationWorkState, + listConversationWorkIds, markConversationMessagesInjected, requestConversationWork, + releaseConversationWork, startConversationWork, type InboundMessageRecord, } from "@/chat/task-execution/store"; @@ -152,6 +154,38 @@ describe("conversation work execution", () => { ]); }); + it("keeps runnable conversation ids when the recovery index overflows", async () => { + const state = getStateAdapter(); + await state.connect(); + const activeConversationId = "conversation-active"; + const newConversationId = "conversation-new"; + await requestConversationWork({ + conversationId: activeConversationId, + nowMs: 1_000, + state, + }); + await state.set( + "junior:conversation-work:index", + [ + activeConversationId, + ...Array.from({ length: 9_999 }, (_, index) => `stale-${index}`), + ], + 60_000, + ); + + await requestConversationWork({ + conversationId: newConversationId, + nowMs: 2_000, + state, + }); + + const ids = await listConversationWorkIds({ state }); + expect(ids).toContain(activeConversationId); + expect(ids).toContain(newConversationId); + expect(ids).not.toContain("stale-0"); + expect(ids).toHaveLength(10_000); + }); + it("defers duplicate queue nudges while a conversation lease is active", async () => { const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); @@ -417,6 +451,43 @@ describe("conversation work execution", () => { await expect(running).resolves.toEqual({ status: "completed" }); }); + it("reports lost lease after periodic check-in loses ownership", async () => { + vi.useFakeTimers({ now: 1_000 }); + const queue = createConversationWorkQueueTestAdapter(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + const entered = deferred<{ + leaseToken: string; + shouldYield: () => boolean; + }>(); + const finish = deferred(); + + const running = processConversationWork(CONVERSATION_ID, { + checkInIntervalMs: 15_000, + queue, + run: async (context) => { + await context.drainMailbox(async () => {}); + entered.resolve({ + leaseToken: context.leaseToken, + shouldYield: context.shouldYield, + }); + await finish.promise; + return { status: context.shouldYield() ? "yielded" : "completed" }; + }, + }); + const runningContext = await entered.promise; + + await releaseConversationWork({ + conversationId: CONVERSATION_ID, + leaseToken: runningContext.leaseToken, + nowMs: 2_000, + }); + await vi.advanceTimersByTimeAsync(15_000); + + expect(runningContext.shouldYield()).toBe(true); + finish.resolve(); + await expect(running).resolves.toEqual({ status: "lost_lease" }); + }); + it("requeues an expired conversation lease from heartbeat", async () => { const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); From 02ee6297a0386a00de3356f936446984d79660e9 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 13:51:36 +0200 Subject: [PATCH 41/45] fix(runtime): Fail terminal timeout resumes closed Keep terminal timeout resume failures on the error path after the session record reaches the slice cap. This prevents Slack delivery from treating an exhausted turn as a successful assistant reply containing an Error-prefixed message. Add a regression that seeds the durable session at the timeout cap and verifies the runtime throws while persisting the failed terminal record. Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/respond.ts | 4 ++ .../runtime/respond-timeout-resume.test.ts | 66 ++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/packages/junior/src/chat/respond.ts b/packages/junior/src/chat/respond.ts index 30f6623a8..96ea35960 100644 --- a/packages/junior/src/chat/respond.ts +++ b/packages/junior/src/chat/respond.ts @@ -1595,6 +1595,10 @@ export async function generateAssistantReply( }, ); } + throw new Error( + sessionRecord.errorMessage ?? + (error instanceof Error ? error.message : String(error)), + ); } // ── MCP auth pause → session continuation ──────────────────────── diff --git a/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts b/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts index e7c287396..eae215a3b 100644 --- a/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts +++ b/packages/junior/tests/unit/runtime/respond-timeout-resume.test.ts @@ -1,5 +1,6 @@ import { Buffer } from "node:buffer"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { PiMessage } from "@/chat/pi/messages"; const { promptAborted, promptMode } = vi.hoisted(() => ({ promptAborted: { value: false }, @@ -7,6 +8,7 @@ const { promptAborted, promptMode } = vi.hoisted(() => ({ value: "settlesAfterAbort" as | "settlesAfterAbort" | "hangsAfterAbort" + | "continueSettlesAfterAbort" | "providerRetryThenHangs", }, })); @@ -46,6 +48,16 @@ vi.mock("@earendil-works/pi-agent-core", () => { } async continue() { + if (promptMode.value === "continueSettlesAfterAbort") { + await new Promise((resolve) => { + this.resolveAbort = resolve; + }); + this.state.messages.push({ + role: "assistant", + content: [{ type: "text", text: "continued partial" }], + }); + return {}; + } if (promptMode.value === "providerRetryThenHangs") { await new Promise((resolve) => { this.resolveAbort = resolve; @@ -191,8 +203,12 @@ import { isRetryableTurnError, isTurnInputCommitLostError, } from "@/chat/runtime/turn"; +import { AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES } from "@/chat/services/turn-session-record"; import { disconnectStateAdapter } from "@/chat/state/adapter"; -import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { + getAgentTurnSessionRecord, + upsertAgentTurnSessionRecord, +} from "@/chat/state/turn-session"; describe("generateAssistantReply timeout resume", () => { beforeEach(async () => { @@ -260,6 +276,54 @@ describe("generateAssistantReply timeout resume", () => { ]); }); + it("throws terminal timeout failures instead of returning an error reply after the slice cap", async () => { + promptMode.value = "continueSettlesAfterAbort"; + const piMessages: PiMessage[] = [ + { + role: "user", + content: [{ type: "text", text: "keep trying" }], + timestamp: 1, + } as PiMessage, + ]; + await upsertAgentTurnSessionRecord({ + conversationId: "conversation-timeout-cap", + sessionId: "turn-timeout-cap", + sliceId: AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES, + state: "awaiting_resume", + piMessages, + resumeReason: "timeout", + }); + + const replyPromise = generateAssistantReply("help me", { + requester: { userId: "U123" }, + correlation: { + conversationId: "conversation-timeout-cap", + turnId: "turn-timeout-cap", + channelId: "C123", + threadTs: "1712345.0006", + }, + }).catch((caught) => caught); + + await vi.advanceTimersByTimeAsync(10_000); + const error = await replyPromise; + + expect(error).toBeInstanceOf(Error); + expect(error).not.toHaveProperty("text"); + expect(isRetryableTurnError(error, "turn_timeout_resume")).toBe(false); + expect(error.message).toContain("slice limit"); + + const sessionRecord = await getAgentTurnSessionRecord( + "conversation-timeout-cap", + "turn-timeout-cap", + ); + expect(sessionRecord).toMatchObject({ + state: "failed", + resumeReason: "timeout", + sliceId: AGENT_TURN_TIMEOUT_RESUME_MAX_SLICES, + errorMessage: expect.stringContaining("slice limit"), + }); + }); + it("records the effective request deadline timeout budget", async () => { const startedAtMs = Date.now(); const replyPromise = generateAssistantReply("help me", { From 07c54c491557a19d073604933516c55e55be0957 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 15:28:18 +0200 Subject: [PATCH 42/45] fix(queue): Avoid duplicate inbound wake nudges Skip duplicate inbound retries when the conversation already has a fresh queue marker. Repair stale or missing markers with a fresh idempotency key so failed handoffs still recover promptly. Add fake queue attempt introspection and Slack retry coverage so duplicate sends are visible in tests. Refs GH-470 Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/task-execution/store.ts | 30 ++++++++++- .../task-execution/conversation-work.test.ts | 52 +++++++++++++++++-- .../tests/fixtures/conversation-work.ts | 27 ++++++++-- ...onversation-turn-steering-behavior.test.ts | 52 +++++++++++++++++++ 4 files changed, 153 insertions(+), 8 deletions(-) diff --git a/packages/junior/src/chat/task-execution/store.ts b/packages/junior/src/chat/task-execution/store.ts index 2ee309fec..fcec5f321 100644 --- a/packages/junior/src/chat/task-execution/store.ts +++ b/packages/junior/src/chat/task-execution/store.ts @@ -87,6 +87,23 @@ export interface RequestConversationWorkResult { status: "created" | "updated"; } +function duplicateInboundNudgeIdempotencyKey( + message: InboundMessageRecord, + nowMs: number, +): string { + return `duplicate:${message.conversationId}:${message.inboundMessageId}:${nowMs}`; +} + +function hasRecentEnqueueMarker( + state: ConversationWorkState, + nowMs: number, +): boolean { + return ( + typeof state.lastEnqueuedAtMs === "number" && + state.lastEnqueuedAtMs + CONVERSATION_WORK_STALE_ENQUEUE_MS > nowMs + ); +} + function stateKey(conversationId: string): string { return `${CONVERSATION_WORK_PREFIX}:state:${conversationId}`; } @@ -517,9 +534,20 @@ export async function appendAndEnqueueInboundMessage(args: { nowMs, state: args.state, }); + let idempotencyKey = args.message.inboundMessageId; + if (appendResult.status === "duplicate") { + const work = await getConversationWorkState({ + conversationId: args.message.conversationId, + state: args.state, + }); + if (!work || hasRecentEnqueueMarker(work, nowMs)) { + return appendResult; + } + idempotencyKey = duplicateInboundNudgeIdempotencyKey(args.message, nowMs); + } const queueResult = await args.queue.send( { conversationId: args.message.conversationId }, - { idempotencyKey: args.message.inboundMessageId }, + { idempotencyKey }, ); await markConversationWorkEnqueued({ conversationId: args.message.conversationId, diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index 3211aa3fb..c59f67963 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -55,7 +55,7 @@ describe("conversation work execution", () => { vi.useRealTimers(); }); - it("stores inbound mailbox messages idempotently", async () => { + it("stores inbound mailbox messages idempotently without duplicate queue attempts", async () => { const queue = createConversationWorkQueueTestAdapter(); await expect( appendAndEnqueueInboundMessage({ @@ -72,7 +72,6 @@ describe("conversation work execution", () => { }), ).resolves.toMatchObject({ status: "duplicate", - queueMessageId: "queue-2", }); const state = await getConversationWorkState({ @@ -80,7 +79,32 @@ describe("conversation work execution", () => { }); expect(state?.messages).toHaveLength(1); expect(state ? countPendingConversationMessages(state) : 0).toBe(1); - expect(queue.sentRecords()).toHaveLength(2); + expect(queue.sendAttempts()).toHaveLength(1); + expect(queue.sentRecords()).toHaveLength(1); + }); + + it("repairs duplicate inbound work when no queue marker was recorded", async () => { + const queue = createConversationWorkQueueTestAdapter(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + appendAndEnqueueInboundMessage({ + message: inboundMessage("m1"), + nowMs: 62_000, + queue, + }), + ).resolves.toMatchObject({ + status: "duplicate", + queueMessageId: "queue-1", + }); + + expect(queue.sendAttempts()).toEqual([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `duplicate:${CONVERSATION_ID}:m1:62000`, + }, + ]); + expect(queue.sentRecords()).toEqual(queue.sendAttempts()); }); it("retries transient conversation work index lock contention", async () => { @@ -761,6 +785,28 @@ describe("conversation work execution", () => { ).resolves.toBe(false); }); + it("deduplicates accepted fake queue payloads by idempotency key", async () => { + const queue = createConversationWorkQueueTestAdapter(); + + await expect( + queue.send({ conversationId: CONVERSATION_ID }, { idempotencyKey: "m1" }), + ).resolves.toEqual({ messageId: "queue-1" }); + await expect( + queue.send({ conversationId: CONVERSATION_ID }, { idempotencyKey: "m1" }), + ).resolves.toEqual({ messageId: "queue-1" }); + + expect(queue.sendAttempts()).toEqual([ + { conversationId: CONVERSATION_ID, idempotencyKey: "m1" }, + { conversationId: CONVERSATION_ID, idempotencyKey: "m1" }, + ]); + expect(queue.sentRecords()).toEqual([ + { conversationId: CONVERSATION_ID, idempotencyKey: "m1" }, + ]); + expect(queue.queuedMessages()).toEqual([ + { conversationId: CONVERSATION_ID }, + ]); + }); + it("maps the generic queue port to Vercel Queue send options", async () => { process.env.JUNIOR_SECRET = "conversation-work-secret"; const sends: Array<{ diff --git a/packages/junior/tests/fixtures/conversation-work.ts b/packages/junior/tests/fixtures/conversation-work.ts index c90b5e13e..373da9755 100644 --- a/packages/junior/tests/fixtures/conversation-work.ts +++ b/packages/junior/tests/fixtures/conversation-work.ts @@ -28,13 +28,16 @@ interface QueueSendHold { /** * In-memory queue adapter for tests that need queue delivery plus send introspection. * - * `send` behaves like the production queue handoff: it records send metadata and - * makes the payload available for callback-style delivery through `takeMessage`. + * `send` behaves like the production queue handoff: it records send attempts and + * makes accepted payloads available for callback-style delivery through + * `takeMessage`. */ export class ConversationWorkQueueTestAdapter implements ConversationWorkQueue { + #idempotentMessageIds = new Map(); #queuedMessages: ConversationQueueMessage[] = []; #rejectSends = false; #sendHolds: QueueSendHold[] = []; + #sendAttempts: ConversationQueueSendRecord[] = []; #sentRecords: ConversationQueueSendRecord[] = []; allowSends(): void { @@ -42,6 +45,7 @@ export class ConversationWorkQueueTestAdapter implements ConversationWorkQueue { } clearSentRecords(): void { + this.#sendAttempts = []; this.#sentRecords = []; } @@ -57,6 +61,10 @@ export class ConversationWorkQueueTestAdapter implements ConversationWorkQueue { this.#rejectSends = true; } + sendAttempts(): ConversationQueueSendRecord[] { + return this.#sendAttempts.map((record) => ({ ...record })); + } + sentRecords(): ConversationQueueSendRecord[] { return this.#sentRecords.map((record) => ({ ...record })); } @@ -75,7 +83,6 @@ export class ConversationWorkQueueTestAdapter implements ConversationWorkQueue { if (this.#rejectSends) { throw new Error("queue unavailable"); } - this.#queuedMessages.push({ ...message }); const record: ConversationQueueSendRecord = { conversationId: message.conversationId, }; @@ -85,13 +92,25 @@ export class ConversationWorkQueueTestAdapter implements ConversationWorkQueue { if (options?.idempotencyKey !== undefined) { record.idempotencyKey = options.idempotencyKey; } + this.#sendAttempts.push(record); + const duplicateMessageId = options?.idempotencyKey + ? this.#idempotentMessageIds.get(options.idempotencyKey) + : undefined; + if (duplicateMessageId) { + return { messageId: duplicateMessageId }; + } + const messageId = `queue-${this.#sentRecords.length + 1}`; + this.#queuedMessages.push({ ...message }); this.#sentRecords.push(record); + if (options?.idempotencyKey) { + this.#idempotentMessageIds.set(options.idempotencyKey, messageId); + } const hold = this.#sendHolds.shift(); if (hold) { hold.entered(); await hold.release; } - return { messageId: `queue-${this.#sentRecords.length}` }; + return { messageId }; } takeMessage(): ConversationQueueMessage { diff --git a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts index 778a36286..1ec01a6bb 100644 --- a/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts +++ b/packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts @@ -134,6 +134,58 @@ describe("Slack behavior: durable turn steering", () => { await disconnectStateAdapter(); }); + it("does not enqueue duplicate Slack event retries for a persisted message", async () => { + const state = getStateAdapter(); + const { conversationId, queue, services } = createTurnHarness({ + generateAssistantReply: async () => ({ + text: "not used", + diagnostics: makeDiagnostics(), + }), + state, + }); + const event = makeMessageEvent({ + eventType: "app_mention", + text: `<@${SLACK_BOT_USER_ID}> start the incident summary`, + ts: THREAD_TS, + }); + + await expect( + handleSlackWebhookAndFlush({ + request: slackWebhookRequest(event), + services, + }), + ).resolves.toMatchObject({ status: 200 }); + await expect( + handleSlackWebhookAndFlush({ + request: slackWebhookRequest(event), + services, + }), + ).resolves.toMatchObject({ status: 200 }); + + const inboundMessageId = `slack:T123:${conversationId}:${THREAD_TS}`; + expect(queue.sendAttempts()).toEqual([ + { + conversationId, + idempotencyKey: inboundMessageId, + }, + ]); + expect(queue.sentRecords()).toEqual([ + { + conversationId, + idempotencyKey: inboundMessageId, + }, + ]); + + const work = await getConversationWorkState({ + conversationId, + state, + }); + expect(work?.messages.map((message) => message.inboundMessageId)).toEqual([ + inboundMessageId, + ]); + expect(work ? countPendingConversationMessages(work) : 0).toBe(1); + }); + it("steers rapid Slack webhook follow-ups into one active worker turn", async () => { const agentEntered = deferred(); const releaseAgent = deferred(); From 674e97be6d988ff4438b74f4d43d17939565fbb1 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 15:31:59 +0200 Subject: [PATCH 43/45] fix(ci): Remove stale heartbeat imports Drop unused imports left after the latest rebase conflict resolution so lint passes with warnings denied. Refs GH-470 Co-Authored-By: GPT-5 Codex --- packages/junior/tests/integration/heartbeat.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/junior/tests/integration/heartbeat.test.ts b/packages/junior/tests/integration/heartbeat.test.ts index 5c248a394..d37bf4af2 100644 --- a/packages/junior/tests/integration/heartbeat.test.ts +++ b/packages/junior/tests/integration/heartbeat.test.ts @@ -26,12 +26,8 @@ import { setAgentPlugins } from "@/chat/plugins/agent-hooks"; import { GET as heartbeat } from "@/handlers/heartbeat"; import { createSlackDirectCredentialSubject } from "@/chat/credentials/subject"; import { createConversationWorkQueueTestAdapter } from "../fixtures/conversation-work"; -import { conversationsInfoOk } from "../fixtures/slack/factories/api"; import { createWaitUntilCollector } from "../fixtures/wait-until"; -import { - getCapturedSlackApiCalls, - queueSlackApiResponse, -} from "../msw/handlers/slack-api"; +import { getCapturedSlackApiCalls } from "../msw/handlers/slack-api"; vi.hoisted(() => { process.env.JUNIOR_STATE_ADAPTER = "memory"; From 91b0e39da317ccd03dbaacea1cfc5b527560f16d Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 15:38:58 +0200 Subject: [PATCH 44/45] fix(runtime): Commit rescheduled Slack inputs When a Slack turn only reschedules an awaiting continuation, persist and commit the input hooks before returning. This lets durable mailbox workers mark the inbound row injected without sending a visible reply. Refs GH-470 Co-Authored-By: GPT-5 Codex --- packages/junior/src/chat/runtime/reply-executor.ts | 2 ++ packages/junior/tests/integration/slack/bot-handlers.test.ts | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index d193fd962..0ba95c742 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -437,6 +437,8 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { await persistThreadState(thread, { conversation: preparedState.conversation, }); + await options.onTurnStatePersisted?.(); + await options.onInputCommitted?.(); return; } diff --git a/packages/junior/tests/integration/slack/bot-handlers.test.ts b/packages/junior/tests/integration/slack/bot-handlers.test.ts index 4c5ff5e02..253ddd88a 100644 --- a/packages/junior/tests/integration/slack/bot-handlers.test.ts +++ b/packages/junior/tests/integration/slack/bot-handlers.test.ts @@ -810,6 +810,8 @@ describe("bot handlers (integration)", () => { expectedVersion: 4, }); const generateAssistantReply = vi.fn(); + const onInputCommitted = vi.fn(); + const onTurnStatePersisted = vi.fn(); const { slackRuntime } = createRuntime({ services: { replyExecutor: { @@ -834,6 +836,7 @@ describe("bot handlers (integration)", () => { text: "what happened?", isMention: true, }), + { onInputCommitted, onTurnStatePersisted }, ), ).resolves.toBeUndefined(); @@ -847,6 +850,8 @@ describe("bot handlers (integration)", () => { expectedVersion: 4, }); expect(generateAssistantReply).not.toHaveBeenCalled(); + expect(onTurnStatePersisted).toHaveBeenCalledOnce(); + expect(onInputCommitted).toHaveBeenCalledOnce(); expect(thread.posts).toEqual([]); const state = thread.getState(); From 1541a53272ee19423a61a2e582ca89d0ec8dca52 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 3 Jun 2026 15:52:24 +0200 Subject: [PATCH 45/45] fix(queue): Requeue lost lease work Mark lost-lease worker exits as runnable before releasing the conversation lease so queued work can recover immediately instead of waiting for lease TTL repair. Refs GH-470 Co-Authored-By: GPT-5 Codex --- .../junior/src/chat/task-execution/worker.ts | 48 +++++++++++++++++++ .../task-execution/conversation-work.test.ts | 31 ++++++++++++ 2 files changed, 79 insertions(+) diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index c901b4062..cf3ebfdbe 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -84,6 +84,42 @@ async function sendWakeNudge(args: { }); } +async function requestLostLeaseRecovery(args: { + conversationId: string; + leaseToken: string; + nowMs: number; + options: ProcessConversationWorkOptions; +}): Promise { + const continuationMarked = await requestConversationContinuation({ + conversationId: args.conversationId, + leaseToken: args.leaseToken, + nowMs: args.nowMs, + state: args.options.state, + }); + if (!continuationMarked) { + return; + } + const released = await releaseConversationWork({ + conversationId: args.conversationId, + leaseToken: args.leaseToken, + nowMs: args.nowMs, + state: args.options.state, + }); + if (!released) { + return; + } + await sendWakeNudge({ + conversationId: args.conversationId, + idempotencyKey: nudgeIdempotencyKey( + "lost_lease", + args.conversationId, + args.nowMs, + ), + nowMs: args.nowMs, + options: args.options, + }); +} + function startLeaseCheckIn(args: { conversationId: string; leaseToken: string; @@ -223,9 +259,21 @@ export async function processConversationWork( try { const result = await options.run(workerContext); if (result.status === "lost_lease") { + await requestLostLeaseRecovery({ + conversationId, + leaseToken: lease.leaseToken, + nowMs: now(options), + options, + }); return { status: "lost_lease" }; } if (leaseLost) { + await requestLostLeaseRecovery({ + conversationId, + leaseToken: lease.leaseToken, + nowMs: now(options), + options, + }); return { status: "lost_lease" }; } if (result.status === "yielded") { diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index c59f67963..3907756c8 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -352,6 +352,37 @@ describe("conversation work execution", () => { ]); }); + it("releases and requeues runnable work when the runner reports lost lease", async () => { + const queue = createConversationWorkQueueTestAdapter(); + let currentNowMs = 1_000; + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + processConversationWork(CONVERSATION_ID, { + nowMs: () => currentNowMs, + queue, + run: async () => { + currentNowMs = 2_000; + return { status: "lost_lease" }; + }, + }), + ).resolves.toEqual({ status: "lost_lease" }); + + const state = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(state?.lease).toBeUndefined(); + expect(state?.needsRun).toBe(true); + expect(state ? countPendingConversationMessages(state) : 0).toBe(1); + expect(state?.lastEnqueuedAtMs).toBe(2_000); + expect(queue.sentRecords()).toEqual([ + { + conversationId: CONVERSATION_ID, + idempotencyKey: `lost_lease:${CONVERSATION_ID}:2000`, + }, + ]); + }); + it("drains pending messages and completes the leased conversation", async () => { const queue = createConversationWorkQueueTestAdapter(); await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 });