fix(routines): hot-enable on LLM key save — resolve chat agent live per run

middleware/src/index.ts

@@ -1659,9 +1659,11 @@ async function main(): Promise<void> {
   // so chat goes live the moment the key is saved — no restart needed.
   //
   // The boot-only wiring guarded on `orchestrator` below (domain-tool
-  // hydration of per-Agent orchestrators, the routines feature) re-applies on
-  // the next restart for advanced stacks (sub-agents / routines). The default
-  // out-of-the-box stack has no domain tools, so chat is fully functional hot.
+  // hydration of per-Agent orchestrators) re-applies on the next restart for
+  // advanced stacks (sub-agents). The default out-of-the-box stack has no
+  // domain tools, so chat is fully functional hot. Routines follows the same
+  // live-resolution pattern as chat (issue #473): its runner resolves
+  // chatAgent per run, so it hot-enables on key save too.
   const chatAgentBundle = serviceRegistry.get<ChatAgentBundle>('chatAgent');
   const orchestrator = chatAgentBundle?.raw;
   if (!chatAgentBundle) {
@@ -1938,8 +1940,12 @@ async function main(): Promise<void> {
 
   // Routines feature (OB-NEW): persistent user-created scheduled agent
   // invocations. Requires Postgres for persistence; skipped in zero-config
-  // dev (in-memory KG backend, no DATABASE_URL). Channel adapters that want
-  // proactive delivery register their `ProactiveSender` into
+  // dev (in-memory KG backend, no DATABASE_URL). The chat agent is NOT
+  // required at wiring time — the runner resolves chatAgent@1 live per run
+  // (same pattern as the chat routes above), so routines hot-enable the
+  // moment the Setup Wizard key save publishes it; keyless fires record an
+  // `error` run naming the missing key (issue #473). Channel adapters that
+  // want proactive delivery register their `ProactiveSender` into
   // `routinesHandle.senderRegistry` after this call (Teams: wrap a
   // long-lived `CloudAdapter.continueConversationAsync` via
   // `createProactiveSender('teams', sendFn)`). Channel adapters MUST also
@@ -1948,11 +1954,11 @@ async function main(): Promise<void> {
   // `manage_routine` tool's `create`/`list` actions return a
   // model-friendly error string and the model degrades gracefully.
   let routinesHandle: RoutinesHandle | undefined;
-  if (graphPool && orchestrator) {
+  if (graphPool) {
     routinesHandle = await initRoutines({
       pool: graphPool,
       scheduler: jobScheduler,
-      orchestrator,
+      getOrchestrator: () => getChatAgentBundle()?.raw,
       registerNativeTool: (name, handler, options) =>
         nativeToolRegistry.register(name, {
           handler,
@@ -1987,15 +1993,11 @@ async function main(): Promise<void> {
       }),
     );
     console.log(
-      '[middleware] routines feature ready (manage_routine tool registered, routinesIntegration published)',
-    );
-  } else if (!graphPool) {
-    console.log(
-      '[middleware] routines feature SKIPPED — no graphPool (in-memory KG backend; set DATABASE_URL to enable)',
+      '[middleware] routines feature ready (manage_routine tool registered, routinesIntegration published, chat agent resolved live per run)',
     );
   } else {
     console.log(
-      '[middleware] routines feature SKIPPED — chatAgent not active (set ANTHROPIC_API_KEY via the Setup Wizard, then restart to enable routines)',
+      '[middleware] routines feature SKIPPED — no graphPool (in-memory KG backend; set DATABASE_URL to enable)',
     );
   }
 

middleware/src/plugins/routines/initRoutines.ts

@@ -21,9 +21,11 @@ import { routineTurnContext } from './routineTurnContext.js';
 
 /**
  * Single-call wiring for the routines feature. The kernel calls this once
- * after the Postgres pool, the JobScheduler, and the ChatAgent are
- * available; everything below the surface is created here so `index.ts`
- * stays thin.
+ * after the Postgres pool and the JobScheduler are available — the chat
+ * agent is NOT required at wiring time: the runner resolves it live per
+ * run via `getOrchestrator`, so routines hot-enable the moment the Setup
+ * Wizard key save publishes chatAgent@1 (issue #473). Everything below
+ * the surface is created here so `index.ts` stays thin.
  *
  * Lifecycle:
  *   - Runs DB migrations (idempotent — `_routine_migrations` tracks).
@@ -47,13 +49,15 @@ export interface InitRoutinesOptions {
   pool: Pool;
   scheduler: JobScheduler;
   /**
-   * Real `Orchestrator` (typically `chatAgentBundle.raw`). The runner
-   * needs the lower-level `runTurn` (not the higher-level `chat`) so it
-   * can persist the per-turn `runTrace` for the call-stack viewer. The
-   * structural type keeps this layer free of a hard import on the
+   * Live resolver for the real `Orchestrator` (typically
+   * `() => chatAgentBundle?.raw` over the service registry). Resolved per
+   * run, never captured — see `RoutineRunnerOptions.getOrchestrator`. The
+   * runner needs the lower-level `runTurn` (not the higher-level `chat`)
+   * so it can persist the per-turn `runTrace` for the call-stack viewer.
+   * The structural type keeps this layer free of a hard import on the
    * orchestrator package.
    */
-  orchestrator: OrchestratorLike;
+  getOrchestrator: () => OrchestratorLike | undefined;
   /**
    * Native-tool registration surface. Production passes the kernel-owned
    * `nativeToolRegistry`. The shape is the minimum used here so a stub
@@ -106,7 +110,7 @@ export async function initRoutines(
     store,
     runsStore,
     scheduler: opts.scheduler,
-    orchestrator: opts.orchestrator,
+    getOrchestrator: opts.getOrchestrator,
     senderRegistry,
     log,
     maxActivePerUser: opts.maxActivePerUser,

middleware/src/plugins/routines/routineRunner.ts

@@ -88,6 +88,15 @@ export class UnknownChannelError extends Error {
   }
 }
 
+export class ChatAgentUnavailableError extends Error {
+  constructor() {
+    super(
+      'chat agent unavailable — configure the LLM API key in Settings to enable routine runs',
+    );
+    this.name = 'ChatAgentUnavailableError';
+  }
+}
+
 export interface RoutineRunnerOptions {
   store: RoutineStore;
   runsStore: RoutineRunsStore;
@@ -98,13 +107,17 @@ export interface RoutineRunnerOptions {
    */
   scheduler: JobSchedulerLike;
   /**
-   * Production wiring passes `chatAgentBundle.raw` (the real
-   * `Orchestrator`); tests pass a stub that returns a synthetic
-   * `ChatTurnResult`. The runner needs `runTurn` (not the higher-level
-   * `chat`) so it can persist the per-turn `runTrace` for the
-   * call-stack viewer.
+   * Live resolver for the plugin-published chat agent — production wiring
+   * passes a closure over `serviceRegistry.get('chatAgent')?.raw`; tests
+   * pass `() => stub`. Resolved per run (never captured) so routines
+   * hot-enable the moment the Setup Wizard key save republishes
+   * chatAgent@1, and a key rotation swaps instances without a restart.
+   * A run that fires while it returns `undefined` records an `error`
+   * run via `ChatAgentUnavailableError`. The runner needs `runTurn`
+   * (not the higher-level `chat`) so it can persist the per-turn
+   * `runTrace` for the call-stack viewer.
    */
-  orchestrator: OrchestratorLike;
+  getOrchestrator: () => OrchestratorLike | undefined;
   senderRegistry: ProactiveSenderRegistry;
   log?: (msg: string) => void;
   /** Override the per-user active-routine cap. */
@@ -133,7 +146,7 @@ export class RoutineRunner {
   private readonly store: RoutineStore;
   private readonly runsStore: RoutineRunsStore;
   private readonly scheduler: JobSchedulerLike;
-  private readonly orchestrator: OrchestratorLike;
+  private readonly getOrchestrator: () => OrchestratorLike | undefined;
   private readonly senders: ProactiveSenderRegistry;
   private readonly log: (msg: string) => void;
   private readonly maxActivePerUser: number;
@@ -145,7 +158,7 @@ export class RoutineRunner {
     this.store = opts.store;
     this.runsStore = opts.runsStore;
     this.scheduler = opts.scheduler;
-    this.orchestrator = opts.orchestrator;
+    this.getOrchestrator = opts.getOrchestrator;
     this.senders = opts.senderRegistry;
     this.log = opts.log ?? ((msg) => console.log(msg));
     this.maxActivePerUser =
@@ -261,6 +274,17 @@ export class RoutineRunner {
     return row ?? null;
   }
 
+  /**
+   * Whether a chat agent currently resolves. The trigger endpoint checks
+   * this to return a synchronous 503 instead of accepting a manual run
+   * that is guaranteed to record `error` — cron fires still go through
+   * `runOnce` and record the failure, keeping the misconfiguration
+   * visible in run history.
+   */
+  chatAgentAvailable(): boolean {
+    return this.getOrchestrator() !== undefined;
+  }
+
   /**
    * Dispose every registered routine. Called on graceful shutdown.
    * In-flight runs receive their AbortSignal via `stopForPlugin`.
@@ -392,20 +416,31 @@ export class RoutineRunner {
     let userId = routine.userId;
 
     try {
-      const sender = this.senders.get(routine.channel);
-      if (!sender) {
-        throw new UnknownChannelError(routine.channel);
-      }
-
       // Re-read the row before invoking. A pause/delete that landed
       // between schedule and trigger should be honoured (the dispose was
-      // best-effort, not transactional).
+      // best-effort, not transactional). Checked first so a raced fire on
+      // a paused row never records a spurious availability error.
       const fresh = await this.store.get(routine.id);
       if (!fresh || fresh.status !== 'active') return;
       prompt = fresh.prompt;
       tenant = fresh.tenant;
       userId = fresh.userId;
 
+      // Resolve the chat agent per fire (issue #473) and BEFORE the
+      // sender lookup: pre-key, the channel plugins (which require
+      // chatAgent@^1) are dark too, so a sender-first check would record
+      // the misleading 'no proactive sender' error when the actual fix
+      // is configuring the key.
+      const orchestrator = this.getOrchestrator();
+      if (!orchestrator) {
+        throw new ChatAgentUnavailableError();
+      }
+
+      const sender = this.senders.get(routine.channel);
+      if (!sender) {
+        throw new UnknownChannelError(routine.channel);
+      }
+
       if (signal.aborted) {
         status = 'timeout';
         errorMessage = signal.reason instanceof Error
@@ -424,7 +459,7 @@ export class RoutineRunner {
       // data sections directly from the tool handler's output instead
       // of letting the LLM author markdown rows. Non-templated routines
       // skip the wrap entirely — byte-identical to pre-C.2 behaviour.
-      const turn = await this.runTurnWithOptionalCapture(fresh);
+      const turn = await this.runTurnWithOptionalCapture(fresh, orchestrator);
       result = turn.result;
       cardBody = turn.cardBody;
 
@@ -530,12 +565,16 @@ export class RoutineRunner {
    */
   private async runTurnWithOptionalCapture(
     routine: Routine,
+    // The instance `runOnce` resolved for this fire — passed down (not
+    // re-resolved) so one run never straddles two orchestrator instances
+    // across a mid-run key rotation.
+    orchestrator: OrchestratorLike,
   ): Promise<{
     readonly result: ChatTurnResult;
     readonly cardBody?: readonly unknown[];
   }> {
     if (routine.outputTemplate === null) {
-      const result = await this.orchestrator.runTurn({
+      const result = await orchestrator.runTurn({
         userMessage: routine.prompt,
         userId: routine.userId,
         sessionScope: `routine:${routine.id}`,
@@ -570,7 +609,7 @@ export class RoutineRunner {
             captureRawToolResult,
           },
           () =>
-            this.orchestrator.runTurn({
+            orchestrator.runTurn({
               userMessage: augmentedPrompt,
               userId: routine.userId,
               sessionScope: `routine:${routine.id}`,

middleware/src/routes/routines.ts

@@ -169,7 +169,8 @@ function toDto(r: Routine): RoutineDto {
  * Routes are mounted under `/api/v1/routines` and gated by `requireAuth`.
  *   GET    /                  → list all routines (cross-tenant)
  *   PATCH  /:id/status        → body `{status: 'active' | 'paused'}`
- *   POST   /:id/trigger       → fire one manual run (records as `manual` in routine_runs)
+ *   POST   /:id/trigger       → fire one manual run (records as `manual` in routine_runs);
+ *                               503 `routines.chat_unavailable` while no chat agent is published
  *   GET    /:id/runs          → list per-routine run history (last 50 by default)
  *   GET    /:id/runs/:runId   → single run with full agentic trace (call-stack viewer)
  *   DELETE /:id               → permanent (cascades into routine_runs)
@@ -254,6 +255,19 @@ export function createRoutinesRouter(deps: RoutinesRouterDeps): Router {
           .json({ code: 'routines.not_found', message: `routine '${id}' not found` });
         return;
       }
+      // Interactive path gets a synchronous 503 while no chat agent is
+      // published (LLM key not configured) instead of a 202 whose run is
+      // guaranteed to fail in the background — the web-ui maps this code
+      // to a "set the key in Settings" message. Cron fires keep recording
+      // `error` runs so the misconfiguration stays visible in history.
+      if (!deps.runner.chatAgentAvailable()) {
+        res.status(503).json({
+          code: 'routines.chat_unavailable',
+          message:
+            'chat agent unavailable — configure the LLM API key in Settings, then trigger again',
+        });
+        return;
+      }
       // Fire-and-forget: the trigger run takes seconds (LLM call +
       // tool roundtrip + validator retry + Teams send). Holding the
       // HTTP connection open the whole way means web-ui proxies time

middleware/test/hrRoutineTemplate.integration.test.ts

@@ -287,11 +287,12 @@ async function makeRunner(template: RoutineOutputTemplate): Promise<{
   const sender = new StubSender();
   const senderRegistry = new InMemoryProactiveSenderRegistry();
   senderRegistry.register(sender);
+  const hrOrchestrator = makeHrOrchestrator();
   const runner = new RoutineRunner({
     store: store as unknown as RoutineStore,
     runsStore: runsStore as unknown as RoutineRunsStore,
     scheduler,
-    orchestrator: makeHrOrchestrator(),
+    getOrchestrator: () => hrOrchestrator,
     senderRegistry,
     log: () => {},
   });
@@ -462,7 +463,7 @@ describe('Phase C.8 — HR routine end-to-end with reference template', () => {
       store: store as unknown as RoutineStore,
       runsStore: runsStore as unknown as RoutineRunsStore,
       scheduler,
-      orchestrator,
+      getOrchestrator: () => orchestrator,
       senderRegistry,
       log: () => {},
     });