diff --git a/dashboard/src/pages/stats.astro b/dashboard/src/pages/stats.astro index f824b75..76a56a7 100644 --- a/dashboard/src/pages/stats.astro +++ b/dashboard/src/pages/stats.astro @@ -156,6 +156,7 @@ const tabs = [ events_posted: number; status: "healthy" | "degraded" | "error"; error: string | null; + consecutive_failures: number; } interface ServiceStats { @@ -565,6 +566,13 @@ const tabs = [ ? "bg-amber-nerv" : "bg-red-nerv"; + const consecutiveFailuresHtml = ch.consecutive_failures > 0 + ? `
+
${ch.consecutive_failures}
+
Consecutive Failures
+
` + : ""; + return `
@@ -587,6 +595,7 @@ const tabs = [
${ch.events_posted}
Events Posted
+ ${consecutiveFailuresHtml} ${ch.error ? `
${ch.error}
` : ""}
diff --git a/src/api/index.ts b/src/api/index.ts index e090d5b..5bea6e9 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -77,6 +77,7 @@ function formatChannelStats( events_posted: s.eventsPosted, status: s.status, error: s.error, + consecutive_failures: s.consecutiveFailures, }; } return result; @@ -88,6 +89,7 @@ interface ChannelStatsResponse { events_posted: number; status: "healthy" | "degraded" | "error"; error: string | null; + consecutive_failures: number; } /** diff --git a/src/channels/calendar/apple-calendar.ts b/src/channels/calendar/apple-calendar.ts index 27ebd6f..ee4f547 100644 --- a/src/channels/calendar/apple-calendar.ts +++ b/src/channels/calendar/apple-calendar.ts @@ -1,10 +1,11 @@ /** * Read Apple Calendar events via osascript (JXA). * - * macOS only. Returns empty array on failure (non-macOS, no Calendar access, etc.). + * macOS only. Returns Result with error type indicating failure reason. */ import { createLogger } from "@shetty4l/core/log"; +import { err, ok, type Result } from "@shetty4l/core/result"; const log = createLogger("wilson:calendar"); @@ -20,6 +21,18 @@ export interface CalendarEvent { calendarName: string; } +/** Error types for calendar read failures */ +export type CalendarReadError = + | { type: "timeout" } + | { type: "osascript_failed"; exitCode: number; stderr: string } + | { type: "parse_error"; message: string } + | { type: "exception"; message: string }; + +export type ReadAppleCalendarResult = Result< + CalendarEvent[], + CalendarReadError +>; + // --- JXA script --- function buildJxaScript( @@ -103,7 +116,7 @@ const defaultSpawn: SpawnFn = async (cmd) => { stderr: string; }>((resolve) => { timeoutId = setTimeout(() => { - proc.kill(); + proc.kill(9); // SIGKILL for hard termination log("osascript timed out after 60s"); resolve({ exitCode: -1, stdout: "", stderr: "osascript timed out" }); }, SPAWN_TIMEOUT_MS); @@ -128,7 +141,7 @@ export interface ReadAppleCalendarOptions { * Read Apple Calendar events for the next N days. * * Uses osascript with JXA (JavaScript for Automation) to query Calendar.app. - * Returns empty array on any failure — never throws. + * Returns Result type — Ok with events or Err with typed error. * * @param options.lookAheadDays - Number of days to look ahead * @param options.includeCalendars - Optional list of calendar names to include (case-insensitive) @@ -136,7 +149,7 @@ export interface ReadAppleCalendarOptions { */ export async function readAppleCalendar( options: ReadAppleCalendarOptions, -): Promise { +): Promise { const { lookAheadDays, includeCalendars, spawn = defaultSpawn } = options; try { const script = buildJxaScript(lookAheadDays, includeCalendars); @@ -149,26 +162,38 @@ export async function readAppleCalendar( ]); if (exitCode !== 0) { + const timedOut = stderr.includes("timed out"); log(`osascript failed (exit ${exitCode}): ${stderr.trim()}`); - return []; + if (timedOut) { + return err({ type: "timeout" }); + } + return err({ type: "osascript_failed", exitCode, stderr: stderr.trim() }); } const trimmed = stdout.trim(); if (!trimmed) { - return []; + return ok([]); } - const events = JSON.parse(trimmed) as CalendarEvent[]; - if (!Array.isArray(events)) { - log("osascript returned non-array result"); - return []; + try { + const events = JSON.parse(trimmed) as CalendarEvent[]; + if (!Array.isArray(events)) { + log("osascript returned non-array result"); + return err({ + type: "parse_error", + message: "osascript returned non-array result", + }); + } + return ok(events); + } catch (parseErr) { + const message = + parseErr instanceof Error ? parseErr.message : String(parseErr); + log(`failed to parse calendar JSON: ${message}`); + return err({ type: "parse_error", message }); } - - return events; } catch (e) { - log( - `failed to read Apple Calendar: ${e instanceof Error ? e.message : String(e)}`, - ); - return []; + const message = e instanceof Error ? e.message : String(e); + log(`failed to read Apple Calendar: ${message}`); + return err({ type: "exception", message }); } } diff --git a/src/channels/calendar/index.ts b/src/channels/calendar/index.ts index 3465b77..847742e 100644 --- a/src/channels/calendar/index.ts +++ b/src/channels/calendar/index.ts @@ -18,6 +18,53 @@ import { readAppleCalendar, type SpawnFn } from "./apple-calendar"; const log = createLogger("wilson:calendar"); +// --- Recovery types --- + +export type RecoveryAction = "kill_osascript" | "restart_calendar"; + +export type RecoverFn = (action: RecoveryAction) => Promise; + +/** + * Default recovery implementation using system commands. + * Returns true if recovery command executed successfully. + */ +const defaultRecover: RecoverFn = async (action) => { + try { + if (action === "kill_osascript") { + // Kill any hung osascript processes related to Calendar + const proc = Bun.spawn(["pkill", "-9", "-f", "osascript.*Calendar"], { + stdout: "pipe", + stderr: "pipe", + }); + await proc.exited; + // pkill returns 0 if processes killed, 1 if none found - both are "success" + return true; + } else if (action === "restart_calendar") { + // Quit Calendar.app gracefully then relaunch + const quit = Bun.spawn( + ["osascript", "-e", 'tell app "Calendar" to quit'], + { + stdout: "pipe", + stderr: "pipe", + }, + ); + await quit.exited; + // Wait a moment for app to fully quit + await new Promise((r) => setTimeout(r, 1000)); + // Relaunch Calendar.app + const launch = Bun.spawn(["open", "-a", "Calendar"], { + stdout: "pipe", + stderr: "pipe", + }); + const exitCode = await launch.exited; + return exitCode === 0; + } + return false; + } catch { + return false; + } +}; + // --- Config --- export interface CalendarChannelConfig { @@ -27,6 +74,14 @@ export interface CalendarChannelConfig { includeCalendars?: string[]; } +// --- Recovery constants --- + +/** Number of consecutive timeouts before attempting recovery */ +const RECOVERY_THRESHOLD = 3; + +/** Minimum time between recovery attempts (5 minutes) */ +const RECOVERY_COOLDOWN_MS = 5 * 60 * 1000; + // --- Channel --- export class CalendarChannel implements Channel { @@ -51,21 +106,42 @@ export class CalendarChannel implements Channel { eventsPosted: 0, status: "healthy" as string, error: null as string | null, + consecutiveFailures: 0, + lastRecoveryAt: null as Date | null, }; + // Recovery function for dependency injection (testability) + private recoverFn: RecoverFn; + constructor( private cortex: CortexClient, private config: CalendarChannelConfig, stateLoaderOrSpawnFn?: StateLoader | SpawnFn, - spawnFn?: SpawnFn, + spawnFnOrRecover?: SpawnFn | RecoverFn, + recoverFn?: RecoverFn, ) { - // Support both old signature (cortex, config, spawnFn) and new (cortex, config, stateLoader, spawnFn) + // Support both old signature (cortex, config, spawnFn) and new (cortex, config, stateLoader, spawnFn, recoverFn) if (typeof stateLoaderOrSpawnFn === "function") { this.stateLoader = null; this.spawnFn = stateLoaderOrSpawnFn; + this.recoverFn = defaultRecover; } else { this.stateLoader = stateLoaderOrSpawnFn ?? null; - this.spawnFn = spawnFn; + // spawnFnOrRecover could be SpawnFn or RecoverFn - detect by arity/name + if (typeof spawnFnOrRecover === "function") { + // If recoverFn is also provided, spawnFnOrRecover is SpawnFn + if (recoverFn) { + this.spawnFn = spawnFnOrRecover as SpawnFn; + this.recoverFn = recoverFn; + } else { + // Only one function provided - assume it's SpawnFn for backwards compat + this.spawnFn = spawnFnOrRecover as SpawnFn; + this.recoverFn = defaultRecover; + } + } else { + this.spawnFn = undefined; + this.recoverFn = defaultRecover; + } } } @@ -105,6 +181,7 @@ export class CalendarChannel implements Channel { eventsPosted: this.state.eventsPosted, status: this.state.status as ChannelStats["status"], error: this.state.error, + consecutiveFailures: this.state.consecutiveFailures ?? 0, }; } // Fallback to in-memory state for tests @@ -114,6 +191,7 @@ export class CalendarChannel implements Channel { eventsPosted: this.memoryState.eventsPosted, status: this.memoryState.status as ChannelStats["status"], error: this.memoryState.error, + consecutiveFailures: this.memoryState.consecutiveFailures, }; } @@ -135,16 +213,53 @@ export class CalendarChannel implements Channel { } // Read events - const events = await readAppleCalendar({ + const result = await readAppleCalendar({ lookAheadDays: windowDays, includeCalendars: this.config.includeCalendars, spawn: this.spawnFn, }); + // Handle read errors + if (!result.ok) { + const error = result.error; + s.consecutiveFailures++; + if (error.type === "timeout") { + log( + `sync: osascript timed out (consecutive: ${s.consecutiveFailures})`, + ); + s.status = "degraded"; + s.error = "Calendar read timed out"; + + // Attempt recovery after hitting threshold + if (s.consecutiveFailures >= RECOVERY_THRESHOLD) { + await this.attemptRecovery(s); + } + } else if (error.type === "osascript_failed") { + log( + `sync: osascript failed (exit ${error.exitCode}): ${error.stderr}`, + ); + s.status = "error"; + s.error = `osascript failed: ${error.stderr}`; + } else if (error.type === "parse_error") { + log(`sync: parse error: ${error.message}`); + s.status = "error"; + s.error = `Parse error: ${error.message}`; + } else { + log(`sync: exception: ${error.message}`); + s.status = "error"; + s.error = error.message; + } + s.lastSyncAt = new Date(); + return; + } + + const events = result.value; + // Update lastSyncAt - we successfully read from Apple Calendar s.lastSyncAt = new Date(); s.status = "healthy"; s.error = null; + s.consecutiveFailures = 0; // Sort for stable hashing const sorted = [...events].sort( @@ -168,7 +283,7 @@ export class CalendarChannel implements Channel { s.lastHash = hash; // Post to cortex - const result = await this.cortex.receive({ + const postResult = await this.cortex.receive({ channel: "calendar", externalId: `cal-sync-${Date.now()}`, data: { events: sorted, windowDays }, @@ -176,16 +291,16 @@ export class CalendarChannel implements Channel { mode: "buffered", }); - if (result.ok) { + if (postResult.ok) { log( - `sync: posted ${sorted.length} events (${windowDays}d window, status: ${result.value.status})`, + `sync: posted ${sorted.length} events (${windowDays}d window, status: ${postResult.value.status})`, ); s.lastPostAt = new Date(); s.eventsPosted = sorted.length; } else { - log(`sync: cortex error: ${result.error}`); + log(`sync: cortex error: ${postResult.error}`); s.status = "degraded"; - s.error = `Cortex error: ${result.error}`; + s.error = `Cortex error: ${postResult.error}`; } } catch (e) { const errorMsg = e instanceof Error ? e.message : String(e); @@ -195,6 +310,49 @@ export class CalendarChannel implements Channel { } } + // --- Recovery --- + + /** + * Attempt to recover from consecutive timeouts. + * + * Recovery is rate-limited to once per 5 minutes. + * Step 1: Kill hung osascript processes + * Step 2: If at double threshold (6 failures), also restart Calendar.app + */ + private async attemptRecovery( + s: CalendarChannelState | typeof this.memoryState, + ): Promise { + const now = Date.now(); + const lastRecovery = s.lastRecoveryAt?.getTime() ?? 0; + + // Rate-limit recovery attempts + if (now - lastRecovery < RECOVERY_COOLDOWN_MS) { + log( + `recovery: skipping (cooldown, last attempt ${Math.round((now - lastRecovery) / 1000)}s ago)`, + ); + return; + } + + s.lastRecoveryAt = new Date(); + + // Step 1: Kill hung osascript processes + log("recovery: killing hung osascript processes"); + await this.recoverFn("kill_osascript"); + + // Step 2: If at double threshold, also restart Calendar.app + if (s.consecutiveFailures >= RECOVERY_THRESHOLD * 2) { + log("recovery: restarting Calendar.app"); + const restartSuccess = await this.recoverFn("restart_calendar"); + if (restartSuccess) { + log("recovery: Calendar.app restarted, will retry on next sync"); + } else { + log("recovery: Calendar.app restart failed"); + } + } else { + log("recovery: osascript processes killed, will retry on next sync"); + } + } + // --- Test helpers --- /** @internal — exposed for testing */ diff --git a/src/channels/index.ts b/src/channels/index.ts index 4ff89f8..c2c70f7 100644 --- a/src/channels/index.ts +++ b/src/channels/index.ts @@ -23,6 +23,8 @@ export interface ChannelStats { status: "healthy" | "degraded" | "error"; /** Last error message if status is error/degraded. */ error: string | null; + /** Number of consecutive failures (timeouts or errors). Resets on success. */ + consecutiveFailures: number; } // --- Channel interface --- diff --git a/src/state/calendar.ts b/src/state/calendar.ts index a3f4b84..d352dc3 100644 --- a/src/state/calendar.ts +++ b/src/state/calendar.ts @@ -28,4 +28,10 @@ export class CalendarChannelState { /** Date of last extended sync ("YYYY-MM-DD"). */ @Field("string") lastExtendedSyncDate: string | null = null; + + /** Number of consecutive failures (timeouts or errors). Resets on success. */ + @Field("number") consecutiveFailures: number = 0; + + /** Timestamp of last recovery attempt (rate-limiting). */ + @Field("date") lastRecoveryAt: Date | null = null; } diff --git a/test/apple-calendar.test.ts b/test/apple-calendar.test.ts index fe1d2ba..9655007 100644 --- a/test/apple-calendar.test.ts +++ b/test/apple-calendar.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { type CalendarEvent, + type CalendarReadError, readAppleCalendar, type SpawnFn, } from "../src/channels/calendar/apple-calendar"; @@ -35,13 +36,16 @@ describe("readAppleCalendar", () => { }); const result = await readAppleCalendar({ lookAheadDays: 14, spawn }); - expect(result).toEqual(events); - expect(result.length).toBe(2); - expect(result[0].title).toBe("Team Meeting"); - expect(result[1].calendarName).toBe("Home"); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value).toEqual(events); + expect(result.value.length).toBe(2); + expect(result.value[0].title).toBe("Team Meeting"); + expect(result.value[1].calendarName).toBe("Home"); + } }); - test("returns empty array on osascript failure", async () => { + test("returns osascript_failed error on non-timeout failure", async () => { const spawn: SpawnFn = async () => ({ exitCode: 1, stdout: "", @@ -49,7 +53,16 @@ describe("readAppleCalendar", () => { }); const result = await readAppleCalendar({ lookAheadDays: 14, spawn }); - expect(result).toEqual([]); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.type).toBe("osascript_failed"); + if (result.error.type === "osascript_failed") { + expect(result.error.exitCode).toBe(1); + expect(result.error.stderr).toBe( + "osascript: not available on this platform", + ); + } + } }); test("returns empty array on empty calendar", async () => { @@ -60,19 +73,28 @@ describe("readAppleCalendar", () => { }); const result = await readAppleCalendar({ lookAheadDays: 14, spawn }); - expect(result).toEqual([]); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value).toEqual([]); + } }); - test("returns empty array when spawn throws", async () => { + test("returns exception error when spawn throws", async () => { const spawn: SpawnFn = async () => { throw new Error("spawn failed"); }; const result = await readAppleCalendar({ lookAheadDays: 14, spawn }); - expect(result).toEqual([]); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.type).toBe("exception"); + if (result.error.type === "exception") { + expect(result.error.message).toBe("spawn failed"); + } + } }); - test("returns empty array on invalid JSON output", async () => { + test("returns parse_error on invalid JSON output", async () => { const spawn: SpawnFn = async () => ({ exitCode: 0, stdout: "not valid json", @@ -80,7 +102,10 @@ describe("readAppleCalendar", () => { }); const result = await readAppleCalendar({ lookAheadDays: 14, spawn }); - expect(result).toEqual([]); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.type).toBe("parse_error"); + } }); test("returns empty array when output is empty string", async () => { @@ -91,13 +116,14 @@ describe("readAppleCalendar", () => { }); const result = await readAppleCalendar({ lookAheadDays: 14, spawn }); - expect(result).toEqual([]); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value).toEqual([]); + } }); describe("timeout handling", () => { - test("returns empty array on timeout (graceful fallback)", async () => { - // Simulate a spawn that never resolves (would time out in real usage) - // We test by returning the timeout result directly + test("returns timeout error on timeout", async () => { const spawn: SpawnFn = async () => ({ exitCode: -1, stdout: "", @@ -105,7 +131,24 @@ describe("readAppleCalendar", () => { }); const result = await readAppleCalendar({ lookAheadDays: 14, spawn }); - expect(result).toEqual([]); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.type).toBe("timeout"); + } + }); + + test("returns osascript_failed error on non-timeout failure", async () => { + const spawn: SpawnFn = async () => ({ + exitCode: 1, + stdout: "", + stderr: "some other error", + }); + + const result = await readAppleCalendar({ lookAheadDays: 14, spawn }); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.type).toBe("osascript_failed"); + } }); }); diff --git a/test/calendar.test.ts b/test/calendar.test.ts index 7a35711..cb8178a 100644 --- a/test/calendar.test.ts +++ b/test/calendar.test.ts @@ -250,13 +250,11 @@ describe("CalendarChannel", () => { // Should not throw await channel.start(); - // osascript failed → empty events → still posts initial (empty array hash differs from null) - expect(cortex.calls.length).toBe(1); - const data = cortex.calls[0].data as { - events: CalendarEvent[]; - windowDays: number; - }; - expect(data.events).toEqual([]); + // osascript failed → error state, no cortex post + expect(cortex.calls.length).toBe(0); + const stats = channel.getStats(); + expect(stats.status).toBe("error"); + expect(stats.error).toContain("osascript"); }); describe("stats tracking", () => { @@ -341,9 +339,9 @@ describe("CalendarChannel", () => { expect(stats.error).toContain("Cortex"); }); - test("getStats() remains healthy when osascript returns empty (graceful fallback)", async () => { + test("getStats() shows error when osascript throws", async () => { const cortex = makeMockCortex(); - // Spawn that throws — readAppleCalendar catches it and returns [] + // Spawn that throws — readAppleCalendar catches it and returns Err const errorSpawn = async () => { throw new Error("spawn failed"); }; @@ -352,11 +350,12 @@ describe("CalendarChannel", () => { await channel.start(); - // readAppleCalendar catches the error and returns [] — sync proceeds normally + // readAppleCalendar catches the error and returns Err — sync records error state const stats = channel.getStats(); - expect(stats.status).toBe("healthy"); // graceful degradation + expect(stats.status).toBe("error"); + expect(stats.error).toContain("spawn failed"); expect(stats.lastSyncAt).toBeGreaterThan(0); - expect(stats.eventsPosted).toBe(0); // empty events + expect(stats.eventsPosted).toBe(0); }); test("getStats() returns a copy (immutable)", async () => { @@ -376,6 +375,228 @@ describe("CalendarChannel", () => { }); }); + describe("consecutive failures tracking", () => { + test("consecutiveFailures starts at 0", () => { + const cortex = makeMockCortex(); + channel = new CalendarChannel( + cortex, + DEFAULT_CONFIG, + makeSpawn(SAMPLE_EVENTS), + ); + + const stats = channel.getStats(); + expect(stats.consecutiveFailures).toBe(0); + }); + + test("consecutiveFailures increments on timeout", async () => { + const cortex = makeMockCortex(); + const timeoutSpawn = async () => ({ + exitCode: -1, + stdout: "", + stderr: "osascript timed out", + }); + + channel = new CalendarChannel(cortex, DEFAULT_CONFIG, timeoutSpawn); + await channel.start(); + + expect(channel.getStats().consecutiveFailures).toBe(1); + + // Trigger another sync + await channel.sync(); + expect(channel.getStats().consecutiveFailures).toBe(2); + }); + + test("consecutiveFailures resets on success", async () => { + const cortex = makeMockCortex(); + let callCount = 0; + + // First two calls timeout, third succeeds + const mixedSpawn = async () => { + callCount++; + if (callCount <= 2) { + return { exitCode: -1, stdout: "", stderr: "osascript timed out" }; + } + return { + exitCode: 0, + stdout: JSON.stringify(SAMPLE_EVENTS), + stderr: "", + }; + }; + + channel = new CalendarChannel(cortex, DEFAULT_CONFIG, mixedSpawn); + await channel.start(); // timeout 1 + expect(channel.getStats().consecutiveFailures).toBe(1); + + await channel.sync(); // timeout 2 + expect(channel.getStats().consecutiveFailures).toBe(2); + + await channel.sync(); // success + expect(channel.getStats().consecutiveFailures).toBe(0); + expect(channel.getStats().status).toBe("healthy"); + }); + + test("consecutiveFailures increments on osascript failure", async () => { + const cortex = makeMockCortex(); + const failSpawn = async () => ({ + exitCode: 1, + stdout: "", + stderr: "osascript: not available", + }); + + channel = new CalendarChannel(cortex, DEFAULT_CONFIG, failSpawn); + await channel.start(); + + expect(channel.getStats().consecutiveFailures).toBe(1); + expect(channel.getStats().status).toBe("error"); + }); + }); + + describe("recovery logic", () => { + test("triggers kill_osascript recovery after 3 consecutive timeouts", async () => { + const cortex = makeMockCortex(); + const recoveryActions: string[] = []; + + const timeoutSpawn = async () => ({ + exitCode: -1, + stdout: "", + stderr: "osascript timed out", + }); + + const mockRecover = async (action: string) => { + recoveryActions.push(action); + return true; + }; + + channel = new CalendarChannel( + cortex, + DEFAULT_CONFIG, + undefined, + timeoutSpawn, + mockRecover, + ); + + await channel.start(); // timeout 1 + await channel.sync(); // timeout 2 + await channel.sync(); // timeout 3 - should trigger recovery + + expect(recoveryActions).toEqual(["kill_osascript"]); + }); + + test("triggers restart_calendar after 6 consecutive timeouts", async () => { + const cortex = makeMockCortex(); + const recoveryActions: string[] = []; + + const timeoutSpawn = async () => ({ + exitCode: -1, + stdout: "", + stderr: "osascript timed out", + }); + + // Mock recover that "succeeds" but doesn't actually fix the problem + const mockRecover = async (action: string) => { + recoveryActions.push(action); + return true; + }; + + channel = new CalendarChannel( + cortex, + DEFAULT_CONFIG, + undefined, + timeoutSpawn, + mockRecover, + ); + + // Trigger 6 consecutive timeouts + await channel.start(); // 1 + await channel.sync(); // 2 + await channel.sync(); // 3 - kill_osascript + // Need to reset cooldown for testing - we'll use a fresh channel + await channel.stop(); + + // Create new channel with state already at 5 failures + const channel2 = new CalendarChannel( + cortex, + DEFAULT_CONFIG, + undefined, + timeoutSpawn, + mockRecover, + ); + // Use internal method to set up state + (channel2 as any).memoryState.consecutiveFailures = 5; + (channel2 as any).running = true; + + await channel2.sync(); // 6 - should trigger restart_calendar + + expect(recoveryActions).toContain("restart_calendar"); + await channel2.stop(); + }); + + test("respects recovery cooldown (5 minutes)", async () => { + const cortex = makeMockCortex(); + const recoveryActions: string[] = []; + + const timeoutSpawn = async () => ({ + exitCode: -1, + stdout: "", + stderr: "osascript timed out", + }); + + const mockRecover = async (action: string) => { + recoveryActions.push(action); + return true; + }; + + channel = new CalendarChannel( + cortex, + DEFAULT_CONFIG, + undefined, + timeoutSpawn, + mockRecover, + ); + + await channel.start(); // 1 + await channel.sync(); // 2 + await channel.sync(); // 3 - triggers recovery + await channel.sync(); // 4 - should NOT trigger recovery (cooldown) + + // Only one recovery action should have been triggered + expect(recoveryActions).toEqual(["kill_osascript"]); + }); + + test("no recovery triggered for non-timeout errors", async () => { + const cortex = makeMockCortex(); + const recoveryActions: string[] = []; + + // Non-timeout error (osascript failed, not timed out) + const failSpawn = async () => ({ + exitCode: 1, + stdout: "", + stderr: "osascript: not available", + }); + + const mockRecover = async (action: string) => { + recoveryActions.push(action); + return true; + }; + + channel = new CalendarChannel( + cortex, + DEFAULT_CONFIG, + undefined, + failSpawn, + mockRecover, + ); + + await channel.start(); // 1 + await channel.sync(); // 2 + await channel.sync(); // 3 + await channel.sync(); // 4 + + // No recovery should be triggered for non-timeout errors + expect(recoveryActions).toEqual([]); + }); + }); + describe("includeCalendars filtering", () => { test("passes includeCalendars to readAppleCalendar", async () => { const cortex = makeMockCortex(); diff --git a/test/channels.test.ts b/test/channels.test.ts index 5735f60..0f38d02 100644 --- a/test/channels.test.ts +++ b/test/channels.test.ts @@ -29,6 +29,7 @@ function makeChannel(name: string, calls: string[]): Channel { eventsPosted: 0, status: "healthy", error: null, + consecutiveFailures: 0, }; }, };