diff --git a/dashboard/src/pages/stats.astro b/dashboard/src/pages/stats.astro
index f824b75..76a56a7 100644
--- a/dashboard/src/pages/stats.astro
+++ b/dashboard/src/pages/stats.astro
@@ -156,6 +156,7 @@ const tabs = [
events_posted: number;
status: "healthy" | "degraded" | "error";
error: string | null;
+ consecutive_failures: number;
}
interface ServiceStats {
@@ -565,6 +566,13 @@ const tabs = [
? "bg-amber-nerv"
: "bg-red-nerv";
+ const consecutiveFailuresHtml = ch.consecutive_failures > 0
+ ? `
+
${ch.consecutive_failures}
+
Consecutive Failures
+
`
+ : "";
+
return `
@@ -587,6 +595,7 @@ const tabs = [
${ch.events_posted}
Events Posted
+ ${consecutiveFailuresHtml}
${ch.error ? `
` : ""}
diff --git a/src/api/index.ts b/src/api/index.ts
index e090d5b..5bea6e9 100644
--- a/src/api/index.ts
+++ b/src/api/index.ts
@@ -77,6 +77,7 @@ function formatChannelStats(
events_posted: s.eventsPosted,
status: s.status,
error: s.error,
+ consecutive_failures: s.consecutiveFailures,
};
}
return result;
@@ -88,6 +89,7 @@ interface ChannelStatsResponse {
events_posted: number;
status: "healthy" | "degraded" | "error";
error: string | null;
+ consecutive_failures: number;
}
/**
diff --git a/src/channels/calendar/apple-calendar.ts b/src/channels/calendar/apple-calendar.ts
index 27ebd6f..ee4f547 100644
--- a/src/channels/calendar/apple-calendar.ts
+++ b/src/channels/calendar/apple-calendar.ts
@@ -1,10 +1,11 @@
/**
* Read Apple Calendar events via osascript (JXA).
*
- * macOS only. Returns empty array on failure (non-macOS, no Calendar access, etc.).
+ * macOS only. Returns Result with error type indicating failure reason.
*/
import { createLogger } from "@shetty4l/core/log";
+import { err, ok, type Result } from "@shetty4l/core/result";
const log = createLogger("wilson:calendar");
@@ -20,6 +21,18 @@ export interface CalendarEvent {
calendarName: string;
}
+/** Error types for calendar read failures */
+export type CalendarReadError =
+ | { type: "timeout" }
+ | { type: "osascript_failed"; exitCode: number; stderr: string }
+ | { type: "parse_error"; message: string }
+ | { type: "exception"; message: string };
+
+export type ReadAppleCalendarResult = Result<
+ CalendarEvent[],
+ CalendarReadError
+>;
+
// --- JXA script ---
function buildJxaScript(
@@ -103,7 +116,7 @@ const defaultSpawn: SpawnFn = async (cmd) => {
stderr: string;
}>((resolve) => {
timeoutId = setTimeout(() => {
- proc.kill();
+ proc.kill(9); // SIGKILL for hard termination
log("osascript timed out after 60s");
resolve({ exitCode: -1, stdout: "", stderr: "osascript timed out" });
}, SPAWN_TIMEOUT_MS);
@@ -128,7 +141,7 @@ export interface ReadAppleCalendarOptions {
* Read Apple Calendar events for the next N days.
*
* Uses osascript with JXA (JavaScript for Automation) to query Calendar.app.
- * Returns empty array on any failure — never throws.
+ * Returns Result type — Ok with events or Err with typed error.
*
* @param options.lookAheadDays - Number of days to look ahead
* @param options.includeCalendars - Optional list of calendar names to include (case-insensitive)
@@ -136,7 +149,7 @@ export interface ReadAppleCalendarOptions {
*/
export async function readAppleCalendar(
options: ReadAppleCalendarOptions,
-): Promise {
+): Promise {
const { lookAheadDays, includeCalendars, spawn = defaultSpawn } = options;
try {
const script = buildJxaScript(lookAheadDays, includeCalendars);
@@ -149,26 +162,38 @@ export async function readAppleCalendar(
]);
if (exitCode !== 0) {
+ const timedOut = stderr.includes("timed out");
log(`osascript failed (exit ${exitCode}): ${stderr.trim()}`);
- return [];
+ if (timedOut) {
+ return err({ type: "timeout" });
+ }
+ return err({ type: "osascript_failed", exitCode, stderr: stderr.trim() });
}
const trimmed = stdout.trim();
if (!trimmed) {
- return [];
+ return ok([]);
}
- const events = JSON.parse(trimmed) as CalendarEvent[];
- if (!Array.isArray(events)) {
- log("osascript returned non-array result");
- return [];
+ try {
+ const events = JSON.parse(trimmed) as CalendarEvent[];
+ if (!Array.isArray(events)) {
+ log("osascript returned non-array result");
+ return err({
+ type: "parse_error",
+ message: "osascript returned non-array result",
+ });
+ }
+ return ok(events);
+ } catch (parseErr) {
+ const message =
+ parseErr instanceof Error ? parseErr.message : String(parseErr);
+ log(`failed to parse calendar JSON: ${message}`);
+ return err({ type: "parse_error", message });
}
-
- return events;
} catch (e) {
- log(
- `failed to read Apple Calendar: ${e instanceof Error ? e.message : String(e)}`,
- );
- return [];
+ const message = e instanceof Error ? e.message : String(e);
+ log(`failed to read Apple Calendar: ${message}`);
+ return err({ type: "exception", message });
}
}
diff --git a/src/channels/calendar/index.ts b/src/channels/calendar/index.ts
index 3465b77..847742e 100644
--- a/src/channels/calendar/index.ts
+++ b/src/channels/calendar/index.ts
@@ -18,6 +18,53 @@ import { readAppleCalendar, type SpawnFn } from "./apple-calendar";
const log = createLogger("wilson:calendar");
+// --- Recovery types ---
+
+export type RecoveryAction = "kill_osascript" | "restart_calendar";
+
+export type RecoverFn = (action: RecoveryAction) => Promise;
+
+/**
+ * Default recovery implementation using system commands.
+ * Returns true if recovery command executed successfully.
+ */
+const defaultRecover: RecoverFn = async (action) => {
+ try {
+ if (action === "kill_osascript") {
+ // Kill any hung osascript processes related to Calendar
+ const proc = Bun.spawn(["pkill", "-9", "-f", "osascript.*Calendar"], {
+ stdout: "pipe",
+ stderr: "pipe",
+ });
+ await proc.exited;
+ // pkill returns 0 if processes killed, 1 if none found - both are "success"
+ return true;
+ } else if (action === "restart_calendar") {
+ // Quit Calendar.app gracefully then relaunch
+ const quit = Bun.spawn(
+ ["osascript", "-e", 'tell app "Calendar" to quit'],
+ {
+ stdout: "pipe",
+ stderr: "pipe",
+ },
+ );
+ await quit.exited;
+ // Wait a moment for app to fully quit
+ await new Promise((r) => setTimeout(r, 1000));
+ // Relaunch Calendar.app
+ const launch = Bun.spawn(["open", "-a", "Calendar"], {
+ stdout: "pipe",
+ stderr: "pipe",
+ });
+ const exitCode = await launch.exited;
+ return exitCode === 0;
+ }
+ return false;
+ } catch {
+ return false;
+ }
+};
+
// --- Config ---
export interface CalendarChannelConfig {
@@ -27,6 +74,14 @@ export interface CalendarChannelConfig {
includeCalendars?: string[];
}
+// --- Recovery constants ---
+
+/** Number of consecutive timeouts before attempting recovery */
+const RECOVERY_THRESHOLD = 3;
+
+/** Minimum time between recovery attempts (5 minutes) */
+const RECOVERY_COOLDOWN_MS = 5 * 60 * 1000;
+
// --- Channel ---
export class CalendarChannel implements Channel {
@@ -51,21 +106,42 @@ export class CalendarChannel implements Channel {
eventsPosted: 0,
status: "healthy" as string,
error: null as string | null,
+ consecutiveFailures: 0,
+ lastRecoveryAt: null as Date | null,
};
+ // Recovery function for dependency injection (testability)
+ private recoverFn: RecoverFn;
+
constructor(
private cortex: CortexClient,
private config: CalendarChannelConfig,
stateLoaderOrSpawnFn?: StateLoader | SpawnFn,
- spawnFn?: SpawnFn,
+ spawnFnOrRecover?: SpawnFn | RecoverFn,
+ recoverFn?: RecoverFn,
) {
- // Support both old signature (cortex, config, spawnFn) and new (cortex, config, stateLoader, spawnFn)
+ // Support both old signature (cortex, config, spawnFn) and new (cortex, config, stateLoader, spawnFn, recoverFn)
if (typeof stateLoaderOrSpawnFn === "function") {
this.stateLoader = null;
this.spawnFn = stateLoaderOrSpawnFn;
+ this.recoverFn = defaultRecover;
} else {
this.stateLoader = stateLoaderOrSpawnFn ?? null;
- this.spawnFn = spawnFn;
+ // spawnFnOrRecover could be SpawnFn or RecoverFn - detect by arity/name
+ if (typeof spawnFnOrRecover === "function") {
+ // If recoverFn is also provided, spawnFnOrRecover is SpawnFn
+ if (recoverFn) {
+ this.spawnFn = spawnFnOrRecover as SpawnFn;
+ this.recoverFn = recoverFn;
+ } else {
+ // Only one function provided - assume it's SpawnFn for backwards compat
+ this.spawnFn = spawnFnOrRecover as SpawnFn;
+ this.recoverFn = defaultRecover;
+ }
+ } else {
+ this.spawnFn = undefined;
+ this.recoverFn = defaultRecover;
+ }
}
}
@@ -105,6 +181,7 @@ export class CalendarChannel implements Channel {
eventsPosted: this.state.eventsPosted,
status: this.state.status as ChannelStats["status"],
error: this.state.error,
+ consecutiveFailures: this.state.consecutiveFailures ?? 0,
};
}
// Fallback to in-memory state for tests
@@ -114,6 +191,7 @@ export class CalendarChannel implements Channel {
eventsPosted: this.memoryState.eventsPosted,
status: this.memoryState.status as ChannelStats["status"],
error: this.memoryState.error,
+ consecutiveFailures: this.memoryState.consecutiveFailures,
};
}
@@ -135,16 +213,53 @@ export class CalendarChannel implements Channel {
}
// Read events
- const events = await readAppleCalendar({
+ const result = await readAppleCalendar({
lookAheadDays: windowDays,
includeCalendars: this.config.includeCalendars,
spawn: this.spawnFn,
});
+ // Handle read errors
+ if (!result.ok) {
+ const error = result.error;
+ s.consecutiveFailures++;
+ if (error.type === "timeout") {
+ log(
+ `sync: osascript timed out (consecutive: ${s.consecutiveFailures})`,
+ );
+ s.status = "degraded";
+ s.error = "Calendar read timed out";
+
+ // Attempt recovery after hitting threshold
+ if (s.consecutiveFailures >= RECOVERY_THRESHOLD) {
+ await this.attemptRecovery(s);
+ }
+ } else if (error.type === "osascript_failed") {
+ log(
+ `sync: osascript failed (exit ${error.exitCode}): ${error.stderr}`,
+ );
+ s.status = "error";
+ s.error = `osascript failed: ${error.stderr}`;
+ } else if (error.type === "parse_error") {
+ log(`sync: parse error: ${error.message}`);
+ s.status = "error";
+ s.error = `Parse error: ${error.message}`;
+ } else {
+ log(`sync: exception: ${error.message}`);
+ s.status = "error";
+ s.error = error.message;
+ }
+ s.lastSyncAt = new Date();
+ return;
+ }
+
+ const events = result.value;
+
// Update lastSyncAt - we successfully read from Apple Calendar
s.lastSyncAt = new Date();
s.status = "healthy";
s.error = null;
+ s.consecutiveFailures = 0;
// Sort for stable hashing
const sorted = [...events].sort(
@@ -168,7 +283,7 @@ export class CalendarChannel implements Channel {
s.lastHash = hash;
// Post to cortex
- const result = await this.cortex.receive({
+ const postResult = await this.cortex.receive({
channel: "calendar",
externalId: `cal-sync-${Date.now()}`,
data: { events: sorted, windowDays },
@@ -176,16 +291,16 @@ export class CalendarChannel implements Channel {
mode: "buffered",
});
- if (result.ok) {
+ if (postResult.ok) {
log(
- `sync: posted ${sorted.length} events (${windowDays}d window, status: ${result.value.status})`,
+ `sync: posted ${sorted.length} events (${windowDays}d window, status: ${postResult.value.status})`,
);
s.lastPostAt = new Date();
s.eventsPosted = sorted.length;
} else {
- log(`sync: cortex error: ${result.error}`);
+ log(`sync: cortex error: ${postResult.error}`);
s.status = "degraded";
- s.error = `Cortex error: ${result.error}`;
+ s.error = `Cortex error: ${postResult.error}`;
}
} catch (e) {
const errorMsg = e instanceof Error ? e.message : String(e);
@@ -195,6 +310,49 @@ export class CalendarChannel implements Channel {
}
}
+ // --- Recovery ---
+
+ /**
+ * Attempt to recover from consecutive timeouts.
+ *
+ * Recovery is rate-limited to once per 5 minutes.
+ * Step 1: Kill hung osascript processes
+ * Step 2: If at double threshold (6 failures), also restart Calendar.app
+ */
+ private async attemptRecovery(
+ s: CalendarChannelState | typeof this.memoryState,
+ ): Promise {
+ const now = Date.now();
+ const lastRecovery = s.lastRecoveryAt?.getTime() ?? 0;
+
+ // Rate-limit recovery attempts
+ if (now - lastRecovery < RECOVERY_COOLDOWN_MS) {
+ log(
+ `recovery: skipping (cooldown, last attempt ${Math.round((now - lastRecovery) / 1000)}s ago)`,
+ );
+ return;
+ }
+
+ s.lastRecoveryAt = new Date();
+
+ // Step 1: Kill hung osascript processes
+ log("recovery: killing hung osascript processes");
+ await this.recoverFn("kill_osascript");
+
+ // Step 2: If at double threshold, also restart Calendar.app
+ if (s.consecutiveFailures >= RECOVERY_THRESHOLD * 2) {
+ log("recovery: restarting Calendar.app");
+ const restartSuccess = await this.recoverFn("restart_calendar");
+ if (restartSuccess) {
+ log("recovery: Calendar.app restarted, will retry on next sync");
+ } else {
+ log("recovery: Calendar.app restart failed");
+ }
+ } else {
+ log("recovery: osascript processes killed, will retry on next sync");
+ }
+ }
+
// --- Test helpers ---
/** @internal — exposed for testing */
diff --git a/src/channels/index.ts b/src/channels/index.ts
index 4ff89f8..c2c70f7 100644
--- a/src/channels/index.ts
+++ b/src/channels/index.ts
@@ -23,6 +23,8 @@ export interface ChannelStats {
status: "healthy" | "degraded" | "error";
/** Last error message if status is error/degraded. */
error: string | null;
+ /** Number of consecutive failures (timeouts or errors). Resets on success. */
+ consecutiveFailures: number;
}
// --- Channel interface ---
diff --git a/src/state/calendar.ts b/src/state/calendar.ts
index a3f4b84..d352dc3 100644
--- a/src/state/calendar.ts
+++ b/src/state/calendar.ts
@@ -28,4 +28,10 @@ export class CalendarChannelState {
/** Date of last extended sync ("YYYY-MM-DD"). */
@Field("string") lastExtendedSyncDate: string | null = null;
+
+ /** Number of consecutive failures (timeouts or errors). Resets on success. */
+ @Field("number") consecutiveFailures: number = 0;
+
+ /** Timestamp of last recovery attempt (rate-limiting). */
+ @Field("date") lastRecoveryAt: Date | null = null;
}
diff --git a/test/apple-calendar.test.ts b/test/apple-calendar.test.ts
index fe1d2ba..9655007 100644
--- a/test/apple-calendar.test.ts
+++ b/test/apple-calendar.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test";
import {
type CalendarEvent,
+ type CalendarReadError,
readAppleCalendar,
type SpawnFn,
} from "../src/channels/calendar/apple-calendar";
@@ -35,13 +36,16 @@ describe("readAppleCalendar", () => {
});
const result = await readAppleCalendar({ lookAheadDays: 14, spawn });
- expect(result).toEqual(events);
- expect(result.length).toBe(2);
- expect(result[0].title).toBe("Team Meeting");
- expect(result[1].calendarName).toBe("Home");
+ expect(result.ok).toBe(true);
+ if (result.ok) {
+ expect(result.value).toEqual(events);
+ expect(result.value.length).toBe(2);
+ expect(result.value[0].title).toBe("Team Meeting");
+ expect(result.value[1].calendarName).toBe("Home");
+ }
});
- test("returns empty array on osascript failure", async () => {
+ test("returns osascript_failed error on non-timeout failure", async () => {
const spawn: SpawnFn = async () => ({
exitCode: 1,
stdout: "",
@@ -49,7 +53,16 @@ describe("readAppleCalendar", () => {
});
const result = await readAppleCalendar({ lookAheadDays: 14, spawn });
- expect(result).toEqual([]);
+ expect(result.ok).toBe(false);
+ if (!result.ok) {
+ expect(result.error.type).toBe("osascript_failed");
+ if (result.error.type === "osascript_failed") {
+ expect(result.error.exitCode).toBe(1);
+ expect(result.error.stderr).toBe(
+ "osascript: not available on this platform",
+ );
+ }
+ }
});
test("returns empty array on empty calendar", async () => {
@@ -60,19 +73,28 @@ describe("readAppleCalendar", () => {
});
const result = await readAppleCalendar({ lookAheadDays: 14, spawn });
- expect(result).toEqual([]);
+ expect(result.ok).toBe(true);
+ if (result.ok) {
+ expect(result.value).toEqual([]);
+ }
});
- test("returns empty array when spawn throws", async () => {
+ test("returns exception error when spawn throws", async () => {
const spawn: SpawnFn = async () => {
throw new Error("spawn failed");
};
const result = await readAppleCalendar({ lookAheadDays: 14, spawn });
- expect(result).toEqual([]);
+ expect(result.ok).toBe(false);
+ if (!result.ok) {
+ expect(result.error.type).toBe("exception");
+ if (result.error.type === "exception") {
+ expect(result.error.message).toBe("spawn failed");
+ }
+ }
});
- test("returns empty array on invalid JSON output", async () => {
+ test("returns parse_error on invalid JSON output", async () => {
const spawn: SpawnFn = async () => ({
exitCode: 0,
stdout: "not valid json",
@@ -80,7 +102,10 @@ describe("readAppleCalendar", () => {
});
const result = await readAppleCalendar({ lookAheadDays: 14, spawn });
- expect(result).toEqual([]);
+ expect(result.ok).toBe(false);
+ if (!result.ok) {
+ expect(result.error.type).toBe("parse_error");
+ }
});
test("returns empty array when output is empty string", async () => {
@@ -91,13 +116,14 @@ describe("readAppleCalendar", () => {
});
const result = await readAppleCalendar({ lookAheadDays: 14, spawn });
- expect(result).toEqual([]);
+ expect(result.ok).toBe(true);
+ if (result.ok) {
+ expect(result.value).toEqual([]);
+ }
});
describe("timeout handling", () => {
- test("returns empty array on timeout (graceful fallback)", async () => {
- // Simulate a spawn that never resolves (would time out in real usage)
- // We test by returning the timeout result directly
+ test("returns timeout error on timeout", async () => {
const spawn: SpawnFn = async () => ({
exitCode: -1,
stdout: "",
@@ -105,7 +131,24 @@ describe("readAppleCalendar", () => {
});
const result = await readAppleCalendar({ lookAheadDays: 14, spawn });
- expect(result).toEqual([]);
+ expect(result.ok).toBe(false);
+ if (!result.ok) {
+ expect(result.error.type).toBe("timeout");
+ }
+ });
+
+ test("returns osascript_failed error on non-timeout failure", async () => {
+ const spawn: SpawnFn = async () => ({
+ exitCode: 1,
+ stdout: "",
+ stderr: "some other error",
+ });
+
+ const result = await readAppleCalendar({ lookAheadDays: 14, spawn });
+ expect(result.ok).toBe(false);
+ if (!result.ok) {
+ expect(result.error.type).toBe("osascript_failed");
+ }
});
});
diff --git a/test/calendar.test.ts b/test/calendar.test.ts
index 7a35711..cb8178a 100644
--- a/test/calendar.test.ts
+++ b/test/calendar.test.ts
@@ -250,13 +250,11 @@ describe("CalendarChannel", () => {
// Should not throw
await channel.start();
- // osascript failed → empty events → still posts initial (empty array hash differs from null)
- expect(cortex.calls.length).toBe(1);
- const data = cortex.calls[0].data as {
- events: CalendarEvent[];
- windowDays: number;
- };
- expect(data.events).toEqual([]);
+ // osascript failed → error state, no cortex post
+ expect(cortex.calls.length).toBe(0);
+ const stats = channel.getStats();
+ expect(stats.status).toBe("error");
+ expect(stats.error).toContain("osascript");
});
describe("stats tracking", () => {
@@ -341,9 +339,9 @@ describe("CalendarChannel", () => {
expect(stats.error).toContain("Cortex");
});
- test("getStats() remains healthy when osascript returns empty (graceful fallback)", async () => {
+ test("getStats() shows error when osascript throws", async () => {
const cortex = makeMockCortex();
- // Spawn that throws — readAppleCalendar catches it and returns []
+ // Spawn that throws — readAppleCalendar catches it and returns Err
const errorSpawn = async () => {
throw new Error("spawn failed");
};
@@ -352,11 +350,12 @@ describe("CalendarChannel", () => {
await channel.start();
- // readAppleCalendar catches the error and returns [] — sync proceeds normally
+ // readAppleCalendar catches the error and returns Err — sync records error state
const stats = channel.getStats();
- expect(stats.status).toBe("healthy"); // graceful degradation
+ expect(stats.status).toBe("error");
+ expect(stats.error).toContain("spawn failed");
expect(stats.lastSyncAt).toBeGreaterThan(0);
- expect(stats.eventsPosted).toBe(0); // empty events
+ expect(stats.eventsPosted).toBe(0);
});
test("getStats() returns a copy (immutable)", async () => {
@@ -376,6 +375,228 @@ describe("CalendarChannel", () => {
});
});
+ describe("consecutive failures tracking", () => {
+ test("consecutiveFailures starts at 0", () => {
+ const cortex = makeMockCortex();
+ channel = new CalendarChannel(
+ cortex,
+ DEFAULT_CONFIG,
+ makeSpawn(SAMPLE_EVENTS),
+ );
+
+ const stats = channel.getStats();
+ expect(stats.consecutiveFailures).toBe(0);
+ });
+
+ test("consecutiveFailures increments on timeout", async () => {
+ const cortex = makeMockCortex();
+ const timeoutSpawn = async () => ({
+ exitCode: -1,
+ stdout: "",
+ stderr: "osascript timed out",
+ });
+
+ channel = new CalendarChannel(cortex, DEFAULT_CONFIG, timeoutSpawn);
+ await channel.start();
+
+ expect(channel.getStats().consecutiveFailures).toBe(1);
+
+ // Trigger another sync
+ await channel.sync();
+ expect(channel.getStats().consecutiveFailures).toBe(2);
+ });
+
+ test("consecutiveFailures resets on success", async () => {
+ const cortex = makeMockCortex();
+ let callCount = 0;
+
+ // First two calls timeout, third succeeds
+ const mixedSpawn = async () => {
+ callCount++;
+ if (callCount <= 2) {
+ return { exitCode: -1, stdout: "", stderr: "osascript timed out" };
+ }
+ return {
+ exitCode: 0,
+ stdout: JSON.stringify(SAMPLE_EVENTS),
+ stderr: "",
+ };
+ };
+
+ channel = new CalendarChannel(cortex, DEFAULT_CONFIG, mixedSpawn);
+ await channel.start(); // timeout 1
+ expect(channel.getStats().consecutiveFailures).toBe(1);
+
+ await channel.sync(); // timeout 2
+ expect(channel.getStats().consecutiveFailures).toBe(2);
+
+ await channel.sync(); // success
+ expect(channel.getStats().consecutiveFailures).toBe(0);
+ expect(channel.getStats().status).toBe("healthy");
+ });
+
+ test("consecutiveFailures increments on osascript failure", async () => {
+ const cortex = makeMockCortex();
+ const failSpawn = async () => ({
+ exitCode: 1,
+ stdout: "",
+ stderr: "osascript: not available",
+ });
+
+ channel = new CalendarChannel(cortex, DEFAULT_CONFIG, failSpawn);
+ await channel.start();
+
+ expect(channel.getStats().consecutiveFailures).toBe(1);
+ expect(channel.getStats().status).toBe("error");
+ });
+ });
+
+ describe("recovery logic", () => {
+ test("triggers kill_osascript recovery after 3 consecutive timeouts", async () => {
+ const cortex = makeMockCortex();
+ const recoveryActions: string[] = [];
+
+ const timeoutSpawn = async () => ({
+ exitCode: -1,
+ stdout: "",
+ stderr: "osascript timed out",
+ });
+
+ const mockRecover = async (action: string) => {
+ recoveryActions.push(action);
+ return true;
+ };
+
+ channel = new CalendarChannel(
+ cortex,
+ DEFAULT_CONFIG,
+ undefined,
+ timeoutSpawn,
+ mockRecover,
+ );
+
+ await channel.start(); // timeout 1
+ await channel.sync(); // timeout 2
+ await channel.sync(); // timeout 3 - should trigger recovery
+
+ expect(recoveryActions).toEqual(["kill_osascript"]);
+ });
+
+ test("triggers restart_calendar after 6 consecutive timeouts", async () => {
+ const cortex = makeMockCortex();
+ const recoveryActions: string[] = [];
+
+ const timeoutSpawn = async () => ({
+ exitCode: -1,
+ stdout: "",
+ stderr: "osascript timed out",
+ });
+
+ // Mock recover that "succeeds" but doesn't actually fix the problem
+ const mockRecover = async (action: string) => {
+ recoveryActions.push(action);
+ return true;
+ };
+
+ channel = new CalendarChannel(
+ cortex,
+ DEFAULT_CONFIG,
+ undefined,
+ timeoutSpawn,
+ mockRecover,
+ );
+
+ // Trigger 6 consecutive timeouts
+ await channel.start(); // 1
+ await channel.sync(); // 2
+ await channel.sync(); // 3 - kill_osascript
+ // Need to reset cooldown for testing - we'll use a fresh channel
+ await channel.stop();
+
+ // Create new channel with state already at 5 failures
+ const channel2 = new CalendarChannel(
+ cortex,
+ DEFAULT_CONFIG,
+ undefined,
+ timeoutSpawn,
+ mockRecover,
+ );
+ // Use internal method to set up state
+ (channel2 as any).memoryState.consecutiveFailures = 5;
+ (channel2 as any).running = true;
+
+ await channel2.sync(); // 6 - should trigger restart_calendar
+
+ expect(recoveryActions).toContain("restart_calendar");
+ await channel2.stop();
+ });
+
+ test("respects recovery cooldown (5 minutes)", async () => {
+ const cortex = makeMockCortex();
+ const recoveryActions: string[] = [];
+
+ const timeoutSpawn = async () => ({
+ exitCode: -1,
+ stdout: "",
+ stderr: "osascript timed out",
+ });
+
+ const mockRecover = async (action: string) => {
+ recoveryActions.push(action);
+ return true;
+ };
+
+ channel = new CalendarChannel(
+ cortex,
+ DEFAULT_CONFIG,
+ undefined,
+ timeoutSpawn,
+ mockRecover,
+ );
+
+ await channel.start(); // 1
+ await channel.sync(); // 2
+ await channel.sync(); // 3 - triggers recovery
+ await channel.sync(); // 4 - should NOT trigger recovery (cooldown)
+
+ // Only one recovery action should have been triggered
+ expect(recoveryActions).toEqual(["kill_osascript"]);
+ });
+
+ test("no recovery triggered for non-timeout errors", async () => {
+ const cortex = makeMockCortex();
+ const recoveryActions: string[] = [];
+
+ // Non-timeout error (osascript failed, not timed out)
+ const failSpawn = async () => ({
+ exitCode: 1,
+ stdout: "",
+ stderr: "osascript: not available",
+ });
+
+ const mockRecover = async (action: string) => {
+ recoveryActions.push(action);
+ return true;
+ };
+
+ channel = new CalendarChannel(
+ cortex,
+ DEFAULT_CONFIG,
+ undefined,
+ failSpawn,
+ mockRecover,
+ );
+
+ await channel.start(); // 1
+ await channel.sync(); // 2
+ await channel.sync(); // 3
+ await channel.sync(); // 4
+
+ // No recovery should be triggered for non-timeout errors
+ expect(recoveryActions).toEqual([]);
+ });
+ });
+
describe("includeCalendars filtering", () => {
test("passes includeCalendars to readAppleCalendar", async () => {
const cortex = makeMockCortex();
diff --git a/test/channels.test.ts b/test/channels.test.ts
index 5735f60..0f38d02 100644
--- a/test/channels.test.ts
+++ b/test/channels.test.ts
@@ -29,6 +29,7 @@ function makeChannel(name: string, calls: string[]): Channel {
eventsPosted: 0,
status: "healthy",
error: null,
+ consecutiveFailures: 0,
};
},
};