HARDEN: Postgres pool config + dependency-aware /ready

apps/server/src/app.ts

@@ -1,7 +1,7 @@
 import { Elysia } from "elysia";
 import { cors } from "@elysiajs/cors";
 import { openapi } from "@elysiajs/openapi";
-import type { Database } from "./db";
+import { pingDatabase, type Database } from "./db";
 import { githubAuth, cliAuth } from "./auth/github";
 import { ingestRoutes } from "./ingest/routes";
 import { socialRoutes } from "./social/routes";
@@ -22,40 +22,55 @@ const INSTALL_SCRIPT = await Bun.file(
 ).text();
 
 export function createApp(db: Database, redis: RedisBridge) {
-  return new Elysia()
-    .use(
-      cors({
-        allowedHeaders: [
-          "content-type",
-          "mcp-session-id",
-          "mcp-protocol-version",
-          "accept",
-          "authorization",
-          "cookie",
-        ],
-        exposeHeaders: ["mcp-session-id", "www-authenticate"],
-      }),
-    )
-    .use(openapi())
-    .get("/health", () => ({ status: "ok" }))
-    .get("/install.sh", ({ set }) => {
-      set.headers["content-type"] = "text/plain";
-      return INSTALL_SCRIPT;
-    })
-    .use(githubAuth(db, redis))
-    .use(cliAuth(db))
-    .use(ingestRoutes(db, redis))
-    .use(socialRoutes(db))
-    .use(sessionRoutes(db))
-    .use(userRoutes(db))
-    .use(claimRoutes(db))
-    .use(orgsRoutes(db))
-    .use(deviceReposRoutes(db))
-    .use(chatRoutes(db))
-    .use(managedAgentSessionRoutes(db))
-    .use(mcpOAuthRoutes(db))
-    .use(mcpRoutes())
-    .use(spotifyPresenceRoutes(db, redis))
-    .use(presenceReadRoutes(db, redis))
-    .use(wsHandler(db, redis));
+  return (
+    new Elysia()
+      .use(
+        cors({
+          allowedHeaders: [
+            "content-type",
+            "mcp-session-id",
+            "mcp-protocol-version",
+            "accept",
+            "authorization",
+            "cookie",
+          ],
+          exposeHeaders: ["mcp-session-id", "www-authenticate"],
+        }),
+      )
+      .use(openapi())
+      // Liveness — process is up enough to answer HTTP. Intentionally cheap so
+      // a saturated dependency (DB pool exhausted, Redis paused) doesn't get
+      // the orchestrator to kill an otherwise-recoverable replica.
+      .get("/health", () => ({ status: "ok" }))
+      // Readiness — process is fit to serve traffic. Pings each load-bearing
+      // dependency with a short timeout. Orchestrators should route traffic
+      // only when this returns 200; a 503 here means "drain me" without
+      // necessarily restarting.
+      .get("/ready", async ({ set }) => {
+        const [database, redisOk] = await Promise.all([pingDatabase(), redis.ping()]);
+        const ok = database && redisOk;
+        if (!ok) set.status = 503;
+        return { status: ok ? "ready" : "degraded", database, redis: redisOk };
+      })
+      .get("/install.sh", ({ set }) => {
+        set.headers["content-type"] = "text/plain";
+        return INSTALL_SCRIPT;
+      })
+      .use(githubAuth(db, redis))
+      .use(cliAuth(db))
+      .use(ingestRoutes(db, redis))
+      .use(socialRoutes(db))
+      .use(sessionRoutes(db))
+      .use(userRoutes(db))
+      .use(claimRoutes(db))
+      .use(orgsRoutes(db))
+      .use(deviceReposRoutes(db))
+      .use(chatRoutes(db))
+      .use(managedAgentSessionRoutes(db))
+      .use(mcpOAuthRoutes(db))
+      .use(mcpRoutes())
+      .use(spotifyPresenceRoutes(db, redis))
+      .use(presenceReadRoutes(db, redis))
+      .use(wsHandler(db, redis))
+  );
 }

apps/server/src/config.ts

@@ -31,6 +31,19 @@ export const config = Object.freeze({
   // either exhaust memory or hold a slot for hours. Defaults: 50 MB / 60 s.
   ingestMaxBytes: parseInt(process.env.INGEST_MAX_BYTES || String(50 * 1024 * 1024), 10),
   ingestDeadlineMs: parseInt(process.env.INGEST_DEADLINE_MS || "60000", 10),
+  // Postgres pool sizing. Without explicit limits the postgres.js default of
+  // max=10/idle_timeout=0 means a deployment under load slowly accumulates
+  // idle connections until the database refuses new ones. Production should
+  // tune dbPoolMax against the Postgres `max_connections` budget divided by
+  // replica count.
+  dbPoolMax: parseInt(process.env.DB_POOL_MAX || "10", 10),
+  dbIdleTimeoutSec: parseInt(process.env.DB_IDLE_TIMEOUT_SEC || "30", 10),
+  dbConnectTimeoutSec: parseInt(process.env.DB_CONNECT_TIMEOUT_SEC || "10", 10),
+  dbMaxLifetimeSec: parseInt(process.env.DB_MAX_LIFETIME_SEC || "1800", 10),
+  // Hard ceiling on a single query's wall-clock — kills runaway analyzer or
+  // feed queries before they drain the pool. Set on every new connection via
+  // postgres.js `connection.statement_timeout`.
+  dbStatementTimeoutMs: parseInt(process.env.DB_STATEMENT_TIMEOUT_MS || "30000", 10),
 });
 
 export type Config = typeof config;

apps/server/src/db/index.ts

@@ -3,8 +3,34 @@ import postgres from "postgres";
 import { config } from "../config";
 import * as schema from "./schema";
 
-const client = postgres(config.databaseUrl);
+const client = postgres(config.databaseUrl, {
+  max: config.dbPoolMax,
+  idle_timeout: config.dbIdleTimeoutSec,
+  connect_timeout: config.dbConnectTimeoutSec,
+  max_lifetime: config.dbMaxLifetimeSec,
+  // Sets PostgreSQL's `statement_timeout` GUC on every new connection so a
+  // runaway query is killed at the database before it monopolizes a pool slot.
+  connection: { statement_timeout: config.dbStatementTimeoutMs },
+});
 
 export const db = drizzle(client, { schema });
 
 export type Database = typeof db;
+
+/** Liveness ping for `/ready` — short-bounded so a hung database can't pin
+ *  the readiness probe. Returns true on a 1-row select; false on any error
+ *  (timeout, connection refusal, etc.). */
+export async function pingDatabase(timeoutMs = 2_000): Promise<boolean> {
+  let timer: NodeJS.Timeout | undefined;
+  try {
+    const timeout = new Promise<never>((_, reject) => {
+      timer = setTimeout(() => reject(new Error("db ping timeout")), timeoutMs);
+    });
+    await Promise.race([client`SELECT 1`, timeout]);
+    return true;
+  } catch {
+    return false;
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}

apps/server/src/ws/redis-bridge.ts

@@ -130,4 +130,22 @@ export class RedisBridge {
     this.sub.disconnect();
     this.pub.disconnect();
   }
+
+  /** Liveness ping for `/ready`. Short-bounded so a hung Redis can't pin the
+   *  readiness probe. Returns true on PONG; false on any error or timeout. */
+  async ping(timeoutMs = 2_000): Promise<boolean> {
+    if (!this.connected) return false;
+    let timer: NodeJS.Timeout | undefined;
+    try {
+      const timeout = new Promise<never>((_, reject) => {
+        timer = setTimeout(() => reject(new Error("redis ping timeout")), timeoutMs);
+      });
+      const result = await Promise.race([this.pub.ping(), timeout]);
+      return result === "PONG";
+    } catch {
+      return false;
+    } finally {
+      if (timer) clearTimeout(timer);
+    }
+  }
 }

apps/server/test/health.test.ts

@@ -0,0 +1,46 @@
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { createApp } from "../src/app";
+import { db, pingDatabase } from "../src/db";
+import { RedisBridge } from "../src/ws/redis-bridge";
+
+let app: ReturnType<typeof createApp>;
+let baseUrl: string;
+let redis: RedisBridge;
+
+beforeAll(async () => {
+  redis = new RedisBridge();
+  await redis.connect();
+  app = createApp(db, redis);
+  app.listen(0);
+  baseUrl = `http://localhost:${app.server!.port}`;
+}, 30_000);
+
+afterAll(async () => {
+  await app.stop();
+  await redis.disconnect();
+});
+
+describe("liveness / readiness", () => {
+  it("/health returns 200 ok without checking dependencies", async () => {
+    const res = await fetch(`${baseUrl}/health`);
+    expect(res.status).toBe(200);
+    expect(await res.json()).toEqual({ status: "ok" });
+  });
+
+  it("/ready returns 200 ready when database and redis are reachable", async () => {
+    const res = await fetch(`${baseUrl}/ready`);
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { status: string; database: boolean; redis: boolean };
+    expect(body.status).toBe("ready");
+    expect(body.database).toBe(true);
+    expect(body.redis).toBe(true);
+  });
+
+  it("pingDatabase resolves to true against the live test database", async () => {
+    expect(await pingDatabase()).toBe(true);
+  });
+
+  it("RedisBridge.ping returns true against live Redis", async () => {
+    expect(await redis.ping()).toBe(true);
+  });
+});