diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 78d426a4..2d5bad97 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -1,7 +1,7 @@ import { Elysia } from "elysia"; import { cors } from "@elysiajs/cors"; import { openapi } from "@elysiajs/openapi"; -import type { Database } from "./db"; +import { pingDatabase, type Database } from "./db"; import { githubAuth, cliAuth } from "./auth/github"; import { ingestRoutes } from "./ingest/routes"; import { socialRoutes } from "./social/routes"; @@ -22,40 +22,55 @@ const INSTALL_SCRIPT = await Bun.file( ).text(); export function createApp(db: Database, redis: RedisBridge) { - return new Elysia() - .use( - cors({ - allowedHeaders: [ - "content-type", - "mcp-session-id", - "mcp-protocol-version", - "accept", - "authorization", - "cookie", - ], - exposeHeaders: ["mcp-session-id", "www-authenticate"], - }), - ) - .use(openapi()) - .get("/health", () => ({ status: "ok" })) - .get("/install.sh", ({ set }) => { - set.headers["content-type"] = "text/plain"; - return INSTALL_SCRIPT; - }) - .use(githubAuth(db, redis)) - .use(cliAuth(db)) - .use(ingestRoutes(db, redis)) - .use(socialRoutes(db)) - .use(sessionRoutes(db)) - .use(userRoutes(db)) - .use(claimRoutes(db)) - .use(orgsRoutes(db)) - .use(deviceReposRoutes(db)) - .use(chatRoutes(db)) - .use(managedAgentSessionRoutes(db)) - .use(mcpOAuthRoutes(db)) - .use(mcpRoutes()) - .use(spotifyPresenceRoutes(db, redis)) - .use(presenceReadRoutes(db, redis)) - .use(wsHandler(db, redis)); + return ( + new Elysia() + .use( + cors({ + allowedHeaders: [ + "content-type", + "mcp-session-id", + "mcp-protocol-version", + "accept", + "authorization", + "cookie", + ], + exposeHeaders: ["mcp-session-id", "www-authenticate"], + }), + ) + .use(openapi()) + // Liveness — process is up enough to answer HTTP. Intentionally cheap so + // a saturated dependency (DB pool exhausted, Redis paused) doesn't get + // the orchestrator to kill an otherwise-recoverable replica. + .get("/health", () => ({ status: "ok" })) + // Readiness — process is fit to serve traffic. Pings each load-bearing + // dependency with a short timeout. Orchestrators should route traffic + // only when this returns 200; a 503 here means "drain me" without + // necessarily restarting. + .get("/ready", async ({ set }) => { + const [database, redisOk] = await Promise.all([pingDatabase(), redis.ping()]); + const ok = database && redisOk; + if (!ok) set.status = 503; + return { status: ok ? "ready" : "degraded", database, redis: redisOk }; + }) + .get("/install.sh", ({ set }) => { + set.headers["content-type"] = "text/plain"; + return INSTALL_SCRIPT; + }) + .use(githubAuth(db, redis)) + .use(cliAuth(db)) + .use(ingestRoutes(db, redis)) + .use(socialRoutes(db)) + .use(sessionRoutes(db)) + .use(userRoutes(db)) + .use(claimRoutes(db)) + .use(orgsRoutes(db)) + .use(deviceReposRoutes(db)) + .use(chatRoutes(db)) + .use(managedAgentSessionRoutes(db)) + .use(mcpOAuthRoutes(db)) + .use(mcpRoutes()) + .use(spotifyPresenceRoutes(db, redis)) + .use(presenceReadRoutes(db, redis)) + .use(wsHandler(db, redis)) + ); } diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index 1e043f17..040a086e 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -31,6 +31,19 @@ export const config = Object.freeze({ // either exhaust memory or hold a slot for hours. Defaults: 50 MB / 60 s. ingestMaxBytes: parseInt(process.env.INGEST_MAX_BYTES || String(50 * 1024 * 1024), 10), ingestDeadlineMs: parseInt(process.env.INGEST_DEADLINE_MS || "60000", 10), + // Postgres pool sizing. Without explicit limits the postgres.js default of + // max=10/idle_timeout=0 means a deployment under load slowly accumulates + // idle connections until the database refuses new ones. Production should + // tune dbPoolMax against the Postgres `max_connections` budget divided by + // replica count. + dbPoolMax: parseInt(process.env.DB_POOL_MAX || "10", 10), + dbIdleTimeoutSec: parseInt(process.env.DB_IDLE_TIMEOUT_SEC || "30", 10), + dbConnectTimeoutSec: parseInt(process.env.DB_CONNECT_TIMEOUT_SEC || "10", 10), + dbMaxLifetimeSec: parseInt(process.env.DB_MAX_LIFETIME_SEC || "1800", 10), + // Hard ceiling on a single query's wall-clock — kills runaway analyzer or + // feed queries before they drain the pool. Set on every new connection via + // postgres.js `connection.statement_timeout`. + dbStatementTimeoutMs: parseInt(process.env.DB_STATEMENT_TIMEOUT_MS || "30000", 10), }); export type Config = typeof config; diff --git a/apps/server/src/db/index.ts b/apps/server/src/db/index.ts index d96d1472..525ed0a5 100644 --- a/apps/server/src/db/index.ts +++ b/apps/server/src/db/index.ts @@ -3,8 +3,34 @@ import postgres from "postgres"; import { config } from "../config"; import * as schema from "./schema"; -const client = postgres(config.databaseUrl); +const client = postgres(config.databaseUrl, { + max: config.dbPoolMax, + idle_timeout: config.dbIdleTimeoutSec, + connect_timeout: config.dbConnectTimeoutSec, + max_lifetime: config.dbMaxLifetimeSec, + // Sets PostgreSQL's `statement_timeout` GUC on every new connection so a + // runaway query is killed at the database before it monopolizes a pool slot. + connection: { statement_timeout: config.dbStatementTimeoutMs }, +}); export const db = drizzle(client, { schema }); export type Database = typeof db; + +/** Liveness ping for `/ready` — short-bounded so a hung database can't pin + * the readiness probe. Returns true on a 1-row select; false on any error + * (timeout, connection refusal, etc.). */ +export async function pingDatabase(timeoutMs = 2_000): Promise { + let timer: NodeJS.Timeout | undefined; + try { + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("db ping timeout")), timeoutMs); + }); + await Promise.race([client`SELECT 1`, timeout]); + return true; + } catch { + return false; + } finally { + if (timer) clearTimeout(timer); + } +} diff --git a/apps/server/src/ws/redis-bridge.ts b/apps/server/src/ws/redis-bridge.ts index c30e55bf..c076694f 100644 --- a/apps/server/src/ws/redis-bridge.ts +++ b/apps/server/src/ws/redis-bridge.ts @@ -130,4 +130,22 @@ export class RedisBridge { this.sub.disconnect(); this.pub.disconnect(); } + + /** Liveness ping for `/ready`. Short-bounded so a hung Redis can't pin the + * readiness probe. Returns true on PONG; false on any error or timeout. */ + async ping(timeoutMs = 2_000): Promise { + if (!this.connected) return false; + let timer: NodeJS.Timeout | undefined; + try { + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("redis ping timeout")), timeoutMs); + }); + const result = await Promise.race([this.pub.ping(), timeout]); + return result === "PONG"; + } catch { + return false; + } finally { + if (timer) clearTimeout(timer); + } + } } diff --git a/apps/server/test/health.test.ts b/apps/server/test/health.test.ts new file mode 100644 index 00000000..b3a83da2 --- /dev/null +++ b/apps/server/test/health.test.ts @@ -0,0 +1,46 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { createApp } from "../src/app"; +import { db, pingDatabase } from "../src/db"; +import { RedisBridge } from "../src/ws/redis-bridge"; + +let app: ReturnType; +let baseUrl: string; +let redis: RedisBridge; + +beforeAll(async () => { + redis = new RedisBridge(); + await redis.connect(); + app = createApp(db, redis); + app.listen(0); + baseUrl = `http://localhost:${app.server!.port}`; +}, 30_000); + +afterAll(async () => { + await app.stop(); + await redis.disconnect(); +}); + +describe("liveness / readiness", () => { + it("/health returns 200 ok without checking dependencies", async () => { + const res = await fetch(`${baseUrl}/health`); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ status: "ok" }); + }); + + it("/ready returns 200 ready when database and redis are reachable", async () => { + const res = await fetch(`${baseUrl}/ready`); + expect(res.status).toBe(200); + const body = (await res.json()) as { status: string; database: boolean; redis: boolean }; + expect(body.status).toBe("ready"); + expect(body.database).toBe(true); + expect(body.redis).toBe(true); + }); + + it("pingDatabase resolves to true against the live test database", async () => { + expect(await pingDatabase()).toBe(true); + }); + + it("RedisBridge.ping returns true against live Redis", async () => { + expect(await redis.ping()).toBe(true); + }); +});