From 0001a2ebff4cf9f58ab3ca504ec40683be267e66 Mon Sep 17 00:00:00 2001 From: Manjusaka Date: Mon, 22 Jun 2026 19:19:41 +0800 Subject: [PATCH] feat: multi-replica readiness (P1 hardening) Prepare PicImpact to run as multiple replicas behind a load balancer. - auth: shorten the better-auth signed cookie-cache TTL from 30m to 60s. The cookie cache is per-instance in-memory state, so a long TTL let a revoked session keep validating on a replica that still held the cached cookie. 60s bounds cross-replica revocation lag while still absorbing repeat reads. - preprocess tick: add an optional shared-secret gate on POST /api/v1/preprocess-tasks/tick via PREPROCESS_TICK_SECRET + x-preprocess-tick-secret header (constant-time compared, never logged). This is the public-cron driver path for multi-replica; when the env var is unset the endpoint stays open so single-instance / internal-ticker deployments are unaffected. - docs: add docs/multi-replica.md covering single-driver ticker (PREPROCESS_TICKER_ENABLED=false + one external cron), DB connection budgeting (N x pool), session revocation latency, and the interim per-replica data-cache behavior. Document the new env vars in .env.example. Co-Authored-By: Claude Opus 4.8 --- .env.example | 11 ++++- docs/multi-replica.md | 98 ++++++++++++++++++++++++++++++++++++++++ hono/preprocess-tasks.ts | 26 ++++++++++- server/auth/index.ts | 7 ++- 4 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 docs/multi-replica.md diff --git a/.env.example b/.env.example index ac5628c8..4da69bc1 100644 --- a/.env.example +++ b/.env.example @@ -7,4 +7,13 @@ BETTER_AUTH_SECRET=LVm22IOrxM4LO6jeSYxOMkvKKehdBDyL # Base URL of your app BETTER_AUTH_URL=http://localhost:3000 # 禁用 Vercel node.js 帮助程序 -NODEJS_HELPERS=0 \ No newline at end of file +NODEJS_HELPERS=0 + +# ── 多 replica 部署（可选，详见 docs/multi-replica.md）── +# 后台预处理 ticker 开关。不设置时默认仅在生产环境（NODE_ENV=production）开启。 +# 多 replica 部署建议所有实例设为 false，改用单个外部 cron 调用 +# POST /api/v1/preprocess-tasks/tick 驱动队列，避免每个实例各跑一个 ticker。 +# PREPROCESS_TICKER_ENABLED=false +# 给 /api/v1/preprocess-tasks/tick 加共享密钥（可选）。设置后,调用方需在 +# x-preprocess-tick-secret 请求头携带该值,否则返回 401;不设置则该端点开放。 +# PREPROCESS_TICK_SECRET= \ No newline at end of file diff --git a/docs/multi-replica.md b/docs/multi-replica.md new file mode 100644 index 00000000..2433aea2 --- /dev/null +++ b/docs/multi-replica.md @@ -0,0 +1,98 @@ +# Multi-replica deployment + +PicImpact can run as multiple replicas behind a load balancer (e.g. several Node +containers or a Kubernetes Deployment with `replicas > 1`). Most state is already +shared and safe across replicas: + +- **Sessions** are persisted in PostgreSQL (better-auth Prisma adapter), so login + state is consistent across replicas. +- **The image-preprocessing task queue** is claimed with a PostgreSQL advisory + lock + a database lease, so two replicas (or a replica and an external cron) + can never process the same task run concurrently. + +A few per-instance concerns need configuration when you scale past one replica. + +## 1. Background preprocess ticker — run a single driver + +Each replica starts the background preprocess ticker by default in production +(`NODE_ENV=production`). The advisory lock means N replicas will not double-process +work, but they will all poll every ~10s, so N−1 replicas just contend for the lock +and waste CPU. + +Recommended for multi-replica: disable the in-process ticker on every replica and +drive the queue with a single external scheduler. + +```bash +# On every replica: +PREPROCESS_TICKER_ENABLED=false +``` + +Then have one external cron (Kubernetes CronJob, systemd timer, etc.) hit the tick +endpoint every 10–60s: + +```bash +curl -X POST https://your-host/api/v1/preprocess-tasks/tick +``` + +Since this endpoint can be reached publicly, you can optionally protect it with a +shared secret. Set `PREPROCESS_TICK_SECRET` and the cron must send it in the +`x-preprocess-tick-secret` header; calls without a matching secret get `401`: + +```bash +curl -X POST https://your-host/api/v1/preprocess-tasks/tick \ + -H "x-preprocess-tick-secret: $PREPROCESS_TICK_SECRET" +``` + +When `PREPROCESS_TICK_SECRET` is unset the endpoint stays open (so single-instance +and internal-ticker deployments are unaffected). + +> Note: `/api/v1/*` admin endpoints currently rely on client-side auth checks +> only (server-side enforcement is a separate, tracked hardening item), so the +> tick secret is the one explicit server-side gate added here for the public cron +> path. If you expose the app publicly, also restrict the admin API at the +> network layer until server-side auth lands. + +Alternatively, enable the ticker on exactly one replica (e.g. a dedicated worker +pod) and set `PREPROCESS_TICKER_ENABLED=false` on the rest. + +> `PREPROCESS_TICKER_ENABLED` is also useful on serverless, where there is no +> long-lived process to run the ticker — disable it and use the external cron. + +## 2. Database connections + +Each replica opens its own Prisma connection pool. With N replicas the total +connection count is roughly `N × pool_size`. Make sure your PostgreSQL +`max_connections` covers that, or put a pooler such as PgBouncer (transaction +mode) in front of the database. + +## 3. Session revocation latency + +Sessions are stored in the database and shared across replicas, but better-auth +keeps a short-lived signed **cookie cache** in each instance's memory. The cache +TTL is 60s, so a sign-out or session revocation propagates to every replica +within ~60s. Lower it further in `server/auth/index.ts` (`session.cookieCache.maxAge`) +if you need faster propagation. + +## 4. Public data cache (current behavior) + +The public read paths (gallery listings, album nav, public config) are cached +with Next.js' Data Cache and invalidated on admin writes via `revalidateTag`. +That cache is **per-replica in-memory** by default, so a `revalidateTag` call only +busts the cache on the replica that handled the write. Other replicas pick up the +change when their cache entry's safety-net TTL expires: + +- gallery listings: ~60s +- album nav / public config: ~1h (admin writes still bust them instantly on the + writing replica; the TTL only bounds cross-replica propagation) + +If you need instant cross-replica consistency for admin changes, a shared +PostgreSQL-backed cache handler is the planned next step (it routes the Data Cache +through Postgres + `LISTEN/NOTIFY` so an invalidation on one replica is seen by all). + +## 5. In-flight task cancellation + +Cancelling a running preprocess/metadata task signals the replica that owns the +run. If the cancel request lands on a different replica, it is still recorded, and +the running replica stops at the next lease checkpoint (within the lease window). +For rare admin actions this lag is acceptable; a fully cross-replica cancel signal +is a planned follow-up. diff --git a/hono/preprocess-tasks.ts b/hono/preprocess-tasks.ts index 242662bb..ce1c54ad 100644 --- a/hono/preprocess-tasks.ts +++ b/hono/preprocess-tasks.ts @@ -1,5 +1,6 @@ import 'server-only' +import { timingSafeEqual } from 'node:crypto' import { Hono } from 'hono' import { HTTPException } from 'hono/http-exception' @@ -14,10 +15,32 @@ import { } from '~/server/tasks/image-preprocess-service' import { ADMIN_TASK_KEY_PREPROCESS_IMAGES, normalizePreprocessTaskScope } from '~/types/admin-tasks' import { ok } from '~/hono/_lib/response' -import { badRequest, conflict, notFound, serverError } from '~/hono/_lib/errors' +import { badRequest, conflict, notFound, serverError, unauthorized } from '~/hono/_lib/errors' const app = new Hono() +const TICK_SECRET_HEADER = 'x-preprocess-tick-secret' + +/** + * The `/tick` endpoint is the external-cron driver for the preprocess queue + * (see docs/multi-replica.md). When running multiple replicas you typically + * disable the in-process ticker and have a single external scheduler hit this + * endpoint, so it can be reached publicly. Optionally gate it with a shared + * secret: set `PREPROCESS_TICK_SECRET` and the caller must send it in the + * `x-preprocess-tick-secret` header. When the env var is unset the endpoint + * stays open (backward compatible with single-instance / internal-ticker + * deployments). Constant-time compared; never logged. + */ +function assertTickAuthorized(provided: string | undefined) { + const expected = process.env.PREPROCESS_TICK_SECRET + if (!expected) return + const providedBuf = Buffer.from(provided ?? '') + const expectedBuf = Buffer.from(expected) + if (providedBuf.length !== expectedBuf.length || !timingSafeEqual(providedBuf, expectedBuf)) { + throw unauthorized('Invalid preprocess tick secret') + } +} + function ensureTaskKey(taskKey: unknown) { if (taskKey !== ADMIN_TASK_KEY_PREPROCESS_IMAGES) { throw badRequest('Unsupported task key') @@ -135,6 +158,7 @@ app.post('/runs/:id/cancel', async (c) => { }) app.post('/tick', async (c) => { + assertTickAuthorized(c.req.header(TICK_SECRET_HEADER)) try { const data = await tickPreprocessTaskRuns() return ok(c, data) diff --git a/server/auth/index.ts b/server/auth/index.ts index b106ab69..e0013088 100644 --- a/server/auth/index.ts +++ b/server/auth/index.ts @@ -17,7 +17,12 @@ export const auth = betterAuth({ updateAge: 60 * 60 * 24, cookieCache: { enabled: true, - maxAge: 30 * 60 // Cache duration in seconds + // Short TTL so a sign-out / ban propagates quickly across replicas. The + // signed cookie cache is per-instance in-memory state, so a longer TTL + // would let a revoked session keep validating on a replica that still + // holds the cached cookie until it expires. 60s bounds that lag while + // still absorbing the bulk of repeat reads. Cache duration in seconds. + maxAge: 60 } }, plugins: [