From 787a703846fce1ffd334809aca1368916894123e Mon Sep 17 00:00:00 2001 From: willbot Date: Wed, 15 Jul 2026 17:08:54 +0200 Subject: [PATCH] fix(state): retry state-store queries past a Prisma Postgres cold-start (FT-5226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A freshly provisioned or idle-resumed Prisma Postgres database refuses connections while its upstream warms up (FT-5226) — the edge proxy answers "Failed to connect to upstream database" until the real Postgres is reachable. The state layer already rides this out on the bootstrap migration (layer.ts retries it for 2 minutes), but the per-op state queries that run afterwards did not: a bare Effect.tryPromise in service.ts (every CRUD op) and in lock.ts (checkLive). So when the state DB had gone idle, the first query during a plan/destroy failed the whole deploy — observed in CI as a StateStoreError at plan.make during destroy. Extend the same cold-start budget to those queries: retryColdStart wraps the CRUD attempt and checkLive, retrying every 5s for up to 2 minutes. The retry is deliberately scoped to connection ESTABLISHMENT failures — ECONNREFUSED/ENOTFOUND/EAI_AGAIN and the "upstream database" message. Mid-session drops (a terminated/reset connection, e.g. postgres.js's CONNECTION_ENDED after the pool ends) are NOT retried: for the state store those are the lost-lease signal checkLive must surface loudly, not paper over. This makes the set narrower than @internal/prisma-cloud's pg-connection.ts, which can't be imported here anyway (it depends on @internal/lowering — importing back would cycle). Co-Authored-By: Claude Opus 4.8 Signed-off-by: willbot Signed-off-by: Will Madden --- .../src/state/__tests__/transient.test.ts | 101 ++++++++++++++++++ .../0-lowering/lowering/src/state/lock.ts | 57 +++++----- .../0-lowering/lowering/src/state/service.ts | 3 +- .../lowering/src/state/transient.ts | 63 +++++++++++ 4 files changed, 198 insertions(+), 26 deletions(-) create mode 100644 packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/transient.test.ts create mode 100644 packages/1-prisma-cloud/0-lowering/lowering/src/state/transient.ts diff --git a/packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/transient.test.ts b/packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/transient.test.ts new file mode 100644 index 00000000..f048c03c --- /dev/null +++ b/packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/transient.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, test } from 'bun:test'; +import { StateStoreError } from 'alchemy/State'; +import * as Effect from 'effect/Effect'; +import * as Schedule from 'effect/Schedule'; +import { isColdStartConnectError, retryColdStart } from '../transient.ts'; + +describe('isColdStartConnectError', () => { + test('the PPg cold/idle-upstream rejection classifies as a cold start', () => { + // The exact message Run 1 died on (surfaced via toStateStoreError, which + // preserves the driver message on the StateStoreError). + expect( + isColdStartConnectError( + new StateStoreError({ message: 'Failed to connect to upstream database.' }), + ), + ).toBe(true); + }); + + test('establishment-refusal codes classify as a cold start', () => { + expect(isColdStartConnectError({ code: 'ECONNREFUSED', message: 'connect ECONNREFUSED' })).toBe( + true, + ); + expect(isColdStartConnectError({ code: 'ENOTFOUND', message: 'getaddrinfo ENOTFOUND' })).toBe( + true, + ); + expect(isColdStartConnectError({ code: 'EAI_AGAIN', message: 'getaddrinfo EAI_AGAIN' })).toBe( + true, + ); + }); + + test('unwraps a StateStoreError-style cause carrying the driver code', () => { + expect(isColdStartConnectError({ message: 'wrapped', cause: { code: 'ECONNREFUSED' } })).toBe( + true, + ); + }); + + test('a client-side dropped/terminated connection is NOT a cold start (the lost-lease signal)', () => { + // What postgres.js throws for a query after `sql.end()` — lock.ts's + // "lease-loss" test drives exactly this, and it must stay loud. + expect( + isColdStartConnectError({ + code: 'CONNECTION_ENDED', + message: 'write CONNECTION_ENDED 127.0.0.1:1', + }), + ).toBe(false); + expect(isColdStartConnectError({ code: 'ECONNRESET', message: 'read ECONNRESET' })).toBe(false); + }); + + test('a lost lease and a real query error are NOT cold starts', () => { + expect( + isColdStartConnectError( + new StateStoreError({ + message: 'the state lock for s/t was lost mid-run; refusing to continue unlocked', + }), + ), + ).toBe(false); + expect( + isColdStartConnectError({ message: 'duplicate key value violates unique constraint' }), + ).toBe(false); + }); + + test('non-object inputs are never cold starts', () => { + expect(isColdStartConnectError(undefined)).toBe(false); + expect(isColdStartConnectError(null)).toBe(false); + expect(isColdStartConnectError('upstream database')).toBe(false); + }); +}); + +describe('retryColdStart', () => { + // Instant schedule (no real delay) so tests don't wait the production window. + const instant = Schedule.recurs(10); + + test('retries past a cold-start rejection, then succeeds', async () => { + let attempts = 0; + const op = Effect.suspend(() => { + attempts += 1; + return attempts < 3 + ? Effect.fail(new StateStoreError({ message: 'Failed to connect to upstream database.' })) + : Effect.succeed('ok'); + }); + + const result = await Effect.runPromise(retryColdStart(op, instant)); + + expect(result).toBe('ok'); + expect(attempts).toBe(3); + }); + + test('does not retry a lost-lease failure — it surfaces on the first attempt', async () => { + let attempts = 0; + const op = Effect.suspend(() => { + attempts += 1; + return Effect.fail( + new StateStoreError({ + message: 'the state lock for s/t was lost mid-run; refusing to continue unlocked', + }), + ); + }); + + await expect(Effect.runPromise(retryColdStart(op, instant))).rejects.toThrow(/lost mid-run/); + expect(attempts).toBe(1); + }); +}); diff --git a/packages/1-prisma-cloud/0-lowering/lowering/src/state/lock.ts b/packages/1-prisma-cloud/0-lowering/lowering/src/state/lock.ts index 7378044c..601d6c9c 100644 --- a/packages/1-prisma-cloud/0-lowering/lowering/src/state/lock.ts +++ b/packages/1-prisma-cloud/0-lowering/lowering/src/state/lock.ts @@ -3,6 +3,7 @@ import * as Data from 'effect/Data'; import * as Effect from 'effect/Effect'; import type postgres from 'postgres'; import { toStateStoreError } from './errors.ts'; +import { retryColdStart } from './transient.ts'; /** Another deploy already holds the lock for this stack/stage. Never queued — fails immediately. */ export class StateLockContentionError extends Data.TaggedError('StateLockContentionError')<{ @@ -82,31 +83,37 @@ export const acquireStateLock = ( // captured at acquire time still holds this advisory lock in // `pg_locks` gets the same answer (a dead or reused backend can't hold // the lock) without ever touching the connection that might be dead. - const checkLive: Effect.Effect = Effect.tryPromise({ - try: async () => { - const rows = await sql<{ live: boolean }[]>` - select exists ( - select 1 from pg_locks - where locktype = 'advisory' - and pid = ${lockPid} - and objsubid = 1 - and granted - and ((classid::bigint << 32) | (objid::bigint & 4294967295)) - = hashtextextended(${key}, 0) - ) as live - `; - return rows[0]?.live ?? false; - }, - catch: toStateStoreError, - }).pipe( - Effect.flatMap((live) => - live - ? Effect.void - : Effect.fail( - new StateStoreError({ - message: `the state lock for ${stack}/${stage} was lost mid-run; refusing to continue unlocked`, - }), - ), + // Retry only a cold-start on the check's own connection; a lost lease + // (`live` = false) and a dropped/terminated connection both stay loud — + // the former is not a connection error, the latter is a mid-session drop + // the cold-start predicate deliberately excludes (see `transient.ts`). + const checkLive: Effect.Effect = retryColdStart( + Effect.tryPromise({ + try: async () => { + const rows = await sql<{ live: boolean }[]>` + select exists ( + select 1 from pg_locks + where locktype = 'advisory' + and pid = ${lockPid} + and objsubid = 1 + and granted + and ((classid::bigint << 32) | (objid::bigint & 4294967295)) + = hashtextextended(${key}, 0) + ) as live + `; + return rows[0]?.live ?? false; + }, + catch: toStateStoreError, + }).pipe( + Effect.flatMap((live) => + live + ? Effect.void + : Effect.fail( + new StateStoreError({ + message: `the state lock for ${stack}/${stage} was lost mid-run; refusing to continue unlocked`, + }), + ), + ), ), ); diff --git a/packages/1-prisma-cloud/0-lowering/lowering/src/state/service.ts b/packages/1-prisma-cloud/0-lowering/lowering/src/state/service.ts index e70e5ad5..bc271ff7 100644 --- a/packages/1-prisma-cloud/0-lowering/lowering/src/state/service.ts +++ b/packages/1-prisma-cloud/0-lowering/lowering/src/state/service.ts @@ -11,9 +11,10 @@ import { import * as Effect from 'effect/Effect'; import type postgres from 'postgres'; import { toStateStoreError } from './errors.ts'; +import { retryColdStart } from './transient.ts'; const attempt = (f: () => Promise): Effect.Effect => - Effect.tryPromise({ try: f, catch: toStateStoreError }); + retryColdStart(Effect.tryPromise({ try: f, catch: toStateStoreError })); /** * Wraps an already-`encodeState`d value as a jsonb-typed bind parameter. diff --git a/packages/1-prisma-cloud/0-lowering/lowering/src/state/transient.ts b/packages/1-prisma-cloud/0-lowering/lowering/src/state/transient.ts new file mode 100644 index 00000000..b6ed2a46 --- /dev/null +++ b/packages/1-prisma-cloud/0-lowering/lowering/src/state/transient.ts @@ -0,0 +1,63 @@ +/** + * Retrying state-store queries past a Prisma Postgres cold-start (FT-5226). + * + * A freshly provisioned or idle-resumed PPg database refuses connections while + * its upstream warms up: the edge proxy answers "Failed to connect to upstream + * database" until the real Postgres is reachable. The bootstrap migration + * already rides this out (see `layer.ts`), but the per-op state queries that + * run afterwards (plan/apply/destroy) did not — so a state DB that had gone + * idle between bootstrap and those queries failed the deploy outright. + * + * The retry is deliberately scoped to connection *establishment* failures — + * a warming upstream. Mid-session drops (a terminated/reset connection) are + * NOT retried: for the state store those are the lost-lease signal that + * `lock.ts`'s `checkLive` must surface loudly, not paper over. This makes the + * set narrower than @internal/prisma-cloud's `pg-connection.ts` (which also + * retries mid-session drops for the runtime store client); that helper can't + * be imported here regardless — @internal/prisma-cloud depends on + * @internal/lowering, so importing it back would cycle. + */ +import type { StateStoreError } from 'alchemy/State'; +import * as Effect from 'effect/Effect'; +import * as Schedule from 'effect/Schedule'; + +/** Codes for "can't reach the host yet" — DNS/refusal, not a mid-session drop. */ +const ESTABLISHMENT_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN']); + +/** Connection-establishment failure messages (no useful `err.code`). `upstream database` is PPg's edge proxy while a cold/idle upstream warms up. */ +const ESTABLISHMENT_MESSAGE_FRAGMENTS = ['upstream database', 'connection refused']; + +/** + * Whether a failure is a connection-establishment error against a warming + * upstream (retry), as opposed to a lost lease, a mid-session drop, or a real + * query error (all of which must surface at once). Unwraps a + * {@link StateStoreError} to its driver `cause` so a code-only error still + * classifies. + */ +export const isColdStartConnectError = (error: unknown): boolean => { + if (typeof error !== 'object' || error === null) return false; + const code = 'code' in error && typeof error.code === 'string' ? error.code : undefined; + if (code !== undefined && ESTABLISHMENT_CODES.has(code)) return true; + const message = + 'message' in error && typeof error.message === 'string' ? error.message.toLowerCase() : ''; + if (ESTABLISHMENT_MESSAGE_FRAGMENTS.some((fragment) => message.includes(fragment))) return true; + const cause = 'cause' in error ? error.cause : undefined; + return cause !== undefined && cause !== error && isColdStartConnectError(cause); +}; + +/** The same ~2-minute budget the bootstrap migration uses (`layer.ts`): retry every 5s, up to 2 minutes. */ +const COLD_START_SCHEDULE = Schedule.both( + Schedule.spaced('5 seconds'), + Schedule.during('2 minutes'), +); + +/** + * Retries a state operation past a cold-start connection rejection only; every + * other failure surfaces immediately. `schedule` is injectable so tests drive + * it without real delay. + */ +export const retryColdStart = ( + operation: Effect.Effect, + schedule: Schedule.Schedule = COLD_START_SCHEDULE, +): Effect.Effect => + Effect.retry(operation, { while: isColdStartConnectError, schedule });