fix(state): retry state-store queries past a Prisma Postgres cold-start (FT-5226)

packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/transient.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, test } from 'bun:test';
+import { StateStoreError } from 'alchemy/State';
+import * as Effect from 'effect/Effect';
+import * as Schedule from 'effect/Schedule';
+import { isColdStartConnectError, retryColdStart } from '../transient.ts';
+
+describe('isColdStartConnectError', () => {
+  test('the PPg cold/idle-upstream rejection classifies as a cold start', () => {
+    // The exact message Run 1 died on (surfaced via toStateStoreError, which
+    // preserves the driver message on the StateStoreError).
+    expect(
+      isColdStartConnectError(
+        new StateStoreError({ message: 'Failed to connect to upstream database.' }),
+      ),
+    ).toBe(true);
+  });
+
+  test('establishment-refusal codes classify as a cold start', () => {
+    expect(isColdStartConnectError({ code: 'ECONNREFUSED', message: 'connect ECONNREFUSED' })).toBe(
+      true,
+    );
+    expect(isColdStartConnectError({ code: 'ENOTFOUND', message: 'getaddrinfo ENOTFOUND' })).toBe(
+      true,
+    );
+    expect(isColdStartConnectError({ code: 'EAI_AGAIN', message: 'getaddrinfo EAI_AGAIN' })).toBe(
+      true,
+    );
+  });
+
+  test('unwraps a StateStoreError-style cause carrying the driver code', () => {
+    expect(isColdStartConnectError({ message: 'wrapped', cause: { code: 'ECONNREFUSED' } })).toBe(
+      true,
+    );
+  });
+
+  test('a client-side dropped/terminated connection is NOT a cold start (the lost-lease signal)', () => {
+    // What postgres.js throws for a query after `sql.end()` — lock.ts's
+    // "lease-loss" test drives exactly this, and it must stay loud.
+    expect(
+      isColdStartConnectError({
+        code: 'CONNECTION_ENDED',
+        message: 'write CONNECTION_ENDED 127.0.0.1:1',
+      }),
+    ).toBe(false);
+    expect(isColdStartConnectError({ code: 'ECONNRESET', message: 'read ECONNRESET' })).toBe(false);
+  });
+
+  test('a lost lease and a real query error are NOT cold starts', () => {
+    expect(
+      isColdStartConnectError(
+        new StateStoreError({
+          message: 'the state lock for s/t was lost mid-run; refusing to continue unlocked',
+        }),
+      ),
+    ).toBe(false);
+    expect(
+      isColdStartConnectError({ message: 'duplicate key value violates unique constraint' }),
+    ).toBe(false);
+  });
+
+  test('non-object inputs are never cold starts', () => {
+    expect(isColdStartConnectError(undefined)).toBe(false);
+    expect(isColdStartConnectError(null)).toBe(false);
+    expect(isColdStartConnectError('upstream database')).toBe(false);
+  });
+});
+
+describe('retryColdStart', () => {
+  // Instant schedule (no real delay) so tests don't wait the production window.
+  const instant = Schedule.recurs(10);
+
+  test('retries past a cold-start rejection, then succeeds', async () => {
+    let attempts = 0;
+    const op = Effect.suspend(() => {
+      attempts += 1;
+      return attempts < 3
+        ? Effect.fail(new StateStoreError({ message: 'Failed to connect to upstream database.' }))
+        : Effect.succeed('ok');
+    });
+
+    const result = await Effect.runPromise(retryColdStart(op, instant));
+
+    expect(result).toBe('ok');
+    expect(attempts).toBe(3);
+  });
+
+  test('does not retry a lost-lease failure — it surfaces on the first attempt', async () => {
+    let attempts = 0;
+    const op = Effect.suspend(() => {
+      attempts += 1;
+      return Effect.fail(
+        new StateStoreError({
+          message: 'the state lock for s/t was lost mid-run; refusing to continue unlocked',
+        }),
+      );
+    });
+
+    await expect(Effect.runPromise(retryColdStart(op, instant))).rejects.toThrow(/lost mid-run/);
+    expect(attempts).toBe(1);
+  });
+});

packages/1-prisma-cloud/0-lowering/lowering/src/state/lock.ts

@@ -3,6 +3,7 @@ import * as Data from 'effect/Data';
 import * as Effect from 'effect/Effect';
 import type postgres from 'postgres';
 import { toStateStoreError } from './errors.ts';
+import { retryColdStart } from './transient.ts';
 
 /** Another deploy already holds the lock for this stack/stage. Never queued — fails immediately. */
 export class StateLockContentionError extends Data.TaggedError('StateLockContentionError')<{
@@ -82,31 +83,37 @@ export const acquireStateLock = (
     // captured at acquire time still holds this advisory lock in
     // `pg_locks` gets the same answer (a dead or reused backend can't hold
     // the lock) without ever touching the connection that might be dead.
-    const checkLive: Effect.Effect<void, StateStoreError, never> = Effect.tryPromise({
-      try: async () => {
-        const rows = await sql<{ live: boolean }[]>`
-          select exists (
-            select 1 from pg_locks
-            where locktype = 'advisory'
-              and pid = ${lockPid}
-              and objsubid = 1
-              and granted
-              and ((classid::bigint << 32) | (objid::bigint & 4294967295))
-                = hashtextextended(${key}, 0)
-          ) as live
-        `;
-        return rows[0]?.live ?? false;
-      },
-      catch: toStateStoreError,
-    }).pipe(
-      Effect.flatMap((live) =>
-        live
-          ? Effect.void
-          : Effect.fail(
-              new StateStoreError({
-                message: `the state lock for ${stack}/${stage} was lost mid-run; refusing to continue unlocked`,
-              }),
-            ),
+    // Retry only a cold-start on the check's own connection; a lost lease
+    // (`live` = false) and a dropped/terminated connection both stay loud —
+    // the former is not a connection error, the latter is a mid-session drop
+    // the cold-start predicate deliberately excludes (see `transient.ts`).
+    const checkLive: Effect.Effect<void, StateStoreError, never> = retryColdStart(
+      Effect.tryPromise({
+        try: async () => {
+          const rows = await sql<{ live: boolean }[]>`
+            select exists (
+              select 1 from pg_locks
+              where locktype = 'advisory'
+                and pid = ${lockPid}
+                and objsubid = 1
+                and granted
+                and ((classid::bigint << 32) | (objid::bigint & 4294967295))
+                  = hashtextextended(${key}, 0)
+            ) as live
+          `;
+          return rows[0]?.live ?? false;
+        },
+        catch: toStateStoreError,
+      }).pipe(
+        Effect.flatMap((live) =>
+          live
+            ? Effect.void
+            : Effect.fail(
+                new StateStoreError({
+                  message: `the state lock for ${stack}/${stage} was lost mid-run; refusing to continue unlocked`,
+                }),
+              ),
+        ),
       ),
     );
 

packages/1-prisma-cloud/0-lowering/lowering/src/state/service.ts

@@ -11,9 +11,10 @@ import {
 import * as Effect from 'effect/Effect';
 import type postgres from 'postgres';
 import { toStateStoreError } from './errors.ts';
+import { retryColdStart } from './transient.ts';
 
 const attempt = <A>(f: () => Promise<A>): Effect.Effect<A, StateStoreError, never> =>
-  Effect.tryPromise({ try: f, catch: toStateStoreError });
+  retryColdStart(Effect.tryPromise({ try: f, catch: toStateStoreError }));
 
 /**
  * Wraps an already-`encodeState`d value as a jsonb-typed bind parameter.

packages/1-prisma-cloud/0-lowering/lowering/src/state/transient.ts

@@ -0,0 +1,63 @@
+/**
+ * Retrying state-store queries past a Prisma Postgres cold-start (FT-5226).
+ *
+ * A freshly provisioned or idle-resumed PPg database refuses connections while
+ * its upstream warms up: the edge proxy answers "Failed to connect to upstream
+ * database" until the real Postgres is reachable. The bootstrap migration
+ * already rides this out (see `layer.ts`), but the per-op state queries that
+ * run afterwards (plan/apply/destroy) did not — so a state DB that had gone
+ * idle between bootstrap and those queries failed the deploy outright.
+ *
+ * The retry is deliberately scoped to connection *establishment* failures —
+ * a warming upstream. Mid-session drops (a terminated/reset connection) are
+ * NOT retried: for the state store those are the lost-lease signal that
+ * `lock.ts`'s `checkLive` must surface loudly, not paper over. This makes the
+ * set narrower than @internal/prisma-cloud's `pg-connection.ts` (which also
+ * retries mid-session drops for the runtime store client); that helper can't
+ * be imported here regardless — @internal/prisma-cloud depends on
+ * @internal/lowering, so importing it back would cycle.
+ */
+import type { StateStoreError } from 'alchemy/State';
+import * as Effect from 'effect/Effect';
+import * as Schedule from 'effect/Schedule';
+
+/** Codes for "can't reach the host yet" — DNS/refusal, not a mid-session drop. */
+const ESTABLISHMENT_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN']);
+
+/** Connection-establishment failure messages (no useful `err.code`). `upstream database` is PPg's edge proxy while a cold/idle upstream warms up. */
+const ESTABLISHMENT_MESSAGE_FRAGMENTS = ['upstream database', 'connection refused'];
+
+/**
+ * Whether a failure is a connection-establishment error against a warming
+ * upstream (retry), as opposed to a lost lease, a mid-session drop, or a real
+ * query error (all of which must surface at once). Unwraps a
+ * {@link StateStoreError} to its driver `cause` so a code-only error still
+ * classifies.
+ */
+export const isColdStartConnectError = (error: unknown): boolean => {
+  if (typeof error !== 'object' || error === null) return false;
+  const code = 'code' in error && typeof error.code === 'string' ? error.code : undefined;
+  if (code !== undefined && ESTABLISHMENT_CODES.has(code)) return true;
+  const message =
+    'message' in error && typeof error.message === 'string' ? error.message.toLowerCase() : '';
+  if (ESTABLISHMENT_MESSAGE_FRAGMENTS.some((fragment) => message.includes(fragment))) return true;
+  const cause = 'cause' in error ? error.cause : undefined;
+  return cause !== undefined && cause !== error && isColdStartConnectError(cause);
+};
+
+/** The same ~2-minute budget the bootstrap migration uses (`layer.ts`): retry every 5s, up to 2 minutes. */
+const COLD_START_SCHEDULE = Schedule.both(
+  Schedule.spaced('5 seconds'),
+  Schedule.during('2 minutes'),
+);
+
+/**
+ * Retries a state operation past a cold-start connection rejection only; every
+ * other failure surfaces immediately. `schedule` is injectable so tests drive
+ * it without real delay.
+ */
+export const retryColdStart = <A>(
+  operation: Effect.Effect<A, StateStoreError, never>,
+  schedule: Schedule.Schedule<unknown, unknown> = COLD_START_SCHEDULE,
+): Effect.Effect<A, StateStoreError, never> =>
+  Effect.retry(operation, { while: isColdStartConnectError, schedule });