Feat/graceful shutdown handler

README.md

@@ -523,6 +523,17 @@ Extend with additional Horizon event ingestion when implementing the full archit
 - Integration notes: `docs/stellar-integration.md`
 - Tests: `src/clients/soroban.test.ts`
 
+## Graceful Shutdown
+
+On `SIGTERM` or `SIGINT`, the Credence Backend API executes an ordered graceful shutdown sequence:
+1. Stops accepting new HTTP connections and allows in-flight requests to drain (`server.close()`).
+2. Closes WebSocket subscription server connections gracefully.
+3. Stops event consumers and background schedulers.
+4. Closes database connection pools (primary, worker, replica) cleanly (`pool.end()`).
+5. Disconnects from Redis connection.
+
+The grace period is configurable via `SHUTDOWN_GRACE_PERIOD_MS` (default: 30,000 ms). For more details, see **[docs/graceful-shutdown.md](docs/graceful-shutdown.md)**.
+
 ## Security
 
 For security policies, reporting, and architecture documentation:

docs/graceful-shutdown.md

@@ -123,6 +123,10 @@ The test suite covers:
 - Double-signal triggers immediate force-exit
 - SIGTERM arriving before HTTP server is ready
 
+## Signal Handling in Production
+
+In the production entrypoint (`src/index.ts`), the process listens for `SIGTERM` and `SIGINT` signals, routing them directly to the `GracefulShutdownManager.shutdown()` method. This ensures that the application executes the full, ordered cleanup process (draining in-flight requests, stopping schedulers, closing database pools, disconnecting Redis, etc.) before exiting.
+
 ## Operational runbook
 
 ### Rolling deploy (Kubernetes)

pr-description.md

@@ -1,29 +1,30 @@
-# feat(trust): ETag + conditional GET for trust score reads
+# feat(shutdown): drain in-flight requests, close DB pools, and exit cleanly on SIGTERM
 
 ## Description
 
-This PR adds ETag support and conditional GET handling (`If-None-Match`) to the `GET /api/trust/:address` endpoint. This allows clients to avoid transferring the full trust score body when it hasn't changed, reducing bandwidth and database load.
+This PR fixes a bug where `SIGTERM` and `SIGINT` signals bypassed the configured `GracefulShutdownManager` in production (due to duplicate signal handlers in `src/index.ts` that immediately called `process.exit(0)` when the HTTP server closed, bypassing the closing of DB pools, Redis connections, and background schedulers).
+
+With this fix, signals are correctly routed to the `GracefulShutdownManager`, which performs a clean, ordered graceful shutdown sequence before exiting.
+
+Closes #<this-issue>
 
 ## Changes
 
--   **`src/routes/trust.ts`**:
-    -   Added a `generateEtag` helper using SHA-256 to create deterministic ETags from the `TrustScore` object.
-    -   Updated the `GET /:address` route handler to compute the ETag and set `ETag` and `Cache-Control` headers.
-    -   Added logic to check for `If-None-Match` and return `304 Not Modified` if the ETag matches.
--   **`src/__tests__/trust.test.ts`**:
-    -   Added unit tests to verify:
-        -   First request returns `200 OK` + `ETag`.
-        -   Subsequent request with matching `If-None-Match` returns `304 Not Modified`.
-        -   Request with changed data returns `200 OK` + new `ETag`.
+- **`src/index.ts`**: Removed duplicate local `shutdown` handler and its signal registrations to ensure `GracefulShutdownManager` manages the shutdown process.
+- **`src/__tests__/gracefulShutdown.test.ts`**: Added a new sequence verification test to ensure that the server closing/draining, database pools closing, and process exiting occur in the correct order.
+- **`README.md`**: Documented the graceful shutdown behavior.
+- **`docs/graceful-shutdown.md`**: Updated signal handling architecture notes.
 
-## Testing
+## Commits
 
--   Ran `vitest src/__tests__/trust.test.ts` and all tests passed.
--   Verified ETag logic by mocking the reputation service.
+1. `fix(shutdown): remove redundant signal handlers in index.ts`
+2. `test(shutdown): add test to verify shutdown sequence order`
+3. `docs(shutdown): document production graceful shutdown and signal handling`
 
 ## Checklist
 
--   [x] ETag implemented and deterministic.
--   [x] `If-None-Match` honored, returns 304.
--   [x] `Cache-Control` headers set.
--   [x] Tests updated/added with ≥95% coverage for the new logic.
+- [x] The change matches the summary above.
+- [x] No regression in the existing test suite.
+- [x] The change is documented where it is observable (README, docs/).
+- [x] Lint, type-check, and tests all pass locally.
+- [x] PR description references this issue with `Closes #<this-issue>`.

src/__tests__/gracefulShutdown.test.ts

@@ -80,6 +80,59 @@ describe('GracefulShutdownManager', () => {
     expect(forceExit).toHaveBeenCalledWith(1)
     vi.useRealTimers()
   })
+
+  it('drains in-flight requests on SIGTERM, then closes the DB pool and Redis, then exits', async () => {
+    const sequence: string[] = []
+    const close = vi.fn((callback: (err?: Error | null) => void) => {
+      sequence.push('server_close_start')
+      callback()
+      sequence.push('server_close_end')
+    })
+    const fakeServer = { close } as unknown as import('http').Server
+
+    const dbPoolEnd = vi.fn(async () => {
+      sequence.push('db_pool_close')
+    })
+    const dbPool = { end: dbPoolEnd }
+
+    const redisDisconnect = vi.fn(async () => {
+      sequence.push('redis_close')
+    })
+    const redis = { disconnect: redisDisconnect }
+
+    const forceExit = vi.fn((code: number) => {
+      sequence.push(`exit_${code}`)
+    })
+
+    const manager = new GracefulShutdownManager({
+      server: fakeServer,
+      dbPools: [dbPool],
+      redis,
+      gracePeriodMs: 1000,
+      forceExit,
+      logger: vi.fn(),
+    })
+
+    await manager.shutdown('SIGTERM')
+
+    expect(close).toHaveBeenCalledOnce()
+    expect(dbPoolEnd).toHaveBeenCalledOnce()
+    expect(redisDisconnect).toHaveBeenCalledOnce()
+    expect(forceExit).toHaveBeenCalledWith(0)
+
+    // Assert the exact sequence: server closes first, then DB/Redis close, then exit
+    expect(sequence[0]).toBe('server_close_start')
+    expect(sequence[1]).toBe('server_close_end')
+
+    const dbIndex = sequence.indexOf('db_pool_close')
+    const redisIndex = sequence.indexOf('redis_close')
+    const exitIndex = sequence.indexOf('exit_0')
+
+    expect(dbIndex).toBeGreaterThan(1)
+    expect(redisIndex).toBeGreaterThan(1)
+    expect(exitIndex).toBeGreaterThan(dbIndex)
+    expect(exitIndex).toBeGreaterThan(redisIndex)
+  })
 })
 
 describe('Health router readiness during shutdown', () => {

src/index.ts

@@ -116,25 +116,6 @@ if (process.env.NODE_ENV !== "test") {
 
     installShutdownHandlers();
 
-    // Graceful shutdown
-    const shutdown = async (signal: string): Promise<void> => {
-      logger.info(`Received ${signal}, shutting down gracefully...`)
-
-      // Close HTTP server
-      server.close(() => {
-        logger.info('HTTP server closed')
-        process.exit(0)
-      })
-
-      // Force shutdown after 10 seconds
-      setTimeout(() => {
-        logger.error('Forced shutdown after timeout')
-        process.exit(1)
-      }, 10000)
-    }
-
-    process.on('SIGTERM', () => void shutdown('SIGTERM'))
-    process.on('SIGINT', () => void shutdown('SIGINT'))
 
     if (process.env.DATABASE_URL) {
       const thresholdSeconds = Number(