From 63a6634054899ada587305175705fad01538d660 Mon Sep 17 00:00:00 2001 From: Antigravity AI Date: Tue, 30 Jun 2026 09:51:58 +0100 Subject: [PATCH 1/4] fix(shutdown): remove redundant signal handlers in index.ts --- src/index.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/index.ts b/src/index.ts index 1fe6e65c..353e92f7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -116,25 +116,6 @@ if (process.env.NODE_ENV !== "test") { installShutdownHandlers(); - // Graceful shutdown - const shutdown = async (signal: string): Promise => { - logger.info(`Received ${signal}, shutting down gracefully...`) - - // Close HTTP server - server.close(() => { - logger.info('HTTP server closed') - process.exit(0) - }) - - // Force shutdown after 10 seconds - setTimeout(() => { - logger.error('Forced shutdown after timeout') - process.exit(1) - }, 10000) - } - - process.on('SIGTERM', () => void shutdown('SIGTERM')) - process.on('SIGINT', () => void shutdown('SIGINT')) if (process.env.DATABASE_URL) { const thresholdSeconds = Number( From 42c231ca5ea9b52a1c1f9b7ee574dc18b5d8fbf3 Mon Sep 17 00:00:00 2001 From: Antigravity AI Date: Tue, 30 Jun 2026 09:52:16 +0100 Subject: [PATCH 2/4] test(shutdown): add test to verify shutdown sequence order --- src/__tests__/gracefulShutdown.test.ts | 53 ++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/__tests__/gracefulShutdown.test.ts b/src/__tests__/gracefulShutdown.test.ts index aa446154..e1879fad 100644 --- a/src/__tests__/gracefulShutdown.test.ts +++ b/src/__tests__/gracefulShutdown.test.ts @@ -80,6 +80,59 @@ describe('GracefulShutdownManager', () => { expect(forceExit).toHaveBeenCalledWith(1) vi.useRealTimers() }) + + it('drains in-flight requests on SIGTERM, then closes the DB pool and Redis, then exits', async () => { + const sequence: string[] = [] + const close = vi.fn((callback: (err?: Error | null) => void) => { + sequence.push('server_close_start') + callback() + sequence.push('server_close_end') + }) + const fakeServer = { close } as unknown as import('http').Server + + const dbPoolEnd = vi.fn(async () => { + sequence.push('db_pool_close') + }) + const dbPool = { end: dbPoolEnd } + + const redisDisconnect = vi.fn(async () => { + sequence.push('redis_close') + }) + const redis = { disconnect: redisDisconnect } + + const forceExit = vi.fn((code: number) => { + sequence.push(`exit_${code}`) + }) + + const manager = new GracefulShutdownManager({ + server: fakeServer, + dbPools: [dbPool], + redis, + gracePeriodMs: 1000, + forceExit, + logger: vi.fn(), + }) + + await manager.shutdown('SIGTERM') + + expect(close).toHaveBeenCalledOnce() + expect(dbPoolEnd).toHaveBeenCalledOnce() + expect(redisDisconnect).toHaveBeenCalledOnce() + expect(forceExit).toHaveBeenCalledWith(0) + + // Assert the exact sequence: server closes first, then DB/Redis close, then exit + expect(sequence[0]).toBe('server_close_start') + expect(sequence[1]).toBe('server_close_end') + + const dbIndex = sequence.indexOf('db_pool_close') + const redisIndex = sequence.indexOf('redis_close') + const exitIndex = sequence.indexOf('exit_0') + + expect(dbIndex).toBeGreaterThan(1) + expect(redisIndex).toBeGreaterThan(1) + expect(exitIndex).toBeGreaterThan(dbIndex) + expect(exitIndex).toBeGreaterThan(redisIndex) + }) }) describe('Health router readiness during shutdown', () => { From 01d9033f10d97b02933d7cd987a3cfe780c0d8b2 Mon Sep 17 00:00:00 2001 From: Antigravity AI Date: Tue, 30 Jun 2026 09:52:54 +0100 Subject: [PATCH 3/4] docs(shutdown): document production graceful shutdown and signal handling --- README.md | 11 +++++++++++ docs/graceful-shutdown.md | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 40fa45a0..17c8583d 100644 --- a/README.md +++ b/README.md @@ -523,6 +523,17 @@ Extend with additional Horizon event ingestion when implementing the full archit - Integration notes: `docs/stellar-integration.md` - Tests: `src/clients/soroban.test.ts` +## Graceful Shutdown + +On `SIGTERM` or `SIGINT`, the Credence Backend API executes an ordered graceful shutdown sequence: +1. Stops accepting new HTTP connections and allows in-flight requests to drain (`server.close()`). +2. Closes WebSocket subscription server connections gracefully. +3. Stops event consumers and background schedulers. +4. Closes database connection pools (primary, worker, replica) cleanly (`pool.end()`). +5. Disconnects from Redis connection. + +The grace period is configurable via `SHUTDOWN_GRACE_PERIOD_MS` (default: 30,000 ms). For more details, see **[docs/graceful-shutdown.md](docs/graceful-shutdown.md)**. + ## Security For security policies, reporting, and architecture documentation: diff --git a/docs/graceful-shutdown.md b/docs/graceful-shutdown.md index 36cc227f..0013bbf0 100644 --- a/docs/graceful-shutdown.md +++ b/docs/graceful-shutdown.md @@ -123,6 +123,10 @@ The test suite covers: - Double-signal triggers immediate force-exit - SIGTERM arriving before HTTP server is ready +## Signal Handling in Production + +In the production entrypoint (`src/index.ts`), the process listens for `SIGTERM` and `SIGINT` signals, routing them directly to the `GracefulShutdownManager.shutdown()` method. This ensures that the application executes the full, ordered cleanup process (draining in-flight requests, stopping schedulers, closing database pools, disconnecting Redis, etc.) before exiting. + ## Operational runbook ### Rolling deploy (Kubernetes) From 3271e0c6c4b1171a661abe74ff8e0bcfbe7b5558 Mon Sep 17 00:00:00 2001 From: Antigravity AI Date: Tue, 30 Jun 2026 09:53:16 +0100 Subject: [PATCH 4/4] docs: update pr-description.md with graceful shutdown fix details --- pr-description.md | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/pr-description.md b/pr-description.md index b303fafc..a20312bf 100644 --- a/pr-description.md +++ b/pr-description.md @@ -1,29 +1,30 @@ -# feat(trust): ETag + conditional GET for trust score reads +# feat(shutdown): drain in-flight requests, close DB pools, and exit cleanly on SIGTERM ## Description -This PR adds ETag support and conditional GET handling (`If-None-Match`) to the `GET /api/trust/:address` endpoint. This allows clients to avoid transferring the full trust score body when it hasn't changed, reducing bandwidth and database load. +This PR fixes a bug where `SIGTERM` and `SIGINT` signals bypassed the configured `GracefulShutdownManager` in production (due to duplicate signal handlers in `src/index.ts` that immediately called `process.exit(0)` when the HTTP server closed, bypassing the closing of DB pools, Redis connections, and background schedulers). + +With this fix, signals are correctly routed to the `GracefulShutdownManager`, which performs a clean, ordered graceful shutdown sequence before exiting. + +Closes # ## Changes -- **`src/routes/trust.ts`**: - - Added a `generateEtag` helper using SHA-256 to create deterministic ETags from the `TrustScore` object. - - Updated the `GET /:address` route handler to compute the ETag and set `ETag` and `Cache-Control` headers. - - Added logic to check for `If-None-Match` and return `304 Not Modified` if the ETag matches. -- **`src/__tests__/trust.test.ts`**: - - Added unit tests to verify: - - First request returns `200 OK` + `ETag`. - - Subsequent request with matching `If-None-Match` returns `304 Not Modified`. - - Request with changed data returns `200 OK` + new `ETag`. +- **`src/index.ts`**: Removed duplicate local `shutdown` handler and its signal registrations to ensure `GracefulShutdownManager` manages the shutdown process. +- **`src/__tests__/gracefulShutdown.test.ts`**: Added a new sequence verification test to ensure that the server closing/draining, database pools closing, and process exiting occur in the correct order. +- **`README.md`**: Documented the graceful shutdown behavior. +- **`docs/graceful-shutdown.md`**: Updated signal handling architecture notes. -## Testing +## Commits -- Ran `vitest src/__tests__/trust.test.ts` and all tests passed. -- Verified ETag logic by mocking the reputation service. +1. `fix(shutdown): remove redundant signal handlers in index.ts` +2. `test(shutdown): add test to verify shutdown sequence order` +3. `docs(shutdown): document production graceful shutdown and signal handling` ## Checklist -- [x] ETag implemented and deterministic. -- [x] `If-None-Match` honored, returns 304. -- [x] `Cache-Control` headers set. -- [x] Tests updated/added with ≥95% coverage for the new logic. +- [x] The change matches the summary above. +- [x] No regression in the existing test suite. +- [x] The change is documented where it is observable (README, docs/). +- [x] Lint, type-check, and tests all pass locally. +- [x] PR description references this issue with `Closes #`.