feat(health): dependency latency budget healthcheck for RPC and Horizon (#848)

apps/backend/.env.example

@@ -82,6 +82,15 @@ STELLAR_TIMEOUT=30000
 STELLAR_RETRY_ATTEMPTS=3
 STELLAR_RETRY_DELAY=1000
 
+# Dependency latency budget health thresholds
+# Latency (ms) above which a dependency is flagged as "degraded" (HTTP 200).
+# Latency (ms) above which a dependency is flagged as "hard-down" (HTTP 503).
+# Leave unset to use the built-in defaults shown here.
+HEALTH_HORIZON_LATENCY_DEGRADED_MS=1000
+HEALTH_HORIZON_LATENCY_HARD_DOWN_MS=4000
+HEALTH_SOROBAN_RPC_LATENCY_DEGRADED_MS=1500
+HEALTH_SOROBAN_RPC_LATENCY_HARD_DOWN_MS=5000
+
 # Soroban contract addresses (set after deployment; leave blank if not yet deployed)
 # Canonical LumenToken contract ID used by backend and exposed to web/mobile apps via the Stellar config API.
 STELLAR_CONTRACT_LUMEN_TOKEN=

apps/backend/src/common/event-type-mapper.ts

@@ -1,6 +1,4 @@
-import {
-  CanonicalEventType,
-} from './event-catalog';
+import { CanonicalEventType } from './event-catalog';
 import { NotificationType } from '../notification/notification.entity';
 import { WebhookEventType } from '../webhook/webhook.types';
 
@@ -23,11 +21,15 @@ const CANONICAL_TO_NOTIFICATION: Record<string, NotificationType> = {
   [CanonicalEventType.MILESTONE_VOTE_STARTED]: NotificationType.MILESTONE,
   [CanonicalEventType.MILESTONE_VOTE_CAST]: NotificationType.MILESTONE,
   [CanonicalEventType.GOVERNANCE_PROPOSAL_CREATED]: NotificationType.GOVERNANCE,
-  [CanonicalEventType.GOVERNANCE_PROPOSAL_EXECUTED]: NotificationType.GOVERNANCE,
-  [CanonicalEventType.GOVERNANCE_PROPOSAL_CANCELLED]: NotificationType.GOVERNANCE,
+  [CanonicalEventType.GOVERNANCE_PROPOSAL_EXECUTED]:
+    NotificationType.GOVERNANCE,
+  [CanonicalEventType.GOVERNANCE_PROPOSAL_CANCELLED]:
+    NotificationType.GOVERNANCE,
   [CanonicalEventType.GOVERNANCE_PROPOSAL_EXPIRED]: NotificationType.GOVERNANCE,
-  [CanonicalEventType.GOVERNANCE_SIGNATURE_COLLECTED]: NotificationType.GOVERNANCE,
-  [CanonicalEventType.GOVERNANCE_MULTISIG_CONFIGURED]: NotificationType.GOVERNANCE,
+  [CanonicalEventType.GOVERNANCE_SIGNATURE_COLLECTED]:
+    NotificationType.GOVERNANCE,
+  [CanonicalEventType.GOVERNANCE_MULTISIG_CONFIGURED]:
+    NotificationType.GOVERNANCE,
   [CanonicalEventType.TOKEN_BURNED]: NotificationType.TOKEN,
   [CanonicalEventType.TOKEN_VESTING_CREATED]: NotificationType.TOKEN,
   [CanonicalEventType.TOKEN_CLAIMED]: NotificationType.TOKEN,

apps/backend/src/config/config.service.ts

@@ -44,29 +44,25 @@ export class ConfigService {
       networkPassphrase: NETWORK_PASSPHRASES[network],
       contracts: {
         lumenToken:
-          (overrides.lumenToken as string | undefined) ??
-          config.stellar.contracts.lumenToken ??
-          null,
+          overrides.lumenToken ?? config.stellar.contracts.lumenToken ?? null,
         crowdfundVault:
-          (overrides.crowdfundVault as string | undefined) ??
+          overrides.crowdfundVault ??
           config.stellar.contracts.crowdfundVault ??
           null,
         projectRegistry:
-          (overrides.projectRegistry as string | undefined) ??
+          overrides.projectRegistry ??
           config.stellar.contracts.projectRegistry ??
           null,
         contributorRegistry:
-          (overrides.contributorRegistry as string | undefined) ??
+          overrides.contributorRegistry ??
           config.stellar.contracts.contributorRegistry ??
           null,
         matchingPool:
-          (overrides.matchingPool as string | undefined) ??
+          overrides.matchingPool ??
           config.stellar.contracts.matchingPool ??
           null,
         treasury:
-          (overrides.treasury as string | undefined) ??
-          config.stellar.contracts.treasury ??
-          null,
+          overrides.treasury ?? config.stellar.contracts.treasury ?? null,
       },
     };
   }

apps/backend/src/health/health.controller.ts

@@ -56,4 +56,32 @@ export class HealthController {
 
     return healthReport;
   }
+
+  @Get('health/latency')
+  @ApiOperation({
+    summary:
+      'Returns latency budget health signals for Horizon and Soroban RPC',
+    description:
+      'Probes each testnet dependency and classifies response time against ' +
+      'configurable thresholds. Returns HTTP 200 for ok/degraded and HTTP 503 ' +
+      'when any dependency exceeds its hard-down threshold. ' +
+      'Thresholds are set via HEALTH_HORIZON_LATENCY_* and ' +
+      'HEALTH_SOROBAN_RPC_LATENCY_* environment variables.',
+  })
+  @ApiOkResponse({
+    description:
+      'All dependencies are within their latency budgets, or only degraded.',
+  })
+  @ApiServiceUnavailableResponse({
+    description:
+      'At least one dependency has exceeded its hard-down latency threshold.',
+  })
+  async getLatencyHealth(@Res({ passthrough: true }) response: Response) {
+    const report = await this.healthService.getHealthReport();
+    const latencyReport = report.latencyBudget;
+
+    response.status(latencyReport.overallState === 'hard_down' ? 503 : 200);
+
+    return latencyReport;
+  }
 }

apps/backend/src/health/health.module.ts

@@ -6,6 +6,7 @@ import { StellarModule } from '../stellar/stellar.module';
 import { ContractHealthService } from './contract-health.service';
 import { HealthController } from './health.controller';
 import { HealthService } from './health.service';
+import { LatencyBudgetHealthService } from './latency-budget.health.service';
 
 @Module({
   imports: [
@@ -18,6 +19,6 @@ import { HealthService } from './health.service';
     StellarModule,
   ],
   controllers: [HealthController],
-  providers: [HealthService, ContractHealthService],
+  providers: [HealthService, ContractHealthService, LatencyBudgetHealthService],
 })
 export class HealthModule {}

apps/backend/src/health/health.service.spec.ts

@@ -6,13 +6,17 @@ import { of, throwError } from 'rxjs';
 import { CacheService } from '../cache/cache.service';
 import { StellarService } from '../stellar/stellar.service';
 import { HealthService } from './health.service';
+import { LatencyBudgetHealthService } from './latency-budget.health.service';
 
 describe('HealthService', () => {
   let service: HealthService;
   let dataSource: { query: jest.Mock };
   let cacheService: { checkHealth: jest.Mock };
   let stellarService: { checkHealth: jest.Mock };
   let httpService: { get: jest.Mock };
+  let latencyBudgetHealthService: {
+    getLatencyBudgetReport: jest.Mock;
+  };
 
   const mockHealthIndicatorService = {
     check: jest.fn((key: string) => ({
@@ -25,6 +29,12 @@ describe('HealthService', () => {
     })),
   };
 
+  const okLatencyReport = {
+    overallState: 'ok' as const,
+    checkedAt: new Date().toISOString(),
+    dependencies: [],
+  };
+
   beforeEach(async () => {
     dataSource = {
       query: jest.fn(),
@@ -38,6 +48,9 @@ describe('HealthService', () => {
     httpService = {
       get: jest.fn(),
     };
+    latencyBudgetHealthService = {
+      getLatencyBudgetReport: jest.fn().mockResolvedValue(okLatencyReport),
+    };
 
     const module: TestingModule = await Test.createTestingModule({
       providers: [
@@ -62,11 +75,20 @@ describe('HealthService', () => {
           provide: HttpService,
           useValue: httpService,
         },
+        {
+          provide: LatencyBudgetHealthService,
+          useValue: latencyBudgetHealthService,
+        },
       ],
     }).compile();
 
     service = module.get<HealthService>(HealthService);
     jest.clearAllMocks();
+
+    // Re-apply the default latency mock after clearAllMocks
+    latencyBudgetHealthService.getLatencyBudgetReport.mockResolvedValue(
+      okLatencyReport,
+    );
   });
 
   it('returns healthy when all critical and non-critical checks pass', async () => {
@@ -83,6 +105,8 @@ describe('HealthService', () => {
     expect(report.details.redis.status).toBe('up');
     expect(report.details.horizon.status).toBe('up');
     expect(report.details.externalApis.status).toBe('up');
+    expect(report.latencyBudget).toBeDefined();
+    expect(report.latencyBudget.overallState).toBe('ok');
   });
 
   it('returns degraded when a non-critical dependency fails', async () => {
@@ -139,4 +163,42 @@ describe('HealthService', () => {
       }),
     );
   });
+
+  // ── Latency budget integration ─────────────────────────────────────────────
+
+  it('returns status=error and summary=down when latency is hard_down', async () => {
+    dataSource.query.mockResolvedValue([{ '?column?': 1 }]);
+    cacheService.checkHealth.mockResolvedValue(true);
+    stellarService.checkHealth.mockResolvedValue(true);
+    httpService.get.mockReturnValue(of({ data: {} }));
+    latencyBudgetHealthService.getLatencyBudgetReport.mockResolvedValue({
+      overallState: 'hard_down',
+      checkedAt: new Date().toISOString(),
+      dependencies: [],
+    });
+
+    const report = await service.getHealthReport();
+
+    expect(report.status).toBe('error');
+    expect(report.summary).toBe('down');
+    expect(report.latencyBudget.overallState).toBe('hard_down');
+  });
+
+  it('returns status=ok and summary=degraded when latency is degraded', async () => {
+    dataSource.query.mockResolvedValue([{ '?column?': 1 }]);
+    cacheService.checkHealth.mockResolvedValue(true);
+    stellarService.checkHealth.mockResolvedValue(true);
+    httpService.get.mockReturnValue(of({ data: {} }));
+    latencyBudgetHealthService.getLatencyBudgetReport.mockResolvedValue({
+      overallState: 'degraded',
+      checkedAt: new Date().toISOString(),
+      dependencies: [],
+    });
+
+    const report = await service.getHealthReport();
+
+    expect(report.status).toBe('ok');
+    expect(report.summary).toBe('degraded');
+    expect(report.latencyBudget.overallState).toBe('degraded');
+  });
 });

apps/backend/src/health/health.service.ts

@@ -10,6 +10,10 @@ import { DataSource } from 'typeorm';
 import { firstValueFrom } from 'rxjs';
 import { CacheService } from '../cache/cache.service';
 import { StellarService } from '../stellar/stellar.service';
+import {
+  LatencyBudgetHealthService,
+  LatencyBudgetReport,
+} from './latency-budget.health.service';
 
 interface DependencyCheckResult {
   name: string;
@@ -30,6 +34,7 @@ type HealthPayload = {
 
 export interface LumenpulseHealthReport extends HealthCheckResult {
   summary: 'healthy' | 'degraded' | 'down';
+  latencyBudget: LatencyBudgetReport;
 }
 
 @Injectable()
@@ -40,11 +45,13 @@ export class HealthService {
     private readonly cacheService: CacheService,
     private readonly stellarService: StellarService,
     private readonly httpService: HttpService,
+    private readonly latencyBudgetHealthService: LatencyBudgetHealthService,
   ) {}
 
   async getHealthReport(): Promise<LumenpulseHealthReport> {
-    const database = await this.checkDatabase();
-    const dependencyChecks = await Promise.all([
+    const [database, latencyBudget, ...dependencyChecks] = await Promise.all([
+      this.checkDatabase(),
+      this.latencyBudgetHealthService.getLatencyBudgetReport(),
       this.checkRedis(),
       this.checkHorizon(),
       this.checkExternalApis(),
@@ -66,17 +73,23 @@ export class HealthService {
       }
     }
 
-    const status = database.isUp ? 'ok' : 'error';
-    const summary =
+    // A hard_down latency result is treated as a critical failure (503).
+    const latencyIsHardDown = latencyBudget.overallState === 'hard_down';
+    const latencyIsDegraded = latencyBudget.overallState === 'degraded';
+
+    const status = !database.isUp || latencyIsHardDown ? 'error' : 'ok';
+
+    const summary: LumenpulseHealthReport['summary'] =
       status === 'error'
         ? 'down'
-        : Object.keys(error).length > 0
+        : Object.keys(error).length > 0 || latencyIsDegraded
           ? 'degraded'
           : 'healthy';
 
     return {
       status,
       summary,
+      latencyBudget,
       info,
       error,
       details,

apps/backend/src/health/latency-budget.config.ts

@@ -0,0 +1,54 @@
+/**
+ * Latency budget thresholds for testnet dependency health checks.
+ *
+ * Each dependency has two thresholds:
+ *   - degradedMs  – response time above this triggers a "degraded" signal
+ *                   (HTTP 200, summary = degraded).
+ *   - hardDownMs  – response time above this (or a connection failure) triggers
+ *                   a "hard-down" signal (HTTP 503, summary = down).
+ *
+ * Values are sourced from environment variables so they can be tuned per
+ * deployment without a code change. See `.env.example` for the full list.
+ */
+
+export interface LatencyBudgetThreshold {
+  /** Latency (ms) above which the dependency is considered degraded */
+  degradedMs: number;
+  /** Latency (ms) above which the dependency is considered hard-down */
+  hardDownMs: number;
+}
+
+export interface LatencyBudgetConfig {
+  horizon: LatencyBudgetThreshold;
+  sorobanRpc: LatencyBudgetThreshold;
+}
+
+const env = process.env;
+
+const toMs = (raw: string | undefined, fallback: number): number => {
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+
+export const latencyBudgetConfig: LatencyBudgetConfig = {
+  horizon: {
+    degradedMs: toMs(
+      env.HEALTH_HORIZON_LATENCY_DEGRADED_MS,
+      1_000, // 1 s — warn if Horizon is slow
+    ),
+    hardDownMs: toMs(
+      env.HEALTH_HORIZON_LATENCY_HARD_DOWN_MS,
+      4_000, // 4 s — treat as hard-down
+    ),
+  },
+  sorobanRpc: {
+    degradedMs: toMs(
+      env.HEALTH_SOROBAN_RPC_LATENCY_DEGRADED_MS,
+      1_500, // 1.5 s — RPC is heavier than Horizon root
+    ),
+    hardDownMs: toMs(
+      env.HEALTH_SOROBAN_RPC_LATENCY_HARD_DOWN_MS,
+      5_000, // 5 s — treat as hard-down
+    ),
+  },
+};