feat(ledger): add liveness/readiness probes and graceful shutdown

services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/GracefulDrainIT.kt

@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: BUSL-1.1
+// SPDX-FileCopyrightText: 2026 FinCore Engine Authors
+
+package com.fincore.ledger.api
+
+import com.fincore.test.containers.PostgresContainerExtension
+import io.kotest.matchers.shouldBe
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.test.context.SpringBootTest
+import org.springframework.boot.test.context.TestConfiguration
+import org.springframework.boot.test.web.client.TestRestTemplate
+import org.springframework.boot.web.context.WebServerApplicationContext
+import org.springframework.context.ApplicationContext
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Import
+import org.springframework.http.HttpEntity
+import org.springframework.http.HttpHeaders
+import org.springframework.http.HttpMethod
+import org.springframework.http.ResponseEntity
+import org.springframework.security.oauth2.jwt.Jwt
+import org.springframework.security.oauth2.jwt.JwtDecoder
+import org.springframework.test.annotation.DirtiesContext
+import org.springframework.test.context.DynamicPropertyRegistry
+import org.springframework.test.context.DynamicPropertySource
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.RestController
+import java.time.Instant
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicReference
+import kotlin.concurrent.thread
+
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ExtendWith(PostgresContainerExtension::class)
+@Import(GracefulDrainIT.TestSecurity::class, GracefulDrainIT.SlowEndpointConfig::class)
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
+class GracefulDrainIT(
+    @Autowired private val rest: TestRestTemplate,
+    @Autowired private val context: ApplicationContext,
+) {
+    @TestConfiguration
+    class TestSecurity {
+        @Bean
+        fun jwtDecoder(): JwtDecoder =
+            JwtDecoder { token ->
+                Jwt
+                    .withTokenValue(token)
+                    .header("alg", "none")
+                    .subject("graceful-drain-it")
+                    .claim("scope", "ledger:read")
+                    .issuedAt(Instant.now())
+                    .expiresAt(Instant.now().plusSeconds(EXPIRY_SECONDS))
+                    .build()
+            }
+    }
+
+    @TestConfiguration
+    class SlowEndpointConfig {
+        @Bean
+        fun slowController(): SlowController = SlowController()
+    }
+
+    @RestController
+    class SlowController {
+        @GetMapping(SLOW_PATH)
+        fun slow(): String {
+            enteredLatch.countDown()
+            proceedLatch.await(DRAIN_TIMEOUT_SECONDS, TimeUnit.SECONDS)
+            return "done"
+        }
+    }
+
+    @BeforeEach
+    fun resetLatches() {
+        enteredLatch = CountDownLatch(1)
+        proceedLatch = CountDownLatch(1)
+    }
+
+    @Test
+    fun `should complete an in-flight request while the web server shuts down gracefully`() {
+        val holder = AtomicReference<ResponseEntity<String>>()
+        val request = issueSlowRequest(holder)
+        enteredLatch.await(ENTRY_TIMEOUT_SECONDS, TimeUnit.SECONDS) shouldBe true
+
+        val drained = CountDownLatch(1)
+        val webServer = (context as WebServerApplicationContext).webServer
+        webServer.shutDownGracefully { drained.countDown() }
+        proceedLatch.countDown()
+
+        request.join(JOIN_TIMEOUT_MILLIS)
+        drained.await(JOIN_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) shouldBe true
+        holder.get().statusCode.value() shouldBe OK
+    }
+
+    private fun issueSlowRequest(holder: AtomicReference<ResponseEntity<String>>): Thread {
+        val entity = HttpEntity<Void>(HttpHeaders().apply { setBearerAuth("drain-token") })
+        return thread { holder.set(rest.exchange(SLOW_PATH, HttpMethod.GET, entity, String::class.java)) }
+    }
+
+    companion object {
+        const val SLOW_PATH = "/test/slow"
+        private const val EXPIRY_SECONDS = 300L
+        private const val OK = 200
+        private const val DRAIN_TIMEOUT_SECONDS = 10L
+        private const val ENTRY_TIMEOUT_SECONDS = 5L
+        private const val JOIN_TIMEOUT_MILLIS = 35_000L
+
+        @Volatile
+        internal var enteredLatch = CountDownLatch(1)
+
+        @Volatile
+        internal var proceedLatch = CountDownLatch(1)
+
+        @JvmStatic
+        @DynamicPropertySource
+        fun datasourceProperties(registry: DynamicPropertyRegistry) {
+            registry.add("spring.datasource.url") { PostgresContainerExtension.jdbcUrl }
+            registry.add("spring.datasource.username") { PostgresContainerExtension.username }
+            registry.add("spring.datasource.password") { PostgresContainerExtension.password }
+            registry.add("spring.datasource.hikari.maximum-pool-size") { "2" }
+            registry.add("spring.jpa.hibernate.ddl-auto") { "none" }
+        }
+    }
+}

services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/HealthProbesIT.kt

@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: BUSL-1.1
+// SPDX-FileCopyrightText: 2026 FinCore Engine Authors
+
+package com.fincore.ledger.api
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fincore.test.containers.PostgresContainerExtension
+import io.kotest.matchers.nulls.shouldNotBeNull
+import io.kotest.matchers.shouldBe
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.test.context.SpringBootTest
+import org.springframework.boot.test.context.TestConfiguration
+import org.springframework.boot.test.web.client.TestRestTemplate
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Import
+import org.springframework.security.oauth2.jwt.Jwt
+import org.springframework.security.oauth2.jwt.JwtDecoder
+import org.springframework.test.context.DynamicPropertyRegistry
+import org.springframework.test.context.DynamicPropertySource
+import java.time.Instant
+
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ExtendWith(PostgresContainerExtension::class)
+@Import(HealthProbesIT.TestSecurity::class)
+class HealthProbesIT(
+    @Autowired private val rest: TestRestTemplate,
+    @Autowired private val objectMapper: ObjectMapper,
+) {
+    @TestConfiguration
+    class TestSecurity {
+        @Bean
+        fun jwtDecoder(): JwtDecoder =
+            JwtDecoder { token ->
+                Jwt
+                    .withTokenValue(token)
+                    .header("alg", "none")
+                    .subject("health-probes-it")
+                    .claim("scope", "ledger:read")
+                    .issuedAt(Instant.now())
+                    .expiresAt(Instant.now().plusSeconds(EXPIRY_SECONDS))
+                    .build()
+            }
+    }
+
+    @Test
+    fun `should serve the liveness probe without a token and report only the jvm state`() {
+        val response = rest.getForEntity("/actuator/health/liveness", String::class.java)
+
+        response.statusCode.value() shouldBe OK
+        val components = objectMapper.readTree(response.body).get("components")
+        components.shouldNotBeNull()
+        components.has("livenessState") shouldBe true
+        components.has("db") shouldBe false
+    }
+
+    @Test
+    fun `should serve the readiness probe without a token aggregating database connectivity`() {
+        val response = rest.getForEntity("/actuator/health/readiness", String::class.java)
+
+        response.statusCode.value() shouldBe OK
+        val components = objectMapper.readTree(response.body).get("components")
+        components.shouldNotBeNull()
+        components.get("db").get("status").asText() shouldBe "UP"
+    }
+
+    @Test
+    fun `should serve the aggregate health endpoint without a token`() {
+        rest.getForEntity("/actuator/health", String::class.java).statusCode.value() shouldBe OK
+    }
+
+    companion object {
+        private const val EXPIRY_SECONDS = 300L
+        private const val OK = 200
+
+        @JvmStatic
+        @DynamicPropertySource
+        fun datasourceProperties(registry: DynamicPropertyRegistry) {
+            registry.add("spring.datasource.url") { PostgresContainerExtension.jdbcUrl }
+            registry.add("spring.datasource.username") { PostgresContainerExtension.username }
+            registry.add("spring.datasource.password") { PostgresContainerExtension.password }
+            registry.add("spring.datasource.hikari.maximum-pool-size") { "2" }
+            registry.add("spring.jpa.hibernate.ddl-auto") { "none" }
+        }
+    }
+}

services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ReadinessDownIT.kt

@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: BUSL-1.1
+// SPDX-FileCopyrightText: 2026 FinCore Engine Authors
+
+package com.fincore.ledger.api
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import io.kotest.matchers.shouldBe
+import org.junit.jupiter.api.Test
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.test.context.SpringBootTest
+import org.springframework.boot.test.context.TestConfiguration
+import org.springframework.boot.test.web.client.TestRestTemplate
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Import
+import org.springframework.security.oauth2.jwt.Jwt
+import org.springframework.security.oauth2.jwt.JwtDecoder
+import org.springframework.test.context.DynamicPropertyRegistry
+import org.springframework.test.context.DynamicPropertySource
+import org.testcontainers.containers.PostgreSQLContainer
+import java.time.Instant
+
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(ReadinessDownIT.TestSecurity::class)
+class ReadinessDownIT(
+    @Autowired private val rest: TestRestTemplate,
+    @Autowired private val objectMapper: ObjectMapper,
+) {
+    @TestConfiguration
+    class TestSecurity {
+        @Bean
+        fun jwtDecoder(): JwtDecoder =
+            JwtDecoder { token ->
+                Jwt
+                    .withTokenValue(token)
+                    .header("alg", "none")
+                    .subject("readiness-down-it")
+                    .claim("scope", "ledger:read")
+                    .issuedAt(Instant.now())
+                    .expiresAt(Instant.now().plusSeconds(EXPIRY_SECONDS))
+                    .build()
+            }
+    }
+
+    @Test
+    fun `should flip readiness to down when the database becomes unreachable`() {
+        rest.getForEntity(READINESS, String::class.java).statusCode.value() shouldBe OK
+
+        database.stop()
+
+        val response = rest.getForEntity(READINESS, String::class.java)
+        response.statusCode.value() shouldBe SERVICE_UNAVAILABLE
+        objectMapper.readTree(response.body).get("status").asText() shouldBe "DOWN"
+    }
+
+    companion object {
+        private const val EXPIRY_SECONDS = 300L
+        private const val OK = 200
+        private const val SERVICE_UNAVAILABLE = 503
+        private const val READINESS = "/actuator/health/readiness"
+
+        @JvmStatic
+        private val database =
+            PostgreSQLContainer("postgres:17-alpine")
+                .withDatabaseName("readiness_down_it")
+                .withUsername("fincore")
+                .withPassword("fincore")
+                .also { it.start() }
+
+        @JvmStatic
+        @DynamicPropertySource
+        fun datasourceProperties(registry: DynamicPropertyRegistry) {
+            registry.add("spring.datasource.url") { database.jdbcUrl }
+            registry.add("spring.datasource.username") { database.username }
+            registry.add("spring.datasource.password") { database.password }
+            registry.add("spring.datasource.hikari.maximum-pool-size") { "2" }
+            registry.add("spring.datasource.hikari.connection-timeout") { "2000" }
+            registry.add("spring.jpa.hibernate.ddl-auto") { "none" }
+        }
+    }
+}

services/ledger/src/main/kotlin/com/fincore/ledger/config/GracefulShutdownLogger.kt

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: BUSL-1.1
+// SPDX-FileCopyrightText: 2026 FinCore Engine Authors
+
+package com.fincore.ledger.config
+
+import org.slf4j.LoggerFactory
+import org.springframework.context.ApplicationListener
+import org.springframework.context.event.ContextClosedEvent
+import org.springframework.stereotype.Component
+
+@Component
+class GracefulShutdownLogger : ApplicationListener<ContextClosedEvent> {
+    private val log = LoggerFactory.getLogger(javaClass)
+
+    override fun onApplicationEvent(event: ContextClosedEvent) {
+        log.info("graceful shutdown commencing, draining in-flight requests within the configured timeout")
+    }
+}

services/ledger/src/main/resources/application-prod.yml

@@ -12,6 +12,8 @@ spring:
         jwt:
           issuer-uri: ${KEYCLOAK_ISSUER_URI}
 management:
+  server:
+    port: 9090
   tracing:
     sampling:
       probability: ${TRACING_SAMPLING_PROBABILITY:0.1}

services/ledger/src/main/resources/application.yml

@@ -1,8 +1,11 @@
 server:
   port: 8080
+  shutdown: graceful
 spring:
   application:
     name: ledger-service
+  lifecycle:
+    timeout-per-shutdown-phase: 30s
   jpa:
     hibernate:
       ddl-auto: none
@@ -27,6 +30,17 @@ logging:
     format:
       console: logstash
 management:
+  endpoint:
+    health:
+      probes:
+        enabled: true
+      show-components: always
+      show-details: never
+      group:
+        liveness:
+          include: livenessState
+        readiness:
+          include: readinessState,db
   endpoints:
     web:
       exposure:

services/ledger/src/test/kotlin/com/fincore/ledger/config/GracefulShutdownLoggerTest.kt

@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: BUSL-1.1
+// SPDX-FileCopyrightText: 2026 FinCore Engine Authors
+
+package com.fincore.ledger.config
+
+import ch.qos.logback.classic.Level
+import ch.qos.logback.classic.Logger
+import ch.qos.logback.classic.spi.ILoggingEvent
+import ch.qos.logback.core.read.ListAppender
+import io.kotest.matchers.shouldBe
+import org.junit.jupiter.api.Test
+import org.slf4j.LoggerFactory
+import org.springframework.context.event.ContextClosedEvent
+import org.springframework.context.support.GenericApplicationContext
+
+class GracefulShutdownLoggerTest {
+    @Test
+    fun `should emit exactly one info event when the context closes`() {
+        val logger = LoggerFactory.getLogger(GracefulShutdownLogger::class.java) as Logger
+        val appender = ListAppender<ILoggingEvent>().apply { start() }
+        logger.addAppender(appender)
+        val context = GenericApplicationContext().apply { refresh() }
+
+        try {
+            GracefulShutdownLogger().onApplicationEvent(ContextClosedEvent(context))
+
+            appender.list.count { it.level == Level.INFO } shouldBe 1
+        } finally {
+            logger.detachAppender(appender)
+            context.close()
+        }
+    }
+}