diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/GracefulDrainIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/GracefulDrainIT.kt new file mode 100644 index 0000000..abc8448 --- /dev/null +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/GracefulDrainIT.kt @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.api + +import com.fincore.test.containers.PostgresContainerExtension +import io.kotest.matchers.shouldBe +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.boot.test.web.client.TestRestTemplate +import org.springframework.boot.web.context.WebServerApplicationContext +import org.springframework.context.ApplicationContext +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Import +import org.springframework.http.HttpEntity +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpMethod +import org.springframework.http.ResponseEntity +import org.springframework.security.oauth2.jwt.Jwt +import org.springframework.security.oauth2.jwt.JwtDecoder +import org.springframework.test.annotation.DirtiesContext +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RestController +import java.time.Instant +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import kotlin.concurrent.thread + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ExtendWith(PostgresContainerExtension::class) +@Import(GracefulDrainIT.TestSecurity::class, GracefulDrainIT.SlowEndpointConfig::class) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class GracefulDrainIT( + @Autowired private val rest: TestRestTemplate, + @Autowired private val context: ApplicationContext, +) { + @TestConfiguration + class TestSecurity { + @Bean + fun jwtDecoder(): JwtDecoder = + JwtDecoder { token -> + Jwt + .withTokenValue(token) + .header("alg", "none") + .subject("graceful-drain-it") + .claim("scope", "ledger:read") + .issuedAt(Instant.now()) + .expiresAt(Instant.now().plusSeconds(EXPIRY_SECONDS)) + .build() + } + } + + @TestConfiguration + class SlowEndpointConfig { + @Bean + fun slowController(): SlowController = SlowController() + } + + @RestController + class SlowController { + @GetMapping(SLOW_PATH) + fun slow(): String { + enteredLatch.countDown() + proceedLatch.await(DRAIN_TIMEOUT_SECONDS, TimeUnit.SECONDS) + return "done" + } + } + + @BeforeEach + fun resetLatches() { + enteredLatch = CountDownLatch(1) + proceedLatch = CountDownLatch(1) + } + + @Test + fun `should complete an in-flight request while the web server shuts down gracefully`() { + val holder = AtomicReference>() + val request = issueSlowRequest(holder) + enteredLatch.await(ENTRY_TIMEOUT_SECONDS, TimeUnit.SECONDS) shouldBe true + + val drained = CountDownLatch(1) + val webServer = (context as WebServerApplicationContext).webServer + webServer.shutDownGracefully { drained.countDown() } + proceedLatch.countDown() + + request.join(JOIN_TIMEOUT_MILLIS) + drained.await(JOIN_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) shouldBe true + holder.get().statusCode.value() shouldBe OK + } + + private fun issueSlowRequest(holder: AtomicReference>): Thread { + val entity = HttpEntity(HttpHeaders().apply { setBearerAuth("drain-token") }) + return thread { holder.set(rest.exchange(SLOW_PATH, HttpMethod.GET, entity, String::class.java)) } + } + + companion object { + const val SLOW_PATH = "/test/slow" + private const val EXPIRY_SECONDS = 300L + private const val OK = 200 + private const val DRAIN_TIMEOUT_SECONDS = 10L + private const val ENTRY_TIMEOUT_SECONDS = 5L + private const val JOIN_TIMEOUT_MILLIS = 35_000L + + @Volatile + internal var enteredLatch = CountDownLatch(1) + + @Volatile + internal var proceedLatch = CountDownLatch(1) + + @JvmStatic + @DynamicPropertySource + fun datasourceProperties(registry: DynamicPropertyRegistry) { + registry.add("spring.datasource.url") { PostgresContainerExtension.jdbcUrl } + registry.add("spring.datasource.username") { PostgresContainerExtension.username } + registry.add("spring.datasource.password") { PostgresContainerExtension.password } + registry.add("spring.datasource.hikari.maximum-pool-size") { "2" } + registry.add("spring.jpa.hibernate.ddl-auto") { "none" } + } + } +} diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/HealthProbesIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/HealthProbesIT.kt new file mode 100644 index 0000000..4cba359 --- /dev/null +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/HealthProbesIT.kt @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.api + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fincore.test.containers.PostgresContainerExtension +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.boot.test.web.client.TestRestTemplate +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Import +import org.springframework.security.oauth2.jwt.Jwt +import org.springframework.security.oauth2.jwt.JwtDecoder +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import java.time.Instant + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ExtendWith(PostgresContainerExtension::class) +@Import(HealthProbesIT.TestSecurity::class) +class HealthProbesIT( + @Autowired private val rest: TestRestTemplate, + @Autowired private val objectMapper: ObjectMapper, +) { + @TestConfiguration + class TestSecurity { + @Bean + fun jwtDecoder(): JwtDecoder = + JwtDecoder { token -> + Jwt + .withTokenValue(token) + .header("alg", "none") + .subject("health-probes-it") + .claim("scope", "ledger:read") + .issuedAt(Instant.now()) + .expiresAt(Instant.now().plusSeconds(EXPIRY_SECONDS)) + .build() + } + } + + @Test + fun `should serve the liveness probe without a token and report only the jvm state`() { + val response = rest.getForEntity("/actuator/health/liveness", String::class.java) + + response.statusCode.value() shouldBe OK + val components = objectMapper.readTree(response.body).get("components") + components.shouldNotBeNull() + components.has("livenessState") shouldBe true + components.has("db") shouldBe false + } + + @Test + fun `should serve the readiness probe without a token aggregating database connectivity`() { + val response = rest.getForEntity("/actuator/health/readiness", String::class.java) + + response.statusCode.value() shouldBe OK + val components = objectMapper.readTree(response.body).get("components") + components.shouldNotBeNull() + components.get("db").get("status").asText() shouldBe "UP" + } + + @Test + fun `should serve the aggregate health endpoint without a token`() { + rest.getForEntity("/actuator/health", String::class.java).statusCode.value() shouldBe OK + } + + companion object { + private const val EXPIRY_SECONDS = 300L + private const val OK = 200 + + @JvmStatic + @DynamicPropertySource + fun datasourceProperties(registry: DynamicPropertyRegistry) { + registry.add("spring.datasource.url") { PostgresContainerExtension.jdbcUrl } + registry.add("spring.datasource.username") { PostgresContainerExtension.username } + registry.add("spring.datasource.password") { PostgresContainerExtension.password } + registry.add("spring.datasource.hikari.maximum-pool-size") { "2" } + registry.add("spring.jpa.hibernate.ddl-auto") { "none" } + } + } +} diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ReadinessDownIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ReadinessDownIT.kt new file mode 100644 index 0000000..2f60637 --- /dev/null +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ReadinessDownIT.kt @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.api + +import com.fasterxml.jackson.databind.ObjectMapper +import io.kotest.matchers.shouldBe +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.boot.test.web.client.TestRestTemplate +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Import +import org.springframework.security.oauth2.jwt.Jwt +import org.springframework.security.oauth2.jwt.JwtDecoder +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.testcontainers.containers.PostgreSQLContainer +import java.time.Instant + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(ReadinessDownIT.TestSecurity::class) +class ReadinessDownIT( + @Autowired private val rest: TestRestTemplate, + @Autowired private val objectMapper: ObjectMapper, +) { + @TestConfiguration + class TestSecurity { + @Bean + fun jwtDecoder(): JwtDecoder = + JwtDecoder { token -> + Jwt + .withTokenValue(token) + .header("alg", "none") + .subject("readiness-down-it") + .claim("scope", "ledger:read") + .issuedAt(Instant.now()) + .expiresAt(Instant.now().plusSeconds(EXPIRY_SECONDS)) + .build() + } + } + + @Test + fun `should flip readiness to down when the database becomes unreachable`() { + rest.getForEntity(READINESS, String::class.java).statusCode.value() shouldBe OK + + database.stop() + + val response = rest.getForEntity(READINESS, String::class.java) + response.statusCode.value() shouldBe SERVICE_UNAVAILABLE + objectMapper.readTree(response.body).get("status").asText() shouldBe "DOWN" + } + + companion object { + private const val EXPIRY_SECONDS = 300L + private const val OK = 200 + private const val SERVICE_UNAVAILABLE = 503 + private const val READINESS = "/actuator/health/readiness" + + @JvmStatic + private val database = + PostgreSQLContainer("postgres:17-alpine") + .withDatabaseName("readiness_down_it") + .withUsername("fincore") + .withPassword("fincore") + .also { it.start() } + + @JvmStatic + @DynamicPropertySource + fun datasourceProperties(registry: DynamicPropertyRegistry) { + registry.add("spring.datasource.url") { database.jdbcUrl } + registry.add("spring.datasource.username") { database.username } + registry.add("spring.datasource.password") { database.password } + registry.add("spring.datasource.hikari.maximum-pool-size") { "2" } + registry.add("spring.datasource.hikari.connection-timeout") { "2000" } + registry.add("spring.jpa.hibernate.ddl-auto") { "none" } + } + } +} diff --git a/services/ledger/src/main/kotlin/com/fincore/ledger/config/GracefulShutdownLogger.kt b/services/ledger/src/main/kotlin/com/fincore/ledger/config/GracefulShutdownLogger.kt new file mode 100644 index 0000000..758192d --- /dev/null +++ b/services/ledger/src/main/kotlin/com/fincore/ledger/config/GracefulShutdownLogger.kt @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.config + +import org.slf4j.LoggerFactory +import org.springframework.context.ApplicationListener +import org.springframework.context.event.ContextClosedEvent +import org.springframework.stereotype.Component + +@Component +class GracefulShutdownLogger : ApplicationListener { + private val log = LoggerFactory.getLogger(javaClass) + + override fun onApplicationEvent(event: ContextClosedEvent) { + log.info("graceful shutdown commencing, draining in-flight requests within the configured timeout") + } +} diff --git a/services/ledger/src/main/resources/application-prod.yml b/services/ledger/src/main/resources/application-prod.yml index ff0b3c3..18e96a0 100644 --- a/services/ledger/src/main/resources/application-prod.yml +++ b/services/ledger/src/main/resources/application-prod.yml @@ -12,6 +12,8 @@ spring: jwt: issuer-uri: ${KEYCLOAK_ISSUER_URI} management: + server: + port: 9090 tracing: sampling: probability: ${TRACING_SAMPLING_PROBABILITY:0.1} diff --git a/services/ledger/src/main/resources/application.yml b/services/ledger/src/main/resources/application.yml index c983d13..b015553 100644 --- a/services/ledger/src/main/resources/application.yml +++ b/services/ledger/src/main/resources/application.yml @@ -1,8 +1,11 @@ server: port: 8080 + shutdown: graceful spring: application: name: ledger-service + lifecycle: + timeout-per-shutdown-phase: 30s jpa: hibernate: ddl-auto: none @@ -27,6 +30,17 @@ logging: format: console: logstash management: + endpoint: + health: + probes: + enabled: true + show-components: always + show-details: never + group: + liveness: + include: livenessState + readiness: + include: readinessState,db endpoints: web: exposure: diff --git a/services/ledger/src/test/kotlin/com/fincore/ledger/config/GracefulShutdownLoggerTest.kt b/services/ledger/src/test/kotlin/com/fincore/ledger/config/GracefulShutdownLoggerTest.kt new file mode 100644 index 0000000..3317299 --- /dev/null +++ b/services/ledger/src/test/kotlin/com/fincore/ledger/config/GracefulShutdownLoggerTest.kt @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.config + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender +import io.kotest.matchers.shouldBe +import org.junit.jupiter.api.Test +import org.slf4j.LoggerFactory +import org.springframework.context.event.ContextClosedEvent +import org.springframework.context.support.GenericApplicationContext + +class GracefulShutdownLoggerTest { + @Test + fun `should emit exactly one info event when the context closes`() { + val logger = LoggerFactory.getLogger(GracefulShutdownLogger::class.java) as Logger + val appender = ListAppender().apply { start() } + logger.addAppender(appender) + val context = GenericApplicationContext().apply { refresh() } + + try { + GracefulShutdownLogger().onApplicationEvent(ContextClosedEvent(context)) + + appender.list.count { it.level == Level.INFO } shouldBe 1 + } finally { + logger.detachAppender(appender) + context.close() + } + } +} diff --git a/services/ledger/src/test/kotlin/com/fincore/ledger/config/ManagementPortPropertiesTest.kt b/services/ledger/src/test/kotlin/com/fincore/ledger/config/ManagementPortPropertiesTest.kt new file mode 100644 index 0000000..4e09e1d --- /dev/null +++ b/services/ledger/src/test/kotlin/com/fincore/ledger/config/ManagementPortPropertiesTest.kt @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.config + +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.junit.jupiter.api.Test +import org.springframework.boot.env.YamlPropertySourceLoader +import org.springframework.core.env.PropertySource +import org.springframework.core.io.ClassPathResource + +class ManagementPortPropertiesTest { + private val base = load("application.yml") + private val prod = load("application-prod.yml") + + @Test + fun `should bind a prod management port distinct from the public application port`() { + val appPort = base.getProperty("server.port") + val managementPort = prod.getProperty("management.server.port") + + managementPort shouldNotBe appPort + } + + @Test + fun `should enable graceful shutdown with the configured drain timeout`() { + base.getProperty("server.shutdown") shouldBe "graceful" + base.getProperty("spring.lifecycle.timeout-per-shutdown-phase") shouldBe "30s" + } + + @Test + fun `should expose liveness as jvm state only and readiness with database connectivity`() { + base.getProperty("management.endpoint.health.probes.enabled") shouldBe true + base.getProperty("management.endpoint.health.group.liveness.include") shouldBe "livenessState" + base.getProperty("management.endpoint.health.group.readiness.include") shouldBe "readinessState,db" + } + + @Test + fun `should show health components but not details on the public probe endpoints`() { + base.getProperty("management.endpoint.health.show-components") shouldBe "always" + base.getProperty("management.endpoint.health.show-details") shouldBe "never" + } + + private companion object { + fun load(resource: String): PropertySource<*> = YamlPropertySourceLoader().load(resource, ClassPathResource(resource)).first() + } +}