From 3b7d3b39772fea1503a712e56ea447d6da820832 Mon Sep 17 00:00:00 2001 From: "@tanya_r" Date: Tue, 16 Jun 2026 00:46:28 -0300 Subject: [PATCH 1/4] feat(ledger): wire prometheus metrics, otlp tracing and structured json logs Add two bounded ledger counters (ledger.transactions.posted with a post/reversal type tag, ledger.balance.reads) incremented on the success path only, enable the http.server.requests percentile histogram, bridge micrometer tracing to opentelemetry with a configurable otlp endpoint and sampling, and switch console logging to logstash structured json on the default, test and prod profiles while keeping the human-readable pattern in dev. Expose /actuator/prometheus without a token, mirroring health. Closes #51 --- gradle/libs.versions.toml | 2 + services/ledger/build.gradle.kts | 2 + .../com/fincore/ledger/api/ObservabilityIT.kt | 167 ++++++++++++++++++ .../ledger/application/AccountEntriesIT.kt | 1 + .../application/AuditRetryTopologyIT.kt | 1 + .../ledger/application/AuditWritePathIT.kt | 1 + .../ledger/application/MetricsTestConfig.kt | 18 ++ .../TransactionBalanceServiceIT.kt | 1 + .../application/TransactionReversalIT.kt | 1 + .../ledger/application/BalanceServiceImpl.kt | 3 + .../ledger/application/LedgerMetrics.kt | 34 ++++ .../ledger/application/TransactionPoster.kt | 4 + .../fincore/ledger/config/SecurityConfig.kt | 1 + .../src/main/resources/application-dev.yml | 8 + .../src/main/resources/application-prod.yml | 4 + .../src/main/resources/application-test.yml | 3 + .../ledger/src/main/resources/application.yml | 14 ++ .../src/main/resources/logback-spring.xml | 29 +-- .../CorrelationIdLogPatternTest.kt | 16 +- .../application/BalanceServiceImplTest.kt | 18 +- .../ledger/application/LedgerMetricsTest.kt | 37 ++++ .../application/TransactionPosterTest.kt | 9 + 22 files changed, 360 insertions(+), 14 deletions(-) create mode 100644 services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt create mode 100644 services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/MetricsTestConfig.kt create mode 100644 services/ledger/src/main/kotlin/com/fincore/ledger/application/LedgerMetrics.kt create mode 100644 services/ledger/src/test/kotlin/com/fincore/ledger/application/LedgerMetricsTest.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 910d2d4..51242d5 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -64,7 +64,9 @@ mapstruct-processor = { module = "org.mapstruct:mapstruct-processor", version.re # Observability micrometer-core = { module = "io.micrometer:micrometer-core", version.ref = "micrometer" } micrometer-registry-prometheus = { module = "io.micrometer:micrometer-registry-prometheus", version.ref = "micrometer" } +micrometer-tracing-bridge-otel = { module = "io.micrometer:micrometer-tracing-bridge-otel" } opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } +opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp" } # OpenAPI / Springdoc springdoc-openapi-starter = { module = "org.springdoc:springdoc-openapi-starter-webmvc-ui", version.ref = "springdoc" } diff --git a/services/ledger/build.gradle.kts b/services/ledger/build.gradle.kts index b7ec7f7..cceed8f 100644 --- a/services/ledger/build.gradle.kts +++ b/services/ledger/build.gradle.kts @@ -39,7 +39,9 @@ dependencies { runtimeOnly(libs.postgres.jdbc) implementation(libs.micrometer.registry.prometheus) + implementation(libs.micrometer.tracing.bridge.otel) implementation(libs.opentelemetry.api) + implementation(libs.opentelemetry.exporter.otlp) testImplementation(project(":libs:fincore-test-support")) testImplementation(libs.spring.boot.starter.test) diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt new file mode 100644 index 0000000..acac36d --- /dev/null +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.api + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fincore.core.Currency +import com.fincore.ledger.api.observability.CorrelationIdAttributes +import com.fincore.ledger.application.AccountService +import com.fincore.ledger.application.CreateAccountCommand +import com.fincore.ledger.application.EntryLine +import com.fincore.ledger.application.PostTransactionCommand +import com.fincore.ledger.application.TransactionService +import com.fincore.ledger.domain.enum.AccountType +import com.fincore.ledger.domain.enum.EntryDirection +import com.fincore.ledger.infrastructure.persistence.OutboxEventRepository +import com.fincore.test.containers.PostgresContainerExtension +import io.kotest.matchers.doubles.shouldBeGreaterThanOrEqual +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.micrometer.tracing.Tracer +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.slf4j.LoggerFactory +import org.slf4j.MDC +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.boot.test.system.CapturedOutput +import org.springframework.boot.test.system.OutputCaptureExtension +import org.springframework.boot.test.web.client.TestRestTemplate +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Import +import org.springframework.core.env.Environment +import org.springframework.security.oauth2.jwt.Jwt +import org.springframework.security.oauth2.jwt.JwtDecoder +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import java.math.BigDecimal +import java.time.Instant +import java.util.UUID + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ExtendWith(PostgresContainerExtension::class, OutputCaptureExtension::class) +@Import(ObservabilityIT.TestSecurity::class) +class ObservabilityIT( + @Autowired private val rest: TestRestTemplate, + @Autowired private val objectMapper: ObjectMapper, + @Autowired private val accountService: AccountService, + @Autowired private val transactionService: TransactionService, + @Autowired private val outboxRepository: OutboxEventRepository, + @Autowired private val tracer: Tracer, + @Autowired private val environment: Environment, +) { + @TestConfiguration + class TestSecurity { + @Bean + fun jwtDecoder(): JwtDecoder = + JwtDecoder { token -> + Jwt + .withTokenValue(token) + .header("alg", "none") + .subject("observability-it") + .claim("scope", "ledger:read ledger:write") + .issuedAt(Instant.now()) + .expiresAt(Instant.now().plusSeconds(EXPIRY_SECONDS)) + .build() + } + } + + @AfterEach + fun cleanOutbox() { + outboxRepository.deleteAll() + } + + @Test + fun `should expose the prometheus exposition without a token`() { + val response = rest.getForEntity("/actuator/prometheus", String::class.java) + + response.statusCode.value() shouldBe OK + (response.body ?: "") shouldContain "# TYPE" + } + + @Test + fun `should report the posted counter after a transaction is posted`() { + postBalanced() + + val body = rest.getForEntity("/actuator/prometheus", String::class.java).body ?: "" + + body shouldContain "ledger_transactions_posted_total" + postedCount(body) shouldBeGreaterThanOrEqual 1.0 + } + + @Test + fun `should emit a structured json log line carrying the correlation id from mdc`(output: CapturedOutput) { + val marker = "obs-probe-${UUID.randomUUID()}" + MDC.put(CorrelationIdAttributes.MDC_KEY, "obs-corr") + try { + LoggerFactory.getLogger(ObservabilityIT::class.java).info(marker) + } finally { + MDC.remove(CorrelationIdAttributes.MDC_KEY) + } + + val line = output.all.lineSequence().first { it.contains(marker) && it.trimStart().startsWith("{") } + val json = objectMapper.readTree(line) + + json.hasNonNull("@timestamp") shouldBe true + json.hasNonNull("level") shouldBe true + json.hasNonNull("logger_name") shouldBe true + json.get("message").asText() shouldBe marker + json.get("correlation_id").asText() shouldBe "obs-corr" + } + + @Test + fun `should expose a tracer bean and bind the tracing properties`() { + tracer.shouldNotBeNull() + environment.getProperty("management.tracing.sampling.probability").shouldNotBeNull() + (environment.getProperty("management.otlp.tracing.endpoint") ?: "") shouldContain "/v1/traces" + } + + private fun postedCount(exposition: String): Double = + POSTED_PATTERN + .find(exposition) + ?.groupValues + ?.get(1) + ?.toDouble() ?: 0.0 + + private fun postBalanced() { + val debit = accountService.create(CreateAccountCommand("obs-${UUID.randomUUID()}", AccountType.USER_WALLET, Currency.EUR, ACTOR)) + val credit = accountService.create(CreateAccountCommand("obs-${UUID.randomUUID()}", AccountType.USER_WALLET, Currency.EUR, ACTOR)) + transactionService.post( + PostTransactionCommand( + reference = "obs-${UUID.randomUUID()}", + description = null, + currency = Currency.EUR, + entries = + listOf( + EntryLine(debit.id, EntryDirection.DEBIT, BigDecimal(AMOUNT)), + EntryLine(credit.id, EntryDirection.CREDIT, BigDecimal(AMOUNT_NEG)), + ), + actor = ACTOR, + correlationId = UUID.randomUUID().toString(), + ), + ) + } + + private companion object { + const val EXPIRY_SECONDS = 3600L + const val OK = 200 + const val ACTOR = "observability-it" + const val AMOUNT = "100.00" + const val AMOUNT_NEG = "-100.00" + val POSTED_PATTERN = Regex("""ledger_transactions_posted_total\{[^}]*type="post"[^}]*}\s+([0-9.eE+]+)""") + + @JvmStatic + @DynamicPropertySource + fun datasourceProperties(registry: DynamicPropertyRegistry) { + registry.add("spring.datasource.url") { PostgresContainerExtension.jdbcUrl } + registry.add("spring.datasource.username") { PostgresContainerExtension.username } + registry.add("spring.datasource.password") { PostgresContainerExtension.password } + registry.add("spring.datasource.hikari.maximum-pool-size") { "2" } + registry.add("spring.jpa.hibernate.ddl-auto") { "none" } + } + } +} diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AccountEntriesIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AccountEntriesIT.kt index 3db378a..e0737a0 100644 --- a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AccountEntriesIT.kt +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AccountEntriesIT.kt @@ -43,6 +43,7 @@ import java.time.Instant AccountEntriesIT.JacksonConfig::class, OutboxEventPublisherImpl::class, AuditTrailWriterImpl::class, + MetricsTestConfig::class, ) class AccountEntriesIT( @Autowired private val entryQueryService: EntryQueryService, diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AuditRetryTopologyIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AuditRetryTopologyIT.kt index 6c1c720..a6e9ad3 100644 --- a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AuditRetryTopologyIT.kt +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AuditRetryTopologyIT.kt @@ -76,6 +76,7 @@ import java.util.concurrent.atomic.AtomicInteger AuditTrailWriterImpl::class, OutboxEventPublisherImpl::class, AuditRetryTopologyIT.JacksonConfig::class, + MetricsTestConfig::class, ) class AuditRetryTopologyIT( @Autowired private val accountService: AccountService, diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AuditWritePathIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AuditWritePathIT.kt index f5eea02..20532c0 100644 --- a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AuditWritePathIT.kt +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/AuditWritePathIT.kt @@ -62,6 +62,7 @@ import java.security.MessageDigest AuditTrailWriterImpl::class, OutboxEventPublisherImpl::class, AuditWritePathIT.JacksonConfig::class, + MetricsTestConfig::class, ) class AuditWritePathIT( @Autowired private val accountService: AccountService, diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/MetricsTestConfig.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/MetricsTestConfig.kt new file mode 100644 index 0000000..24bd211 --- /dev/null +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/MetricsTestConfig.kt @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.application + +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.context.annotation.Bean + +@TestConfiguration +class MetricsTestConfig { + @Bean + fun meterRegistry(): MeterRegistry = SimpleMeterRegistry() + + @Bean + fun ledgerMetrics(registry: MeterRegistry): LedgerMetrics = LedgerMetrics(registry) +} diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/TransactionBalanceServiceIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/TransactionBalanceServiceIT.kt index a974709..d2d4a27 100644 --- a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/TransactionBalanceServiceIT.kt +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/TransactionBalanceServiceIT.kt @@ -47,6 +47,7 @@ import java.time.Instant TransactionBalanceServiceIT.JacksonConfig::class, OutboxEventPublisherImpl::class, AuditTrailWriterImpl::class, + MetricsTestConfig::class, ) class TransactionBalanceServiceIT( @Autowired private val transactionService: TransactionService, diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/TransactionReversalIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/TransactionReversalIT.kt index 114e14f..0f4f21e 100644 --- a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/TransactionReversalIT.kt +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/application/TransactionReversalIT.kt @@ -45,6 +45,7 @@ import java.math.BigDecimal TransactionReversalIT.JacksonConfig::class, OutboxEventPublisherImpl::class, AuditTrailWriterImpl::class, + MetricsTestConfig::class, ) class TransactionReversalIT( @Autowired private val transactionService: TransactionService, diff --git a/services/ledger/src/main/kotlin/com/fincore/ledger/application/BalanceServiceImpl.kt b/services/ledger/src/main/kotlin/com/fincore/ledger/application/BalanceServiceImpl.kt index b5b1539..d23c16b 100644 --- a/services/ledger/src/main/kotlin/com/fincore/ledger/application/BalanceServiceImpl.kt +++ b/services/ledger/src/main/kotlin/com/fincore/ledger/application/BalanceServiceImpl.kt @@ -17,6 +17,7 @@ import java.time.Instant class BalanceServiceImpl( private val balanceRepository: AccountBalanceRepository, private val entryRepository: EntryRepository, + private val ledgerMetrics: LedgerMetrics, ) : BalanceService { @Transactional(readOnly = true) override fun current( @@ -24,6 +25,7 @@ class BalanceServiceImpl( currency: Currency, ): AccountBalance { val row = balanceRepository.findById(AccountBalanceKey(accountId.value, currency.code)).orElse(null) + ledgerMetrics.recordBalanceRead() return if (row == null) { AccountBalance(accountId, Money.zero(currency), null) } else { @@ -38,6 +40,7 @@ class BalanceServiceImpl( instant: Instant, ): AccountBalance { val sum = entryRepository.sumAmount(accountId.value, currency.code, instant) + ledgerMetrics.recordBalanceRead() return AccountBalance(accountId, Money.of(sum, currency), null) } } diff --git a/services/ledger/src/main/kotlin/com/fincore/ledger/application/LedgerMetrics.kt b/services/ledger/src/main/kotlin/com/fincore/ledger/application/LedgerMetrics.kt new file mode 100644 index 0000000..a26b91f --- /dev/null +++ b/services/ledger/src/main/kotlin/com/fincore/ledger/application/LedgerMetrics.kt @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.application + +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.stereotype.Component + +@Component +class LedgerMetrics( + registry: MeterRegistry, +) { + private val postedCounter: Counter = + Counter.builder(METRIC_POSTED).tag(TAG_TYPE, TYPE_POST).register(registry) + private val reversalCounter: Counter = + Counter.builder(METRIC_POSTED).tag(TAG_TYPE, TYPE_REVERSAL).register(registry) + private val balanceReadsCounter: Counter = + Counter.builder(METRIC_BALANCE_READS).register(registry) + + fun recordPost() = postedCounter.increment() + + fun recordReversal() = reversalCounter.increment() + + fun recordBalanceRead() = balanceReadsCounter.increment() + + private companion object { + const val METRIC_POSTED = "ledger.transactions.posted" + const val METRIC_BALANCE_READS = "ledger.balance.reads" + const val TAG_TYPE = "type" + const val TYPE_POST = "post" + const val TYPE_REVERSAL = "reversal" + } +} diff --git a/services/ledger/src/main/kotlin/com/fincore/ledger/application/TransactionPoster.kt b/services/ledger/src/main/kotlin/com/fincore/ledger/application/TransactionPoster.kt index e70ed4e..2bc3b8e 100644 --- a/services/ledger/src/main/kotlin/com/fincore/ledger/application/TransactionPoster.kt +++ b/services/ledger/src/main/kotlin/com/fincore/ledger/application/TransactionPoster.kt @@ -39,6 +39,7 @@ import java.time.Instant import java.util.UUID @Component +@Suppress("LongParameterList") class TransactionPoster( private val accountRepository: AccountRepository, private val transactionRepository: TransactionRepository, @@ -47,6 +48,7 @@ class TransactionPoster( private val outboxEventPublisher: OutboxEventPublisher, private val auditWriter: AuditTrailWriter, private val adapter: TransactionPersistenceAdapter, + private val ledgerMetrics: LedgerMetrics, ) { @Transactional fun post(command: PostTransactionCommand): PostedTransaction { @@ -70,6 +72,7 @@ class TransactionPoster( requestHash = command.requestHash, ), ) + ledgerMetrics.recordPost() return PostedTransaction(transaction.id, transaction.reference, transaction.status, postedAt) } @@ -96,6 +99,7 @@ class TransactionPoster( throw TransactionAlreadyReversedException(originalId, conflict) } recordReversalAudit(originalId, actor, requestHash, reason, compensating.id.toString()) + ledgerMetrics.recordReversal() return PostedTransaction(compensating.id, compensating.reference, compensating.status, postedAt) } diff --git a/services/ledger/src/main/kotlin/com/fincore/ledger/config/SecurityConfig.kt b/services/ledger/src/main/kotlin/com/fincore/ledger/config/SecurityConfig.kt index 42b2cfd..60c6867 100644 --- a/services/ledger/src/main/kotlin/com/fincore/ledger/config/SecurityConfig.kt +++ b/services/ledger/src/main/kotlin/com/fincore/ledger/config/SecurityConfig.kt @@ -55,6 +55,7 @@ class SecurityConfig { "/swagger-ui.html", "/actuator/health", "/actuator/health/**", + "/actuator/prometheus", ) } } diff --git a/services/ledger/src/main/resources/application-dev.yml b/services/ledger/src/main/resources/application-dev.yml index fdc64f3..f566b52 100644 --- a/services/ledger/src/main/resources/application-dev.yml +++ b/services/ledger/src/main/resources/application-dev.yml @@ -11,3 +11,11 @@ spring: resourceserver: jwt: issuer-uri: ${KEYCLOAK_ISSUER_URI:http://localhost:8081/realms/fincore} +logging: + structured: + format: + console: "" +management: + tracing: + sampling: + probability: 1.0 diff --git a/services/ledger/src/main/resources/application-prod.yml b/services/ledger/src/main/resources/application-prod.yml index efb8893..ff0b3c3 100644 --- a/services/ledger/src/main/resources/application-prod.yml +++ b/services/ledger/src/main/resources/application-prod.yml @@ -11,3 +11,7 @@ spring: resourceserver: jwt: issuer-uri: ${KEYCLOAK_ISSUER_URI} +management: + tracing: + sampling: + probability: ${TRACING_SAMPLING_PROBABILITY:0.1} diff --git a/services/ledger/src/main/resources/application-test.yml b/services/ledger/src/main/resources/application-test.yml index 27ac4ca..a411fad 100644 --- a/services/ledger/src/main/resources/application-test.yml +++ b/services/ledger/src/main/resources/application-test.yml @@ -7,3 +7,6 @@ spring: resourceserver: jwt: issuer-uri: http://localhost/realms/test +management: + tracing: + enabled: false diff --git a/services/ledger/src/main/resources/application.yml b/services/ledger/src/main/resources/application.yml index a904ce0..c983d13 100644 --- a/services/ledger/src/main/resources/application.yml +++ b/services/ledger/src/main/resources/application.yml @@ -22,8 +22,22 @@ springdoc: path: /v3/api-docs swagger-ui: path: /swagger-ui.html +logging: + structured: + format: + console: logstash management: endpoints: web: exposure: include: health,info,prometheus + metrics: + distribution: + percentiles-histogram: + http.server.requests: true + tracing: + sampling: + probability: 0.1 + otlp: + tracing: + endpoint: ${OTLP_TRACING_ENDPOINT:http://localhost:4318/v1/traces} diff --git a/services/ledger/src/main/resources/logback-spring.xml b/services/ledger/src/main/resources/logback-spring.xml index 434b50a..78fdcb7 100644 --- a/services/ledger/src/main/resources/logback-spring.xml +++ b/services/ledger/src/main/resources/logback-spring.xml @@ -1,13 +1,22 @@ - - - - ${LOG_PATTERN} - - - - - + + + + + ${LOG_PATTERN} + + + + + + + + + + + + + diff --git a/services/ledger/src/test/kotlin/com/fincore/ledger/api/observability/CorrelationIdLogPatternTest.kt b/services/ledger/src/test/kotlin/com/fincore/ledger/api/observability/CorrelationIdLogPatternTest.kt index ebea4d7..206a67f 100644 --- a/services/ledger/src/test/kotlin/com/fincore/ledger/api/observability/CorrelationIdLogPatternTest.kt +++ b/services/ledger/src/test/kotlin/com/fincore/ledger/api/observability/CorrelationIdLogPatternTest.kt @@ -7,10 +7,20 @@ import io.kotest.matchers.string.shouldContain import org.junit.jupiter.api.Test class CorrelationIdLogPatternTest { + private val logback: String = + requireNotNull(this::class.java.getResource("/logback-spring.xml")?.readText()) { + "logback-spring.xml not found on the classpath" + } + @Test - fun `console log pattern renders the correlation id with an empty default`() { - val logback = this::class.java.getResource("/logback-spring.xml")?.readText() - requireNotNull(logback) { "logback-spring.xml not found on the classpath" } + fun `dev profile keeps the human-readable correlation id pattern`() { + logback shouldContain "" logback shouldContain "%X{${CorrelationIdAttributes.MDC_KEY}:-}" } + + @Test + fun `non-dev profiles delegate to the boot console appender for structured output`() { + logback shouldContain "" + logback shouldContain "org/springframework/boot/logging/logback/console-appender.xml" + } } diff --git a/services/ledger/src/test/kotlin/com/fincore/ledger/application/BalanceServiceImplTest.kt b/services/ledger/src/test/kotlin/com/fincore/ledger/application/BalanceServiceImplTest.kt index ddba801..392d5a7 100644 --- a/services/ledger/src/test/kotlin/com/fincore/ledger/application/BalanceServiceImplTest.kt +++ b/services/ledger/src/test/kotlin/com/fincore/ledger/application/BalanceServiceImplTest.kt @@ -10,6 +10,7 @@ import com.fincore.ledger.infrastructure.persistence.AccountBalanceKey import com.fincore.ledger.infrastructure.persistence.AccountBalanceRepository import com.fincore.ledger.infrastructure.persistence.EntryRepository import io.kotest.matchers.shouldBe +import io.micrometer.core.instrument.simple.SimpleMeterRegistry import io.mockk.every import io.mockk.mockk import org.junit.jupiter.api.Test @@ -20,9 +21,12 @@ import java.util.Optional class BalanceServiceImplTest { private val balanceRepository = mockk() private val entryRepository = mockk() - private val service = BalanceServiceImpl(balanceRepository, entryRepository) + private val registry = SimpleMeterRegistry() + private val service = BalanceServiceImpl(balanceRepository, entryRepository, LedgerMetrics(registry)) private val accountId = AccountId.generate() + private fun balanceReads() = registry.counter("ledger.balance.reads").count() + @Test fun `should return zero when no balance row exists`() { every { balanceRepository.findById(any()) } returns Optional.empty() @@ -55,4 +59,16 @@ class BalanceServiceImplTest { balance.amount.amount.compareTo(BigDecimal("75.00")) shouldBe 0 } + + @Test + fun `should count one balance read per current and asOf call`() { + val key = AccountBalanceKey(accountId.value, "USD") + every { balanceRepository.findById(key) } returns Optional.empty() + every { entryRepository.sumAmount(accountId.value, "USD", any()) } returns BigDecimal.ZERO + + service.current(accountId, Currency.USD) + service.asOf(accountId, Currency.USD, Instant.parse("2026-06-05T12:00:00Z")) + + balanceReads() shouldBe 2.0 + } } diff --git a/services/ledger/src/test/kotlin/com/fincore/ledger/application/LedgerMetricsTest.kt b/services/ledger/src/test/kotlin/com/fincore/ledger/application/LedgerMetricsTest.kt new file mode 100644 index 0000000..08785bd --- /dev/null +++ b/services/ledger/src/test/kotlin/com/fincore/ledger/application/LedgerMetricsTest.kt @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.application + +import io.kotest.matchers.shouldBe +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import org.junit.jupiter.api.Test + +class LedgerMetricsTest { + private val registry = SimpleMeterRegistry() + private val metrics = LedgerMetrics(registry) + + @Test + fun `should increment the post-tagged counter on a recorded post`() { + metrics.recordPost() + + registry.counter("ledger.transactions.posted", "type", "post").count() shouldBe 1.0 + registry.counter("ledger.transactions.posted", "type", "reversal").count() shouldBe 0.0 + } + + @Test + fun `should increment the reversal-tagged counter on a recorded reversal`() { + metrics.recordReversal() + + registry.counter("ledger.transactions.posted", "type", "reversal").count() shouldBe 1.0 + registry.counter("ledger.transactions.posted", "type", "post").count() shouldBe 0.0 + } + + @Test + fun `should increment the balance reads counter once per recorded read`() { + metrics.recordBalanceRead() + metrics.recordBalanceRead() + + registry.counter("ledger.balance.reads").count() shouldBe 2.0 + } +} diff --git a/services/ledger/src/test/kotlin/com/fincore/ledger/application/TransactionPosterTest.kt b/services/ledger/src/test/kotlin/com/fincore/ledger/application/TransactionPosterTest.kt index 059a147..22dd59a 100644 --- a/services/ledger/src/test/kotlin/com/fincore/ledger/application/TransactionPosterTest.kt +++ b/services/ledger/src/test/kotlin/com/fincore/ledger/application/TransactionPosterTest.kt @@ -28,6 +28,7 @@ import com.fincore.ledger.infrastructure.persistence.TransactionPersistenceAdapt import com.fincore.ledger.infrastructure.persistence.TransactionRepository import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.shouldBe +import io.micrometer.core.instrument.simple.SimpleMeterRegistry import io.mockk.every import io.mockk.justRun import io.mockk.mockk @@ -45,6 +46,8 @@ class TransactionPosterTest { private val balanceRepository = mockk() private val outboxEventPublisher = mockk() private val auditWriter = mockk(relaxed = true) + private val registry = SimpleMeterRegistry() + private val ledgerMetrics = LedgerMetrics(registry) private val poster = TransactionPoster( accountRepository, @@ -54,8 +57,11 @@ class TransactionPosterTest { outboxEventPublisher, auditWriter, TransactionPersistenceAdapter(), + ledgerMetrics, ) + private fun postedCount(type: String) = registry.counter("ledger.transactions.posted", "type", type).count() + private val now = Instant.parse("2026-06-05T12:00:00Z") private val accountA = UUID.randomUUID() private val accountB = UUID.randomUUID() @@ -105,6 +111,7 @@ class TransactionPosterTest { verify(exactly = 2) { entryRepository.saveAndFlush(any()) } verify(exactly = 2) { balanceRepository.saveAndFlush(any()) } verify(exactly = 1) { outboxEventPublisher.publish(any(), any(), any(), any(), any()) } + postedCount("post") shouldBe 1.0 } @Test @@ -150,6 +157,7 @@ class TransactionPosterTest { shouldThrow { poster.post(command()) } verify(exactly = 0) { transactionRepository.saveAndFlush(any()) } + postedCount("post") shouldBe 0.0 } private fun transactionEntity( @@ -192,6 +200,7 @@ class TransactionPosterTest { entrySlots[1].direction shouldBe EntryDirection.DEBIT entrySlots[1].amount.compareTo(BigDecimal("100.00")) shouldBe 0 verify(exactly = 1) { outboxEventPublisher.publish(any(), any(), any(), any(), any()) } + postedCount("reversal") shouldBe 1.0 } @Test From e36cc17dbd301ecf3c5dbf8e69fcc6df89a85aa0 Mon Sep 17 00:00:00 2001 From: "@tanya_r" Date: Tue, 16 Jun 2026 01:09:45 -0300 Subject: [PATCH 2/4] fix(ledger): align micrometer and otel to the boot bom and unit-test structured logs Drop the explicit micrometer 1.13.7 and opentelemetry 1.41.0 pins so the boot bom manages every micrometer and opentelemetry artifact uniformly. The pins left micrometer-registry-prometheus at 1.13.7 while the tracing bridge dragged micrometer-core to 1.15.1, and opentelemetry-api at 1.41.0 against a 1.49.0 sdk, which made the prometheus scrape return 500. Move the structured-log assertion off the flaky OutputCapture path onto Boot StructuredLogEncoder in a unit test. --- gradle/libs.versions.toml | 8 ++-- .../com/fincore/ledger/api/ObservabilityIT.kt | 29 +------------ .../observability/StructuredLogFormatTest.kt | 43 +++++++++++++++++++ 3 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 services/ledger/src/test/kotlin/com/fincore/ledger/api/observability/StructuredLogFormatTest.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 51242d5..6556027 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,8 +11,6 @@ kotest = "5.9.1" mockk = "1.13.13" junit = "5.11.4" ulid = "5.2.3" -micrometer = "1.13.7" -opentelemetry = "1.41.0" ktlint = "1.5.0" detekt = "1.23.8" springdoc = "2.7.0" @@ -62,10 +60,10 @@ mapstruct-core = { module = "org.mapstruct:mapstruct", version.ref = "mapstruct" mapstruct-processor = { module = "org.mapstruct:mapstruct-processor", version.ref = "mapstruct" } # Observability -micrometer-core = { module = "io.micrometer:micrometer-core", version.ref = "micrometer" } -micrometer-registry-prometheus = { module = "io.micrometer:micrometer-registry-prometheus", version.ref = "micrometer" } +micrometer-core = { module = "io.micrometer:micrometer-core" } +micrometer-registry-prometheus = { module = "io.micrometer:micrometer-registry-prometheus" } micrometer-tracing-bridge-otel = { module = "io.micrometer:micrometer-tracing-bridge-otel" } -opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } +opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api" } opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp" } # OpenAPI / Springdoc diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt index acac36d..cc0dfd1 100644 --- a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt @@ -3,9 +3,7 @@ package com.fincore.ledger.api -import com.fasterxml.jackson.databind.ObjectMapper import com.fincore.core.Currency -import com.fincore.ledger.api.observability.CorrelationIdAttributes import com.fincore.ledger.application.AccountService import com.fincore.ledger.application.CreateAccountCommand import com.fincore.ledger.application.EntryLine @@ -23,13 +21,9 @@ import io.micrometer.tracing.Tracer import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith -import org.slf4j.LoggerFactory -import org.slf4j.MDC import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.boot.test.context.TestConfiguration -import org.springframework.boot.test.system.CapturedOutput -import org.springframework.boot.test.system.OutputCaptureExtension import org.springframework.boot.test.web.client.TestRestTemplate import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Import @@ -43,11 +37,10 @@ import java.time.Instant import java.util.UUID @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ExtendWith(PostgresContainerExtension::class, OutputCaptureExtension::class) +@ExtendWith(PostgresContainerExtension::class) @Import(ObservabilityIT.TestSecurity::class) class ObservabilityIT( @Autowired private val rest: TestRestTemplate, - @Autowired private val objectMapper: ObjectMapper, @Autowired private val accountService: AccountService, @Autowired private val transactionService: TransactionService, @Autowired private val outboxRepository: OutboxEventRepository, @@ -93,26 +86,6 @@ class ObservabilityIT( postedCount(body) shouldBeGreaterThanOrEqual 1.0 } - @Test - fun `should emit a structured json log line carrying the correlation id from mdc`(output: CapturedOutput) { - val marker = "obs-probe-${UUID.randomUUID()}" - MDC.put(CorrelationIdAttributes.MDC_KEY, "obs-corr") - try { - LoggerFactory.getLogger(ObservabilityIT::class.java).info(marker) - } finally { - MDC.remove(CorrelationIdAttributes.MDC_KEY) - } - - val line = output.all.lineSequence().first { it.contains(marker) && it.trimStart().startsWith("{") } - val json = objectMapper.readTree(line) - - json.hasNonNull("@timestamp") shouldBe true - json.hasNonNull("level") shouldBe true - json.hasNonNull("logger_name") shouldBe true - json.get("message").asText() shouldBe marker - json.get("correlation_id").asText() shouldBe "obs-corr" - } - @Test fun `should expose a tracer bean and bind the tracing properties`() { tracer.shouldNotBeNull() diff --git a/services/ledger/src/test/kotlin/com/fincore/ledger/api/observability/StructuredLogFormatTest.kt b/services/ledger/src/test/kotlin/com/fincore/ledger/api/observability/StructuredLogFormatTest.kt new file mode 100644 index 0000000..741e974 --- /dev/null +++ b/services/ledger/src/test/kotlin/com/fincore/ledger/api/observability/StructuredLogFormatTest.kt @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: 2026 FinCore Engine Authors + +package com.fincore.ledger.api.observability + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.LoggerContext +import ch.qos.logback.classic.spi.LoggingEvent +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.matchers.shouldBe +import org.junit.jupiter.api.Test +import org.springframework.boot.logging.logback.StructuredLogEncoder +import org.springframework.core.env.Environment +import org.springframework.core.env.StandardEnvironment + +class StructuredLogFormatTest { + private val encoder = + StructuredLogEncoder().apply { + setFormat("logstash") + context = LoggerContext().apply { putObject(Environment::class.java.name, StandardEnvironment()) } + start() + } + + @Test + fun `logstash encoder renders one json object with the standard fields and mdc`() { + val event = + LoggingEvent().apply { + loggerName = "com.fincore.ledger.test" + level = Level.INFO + setMessage("structured probe") + timeStamp = System.currentTimeMillis() + mdcPropertyMap = mapOf(CorrelationIdAttributes.MDC_KEY to "corr-123") + } + + val json = jacksonObjectMapper().readTree(encoder.encode(event)) + + json.hasNonNull("@timestamp") shouldBe true + json.get("level").asText() shouldBe "INFO" + json.get("logger_name").asText() shouldBe "com.fincore.ledger.test" + json.get("message").asText() shouldBe "structured probe" + json.get(CorrelationIdAttributes.MDC_KEY).asText() shouldBe "corr-123" + } +} From 37a55c2772f0314d18dce0d141b0200a8cf9d4a9 Mon Sep 17 00:00:00 2001 From: "@tanya_r" Date: Tue, 16 Jun 2026 01:17:17 -0300 Subject: [PATCH 3/4] test(ledger): diagnostic direct prometheus scrape to surface the 500 cause --- .../kotlin/com/fincore/ledger/api/ObservabilityIT.kt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt index cc0dfd1..229c5fe 100644 --- a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt @@ -41,6 +41,7 @@ import java.util.UUID @Import(ObservabilityIT.TestSecurity::class) class ObservabilityIT( @Autowired private val rest: TestRestTemplate, + @Autowired private val meterRegistry: io.micrometer.core.instrument.MeterRegistry, @Autowired private val accountService: AccountService, @Autowired private val transactionService: TransactionService, @Autowired private val outboxRepository: OutboxEventRepository, @@ -68,6 +69,13 @@ class ObservabilityIT( outboxRepository.deleteAll() } + @Test + fun `diagnostic direct scrape`() { + val pmr = meterRegistry as io.micrometer.prometheusmetrics.PrometheusMeterRegistry + val scraped = pmr.scrape() + scraped shouldContain "# TYPE" + } + @Test fun `should expose the prometheus exposition without a token`() { val response = rest.getForEntity("/actuator/prometheus", String::class.java) From 6db76f02c870f7197d0239163536aa81c85af084 Mon Sep 17 00:00:00 2001 From: "@tanya_r" Date: Tue, 16 Jun 2026 01:24:25 -0300 Subject: [PATCH 4/4] fix(ledger): enable observability registries in the prometheus it Spring boot disables metrics export registries in @SpringBootTest by default, so the prometheus registry was absent and /actuator/prometheus returned 500. Add @AutoConfigureObservability so the test context wires the prometheus registry, and drop the diagnostic scrape probe. --- .../kotlin/com/fincore/ledger/api/ObservabilityIT.kt | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt index 229c5fe..b17dd05 100644 --- a/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt +++ b/services/ledger/src/integrationTest/kotlin/com/fincore/ledger/api/ObservabilityIT.kt @@ -22,6 +22,7 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability import org.springframework.boot.test.context.SpringBootTest import org.springframework.boot.test.context.TestConfiguration import org.springframework.boot.test.web.client.TestRestTemplate @@ -37,11 +38,11 @@ import java.time.Instant import java.util.UUID @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@AutoConfigureObservability @ExtendWith(PostgresContainerExtension::class) @Import(ObservabilityIT.TestSecurity::class) class ObservabilityIT( @Autowired private val rest: TestRestTemplate, - @Autowired private val meterRegistry: io.micrometer.core.instrument.MeterRegistry, @Autowired private val accountService: AccountService, @Autowired private val transactionService: TransactionService, @Autowired private val outboxRepository: OutboxEventRepository, @@ -69,13 +70,6 @@ class ObservabilityIT( outboxRepository.deleteAll() } - @Test - fun `diagnostic direct scrape`() { - val pmr = meterRegistry as io.micrometer.prometheusmetrics.PrometheusMeterRegistry - val scraped = pmr.scrape() - scraped shouldContain "# TYPE" - } - @Test fun `should expose the prometheus exposition without a token`() { val response = rest.getForEntity("/actuator/prometheus", String::class.java)