diff --git a/gateway/build-manifest.yaml b/gateway/build-manifest.yaml index ed4e647f5a..0a9c348929 100644 --- a/gateway/build-manifest.yaml +++ b/gateway/build-manifest.yaml @@ -1,7 +1,7 @@ version: v1 policies: - name: advanced-ratelimit - version: v1.1.2 + version: v1.2.0 gomodule: github.com/wso2/gateway-controllers/policies/advanced-ratelimit@v1 - name: analytics-header-filter version: v1.0.1 diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index df5fb32b39..b5e0466bbf 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -789,6 +789,18 @@ allow_payloads = false # during validation (with a warning). Prefer setting [collector] directly. send_request_body = false send_response_body = false +# Publishers each enabled event is delivered to. Any combination of: +# +# moesif maps the event onto Moesif's event model and posts it with the Moesif +# SDK. Configured under [analytics.publishers.moesif] below. +# otel exports the event to an OpenTelemetry collector as an OTLP log record +# (one record per transaction, event.name=wso2.api.transaction) over +# OTLP/HTTP. From the collector, any OTLP-capable backend — Datadog, +# Splunk, Elastic, Dynatrace, Grafana — is one exporter block away. +# Configured under [analytics.publishers.otel] below. +# +# Publishers are additive and independent: ["moesif", "otel"] delivers every event +# to both. An unknown name is a startup error, not a silent no-op. enabled_publishers = ["moesif"] [analytics.publishers.moesif] @@ -799,6 +811,85 @@ event_queue_size = 10000 batch_size = 50 timer_wakeup_seconds = 3 +[analytics.publishers.otel] +# Full OTLP/HTTP logs URL, including the /v1/logs path. Point it at a collector +# you run, or directly at a vendor's OTLP intake. +endpoint = "http://otel-collector:4318/v1/logs" +# Permits a plaintext http:// endpoint, and is required by the example endpoint +# above. Off by default in code +allow_insecure_transport = true +# Identifies the service +service_name = "gateway-runtime" +# Identifies the deployed build for rollout correlation. +service_version = '{{ env "VERSION" "" }}' +# Record count that triggers an export before flush_interval elapses. +batch_size = 100 +# How long a record waits when traffic is too slow to fill a batch. +flush_interval = "5s" +# Records held in memory when the endpoint is slow. Must be >= batch_size. +# Once full, on_queue_full decides which record is dropped — the queue never +# grows unbounded. +queue_capacity = 10000 +# Which record is discarded once the queue is full. Analytics is strictly +# downstream of request handling, so one of the two always happens; the choice is +# whether a slow endpoint costs the newest records or the oldest. +# "drop_new" — discard the incoming record, preserving older queued ones. +# Keeps the earliest view of an incident. +# "drop_oldest" — evict the oldest queued record to make room. Keeps the most +# recent traffic, at the cost of the start of the backlog. +# Same two values as traffic_logging.http.on_queue_full. +on_queue_full = "drop_new" +# Bounds a single export attempt. +timeout = "10s" +# Retry attempts after the initial one. Only transport errors, 429 and 5xx are +# retried — any other 4xx means the endpoint rejected the payload's shape, which +# retrying can only amplify. 0 disables retries. +max_retries = 3 +# Base delay for exponential backoff, with jitter applied per attempt so replicas +# retrying after a shared outage do not resynchronize into a thundering herd. A +# Retry-After from the endpoint replaces this rather than adding to it. +retry_backoff = "1s" +# Fraction of queue_capacity at which a retrying batch abandons its remaining +# budget and returns to draining. One worker exports, so nothing drains the queue +# while a batch retries: past this depth, retrying to save one batch costs more +# records than it rescues. 0 lets every batch use its full budget. +retry_abort_queue_ratio = 0.5 +# "none" (default, matching the OTLP specification's own default) or "gzip". gzip +# trades CPU on the export worker for a large egress reduction — these records are +# verbose JSON — and every OTLP/HTTP receiver is required to support it. Worth +# enabling when the endpoint is across a network you pay for. +compression = "none" + +# ==== UNCOMMENT WHEN REQUIRED ================================ +# Headers sent on every export, for a vendor OTLP intake that authenticates by +# header. Values are secrets: they are never logged. +# [analytics.publishers.otel.headers] +# X-Moesif-Application-Id = '{{ env "MOESIF_APP_ID" "" }}' +# Extra OTLP resource attributes, e.g. deployment.environment.name. +# [analytics.publishers.otel.resource_attributes] +# "deployment.environment.name" = "prod" + +# TLS to the OTLP endpoint. Ignored when `endpoint` is http. +[analytics.publishers.otel.tls] +# PEM bundle used to verify the endpoint's certificate. Empty means the system +# trust store — correct for a vendor's OTLP intake, usually wrong for an +# in-cluster collector fronted by a private CA. +ca_file = "" +# Client certificate and key for mTLS, in PEM form. Both or neither: setting one +# without the other is a startup error rather than a silently unauthenticated +# connection. Both files are read and parsed at startup, so a wrong path or a +# mismatched pair fails immediately instead of at the first export. +# +# cert_file = "/secrets/gateway-runtime/otel-client.crt" +# key_file = "/secrets/gateway-runtime/otel-client.key" +cert_file = "" +key_file = "" +# Disables verification of the endpoint's certificate. Off by default; when on, +# startup logs a warning naming the endpoint, because analytics records carry +# request metadata and, with collector body capture enabled, payloads. +insecure_skip_verify = false + + # ============================================================================= # TRAFFIC LOGGING (consumer — enabling it activates the collector) # ============================================================================= diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 8d0c4b33e2..4957596d56 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -2730,6 +2730,10 @@ func (t *Translator) createGRPCAccessLog() (*accesslog.AccessLog, error) { Timeout: durationpb.New(time.Duration(grpcConfig.GRPCRequestTimeout)), }, }, + // populates HTTPResponseProperties.ResponseHeaders + // so responseContentType can resolve for a request that matched no route and + // therefore never reached the policy chain. + AdditionalResponseHeadersToLog: []string{"content-type"}, } grpcAccessLogAny, err := anypb.New(httpGrpcAccessLog) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 97ceda48a5..2a5cbe0629 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -26,6 +26,7 @@ import ( "maps" "net" "strconv" + "strings" "sync" "time" @@ -63,6 +64,8 @@ const ( DefaultAnalyticsPublisher = "default" // MoesifAnalyticsPublisher represents the Moesif analytics publisher. MoesifAnalyticsPublisher = "moesif" + // OTelAnalyticsPublisher represents the OpenTelemetry analytics publisher + OTelAnalyticsPublisher = "otel" // HeaderKeys represents the header keys. RequestHeadersKey = "request_headers" @@ -82,6 +85,10 @@ const ( // AIProviderAPIVersionMetadataKey represents the AI provider API version metadata key. AIProviderAPIVersionMetadataKey string = "ai:providerversion" + // RequestModelIDMetadataKey represents the model named in the request + // (Separate from ModelIDMetadataKey (which resolves to the response model). + RequestModelIDMetadataKey string = "aitoken:requestmodelid" + // UserIDMetadataKey represents the user ID metadata key for analytics. UserIDMetadataKey string = "x-wso2-user-id" @@ -122,6 +129,16 @@ func NewAnalytics(cfg *config.Config) *Analytics { publishers = append(publishers, publisher) slog.Info("Moesif publisher added") } + case OTelAnalyticsPublisher: + publisher, err := analytics_publisher.NewOTel(&analyticsCfg.Publishers.OTel) + if err != nil { + // Fail closed on invalid TLS material to avoid a healthy-looking gateway + // silently exporting nothing. Validation already confirms the material loads. + slog.Error("Failed to initialize the OTel analytics publisher; refusing to start", "error", err) + panic(fmt.Sprintf("otel analytics publisher configuration is unusable: %v", err)) + } + publishers = append(publishers, publisher) + slog.Info("OTel publisher added") default: slog.Warn("Unknown publisher type", "type", publisherName) } @@ -330,21 +347,26 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E request := logEntry.GetRequest() response := logEntry.GetResponse() + // Strip the query once at the source since it is shared across publishers and may contain credentials. + requestPath, _, _ := strings.Cut(request.GetPath(), "?") + // OriginalPath is the pre-rewrite :path, so it carries the client's query too. + originalPath, _, _ := strings.Cut(request.GetOriginalPath(), "?") + // Prepare operation operation := dto.Operation{} // operation.APIResourceTemplate = keyValuePairsFromMetadata[APIResourceTemplateKey] if request != nil { - operation.APIResourceTemplate = logEntry.GetRequest().GetOriginalPath() + operation.APIResourceTemplate = originalPath operation.APIMethod = logEntry.Request.GetRequestMethod().String() } // Prepare target target := dto.Target{} - target.ResponseCacheHit = false + target.ResponseCacheHit = isCacheHit(logEntry) if response != nil { target.TargetResponseCode = int(logEntry.GetResponse().GetResponseCode().Value) // target.Destination = keyValuePairsFromMetadata[DestinationKey] - target.Destination = logEntry.GetRequest().GetAuthority() + logEntry.GetRequest().GetPath() + target.Destination = logEntry.GetRequest().GetAuthority() + requestPath target.ResponseCodeDetail = logEntry.GetResponse().GetResponseCodeDetails() } @@ -527,6 +549,9 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E aiMetadata.LLMCost = parsedLLMCost } event.Properties["aiMetadata"] = aiMetadata + if requestModel := keyValuePairsFromMetadata[RequestModelIDMetadataKey]; requestModel != "" { + event.Properties[constants.RequestModelPropertyKey] = requestModel + } aiTokenUsage := dto.AITokenUsage{} // Prompt tokens @@ -599,6 +624,11 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E // requestSize is common to all API kinds; mirror responseSize using the Envoy access-log byte count. if request != nil { event.Properties["requestSize"] = request.GetRequestBodyBytes() + + // Store the concrete request path (without query parameters), separate from the route template. + if requestPath != "" { + event.Properties[constants.RequestPathPropertyKey] = requestPath + } } //Adding request and response headers for the analytics event @@ -666,6 +696,18 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E event.Properties["mcpAnalytics"] = mcpAnalytics } + // Fault classification, last so it sees the finished event. + fault := classifyFault(logEntry) + event.ErrorType = string(fault.ErrorType) + if fault.SubCategory != "" { + event.Error = &dto.Error{ + // The client-visible status. The in-development fault flow owns the + // real WSO2 numeric codes and should supply them here instead. + ErrorCode: event.ProxyResponseCode, + ErrorMessage: fault.SubCategory, + } + } + return event } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 2969693f9c..4f4ec64eb5 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -20,6 +20,7 @@ package analytics import ( "bytes" "log/slog" + "strings" "testing" "time" @@ -1312,3 +1313,122 @@ func createLogEntryWithStreamID(streamID string) *v3.HTTPAccessLogEntry { }, } } + +// The concrete request path is the only record of what a client actually asked +// for: the route template groups requests for an operation, and for a request +// that matched no route there is no template at all. The query string is cut off +// here rather than at each publisher, because an API key or token in a query +// parameter is an ordinary pattern in this product and publishers forward +// analytics to third parties. +func TestPrepareAnalyticEvent_RequestPathDropsQueryString(t *testing.T) { + for name, tc := range map[string]struct { + path string + want interface{} // nil = the property must be absent + }{ + "no query": {"/petstore/pet/12345", "/petstore/pet/12345"}, + "single param": {"/petstore/pet/12345?apikey=secret", "/petstore/pet/12345"}, + "multiple params": {"/search?q=cat&token=abc123&page=2", "/search"}, + "empty query": {"/petstore/pet/12345?", "/petstore/pet/12345"}, + "query only": {"?apikey=secret", nil}, + "root": {"/", "/"}, + "absent path": {"", nil}, + "encoded question mark": {"/pet/a%3Fb", "/pet/a%3Fb"}, + } { + t.Run(name, func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{}) + logEntry.Request.Path = tc.path + + event := NewAnalytics(&config.Config{}).prepareAnalyticEvent(logEntry) + actual, present := event.Properties[constants.RequestPathPropertyKey] + + if tc.want == nil { + if present { + t.Errorf("%s = %v; want absent", constants.RequestPathPropertyKey, actual) + } + return + } + if !present { + t.Fatalf("%s is missing for path %q", constants.RequestPathPropertyKey, tc.path) + } + if actual != tc.want { + t.Errorf("%s = %v, want %v", constants.RequestPathPropertyKey, actual, tc.want) + } + }) + } +} + +// APIResourceTemplate comes from Envoy's original_path (the pre-rewrite :path, +// which every proxied route here produces via context-path stripping), so it +// carries the client's query string just as Path does. It reaches http.route on +// the OTel publisher, the Moesif uri and the traffic-log path, so it is cut at +// this same single point rather than at each of them. +func TestPrepareAnalyticEvent_APIResourceTemplateDropsQueryString(t *testing.T) { + for name, tc := range map[string]struct { + originalPath string + path string + want string + }{ + "no query": {"/petstore/pet/12345", "/pet/12345", "/petstore/pet/12345"}, + "single param": {"/petstore/pet/12345?apikey=secret", "/pet/12345", "/petstore/pet/12345"}, + "multiple params": {"/search?q=cat&token=abc123", "/search", "/search"}, + "empty query": {"/petstore/pet?", "/pet", "/petstore/pet"}, + "query only": {"?apikey=secret", "/pet", ""}, + "absent original path": {"", "/pet/12345", ""}, + "encoded question mark": {"/pet/a%3Fb", "/a%3Fb", "/pet/a%3Fb"}, + // The reviewer's case: the rewritten Path is clean while the pre-rewrite + // OriginalPath still carries the credential. + "query only on original path": {"/petstore/pet?apikey=secret", "/pet", "/petstore/pet"}, + } { + t.Run(name, func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{}) + logEntry.Request.OriginalPath = tc.originalPath + logEntry.Request.Path = tc.path + + event := NewAnalytics(&config.Config{}).prepareAnalyticEvent(logEntry) + + if got := event.Operation.APIResourceTemplate; got != tc.want { + t.Errorf("APIResourceTemplate = %q, want %q", got, tc.want) + } + if strings.Contains(event.Operation.APIResourceTemplate, "?") { + t.Errorf("APIResourceTemplate %q still carries a query string", + event.Operation.APIResourceTemplate) + } + }) + } +} + +// Target.Destination reaches three consumers — the OTel publisher's +// wso2.upstream.destination, the traffic log's destination field, and the +// target.destination policy expression — so the query string is stripped at this +// single point rather than in each of them. An API key or token in a query +// parameter is an ordinary pattern here, and all three carry records off-box. +func TestPrepareAnalyticEvent_DestinationDropsQueryString(t *testing.T) { + for name, tc := range map[string]struct { + authority string + path string + want string + }{ + "no query": {"api.example.com", "/orders/v1.0/listings", "api.example.com/orders/v1.0/listings"}, + "credential": {"api.example.com", "/orders/v1.0/listings?apikey=secret", "api.example.com/orders/v1.0/listings"}, + "multiple params": {"localhost:8080", "/nofilter/anything?q=1&token=abc", "localhost:8080/nofilter/anything"}, + "bare question": {"localhost:8080", "/everything?", "localhost:8080/everything"}, + "no path": {"api.example.com", "", "api.example.com"}, + } { + t.Run(name, func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{}) + logEntry.Request.Authority = tc.authority + logEntry.Request.Path = tc.path + + event := NewAnalytics(&config.Config{}).prepareAnalyticEvent(logEntry) + if event.Target == nil { + t.Fatal("event.Target is nil") + } + if event.Target.Destination != tc.want { + t.Errorf("Destination = %q, want %q", event.Target.Destination, tc.want) + } + if strings.Contains(event.Target.Destination, "?") { + t.Errorf("Destination %q still carries a query string", event.Target.Destination) + } + }) + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go b/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go new file mode 100644 index 0000000000..2add99bb42 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package analytics + +import ( + "net/http" + + v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" +) + +// responseCodeDetailsViaUpstream is the value Envoy sets when the upstream +// produced the response. Any other value means Envoy or a filter synthesized it, +// which is the difference between "the backend answered 503" and "the backend +// never saw the request" — a distinction the status code alone cannot make. +const responseCodeDetailsViaUpstream = "via_upstream" + +// faultClassification is the derived error view of one request, written onto the +// canonical event so every publisher reads one classification instead of each +// inventing its own. +type faultClassification struct { + // ErrorType is the fault category: one of dto.FaultCategory's four values, + // or empty when the request was not a gateway fault. + ErrorType dto.FaultCategory + // SubCategory is set only where the response flag determines it. Where the + // flag proves the category but not the specific cause + SubCategory dto.FaultSubCategory +} + +// flagFault maps one Envoy response flag onto a classification. +type flagFault struct { + name string + set func(*v3.ResponseFlags) bool + category dto.FaultCategory + subCategory dto.FaultSubCategory +} + +// flagFaults is deliberately an ordered slice, not a map: several flags are set +// together on the same request (UpstreamRequestTimeout alongside +// UpstreamRetryLimitExceeded, for instance), and Go randomizes map iteration, so +// a map would yield a different error.type for identical requests. Most specific +// first. +// +// Three flags are deliberately absent because they do not describe a failure: +// DelayInjected and FaultInjected mark deliberate fault injection, and +// ResponseFromCacheFilter marks a cache hit. Treating "any flag set" as a fault +// would count every cached response as an error. +var flagFaults = []flagFault{ + // Access control, before anything upstream is attempted. + {"unauthorized_details", func(f *v3.ResponseFlags) bool { return f.GetUnauthorizedDetails() != nil }, + dto.FaultCategoryAuth, dto.AuthenticationOther}, + {"rate_limited", (*v3.ResponseFlags).GetRateLimited, + dto.FaultCategoryThrottled, dto.ThrottlingOther}, + // The rate-limit service itself failed. That is gateway infrastructure, not + // the client being throttled, so it must not inflate throttling counts. + {"rate_limit_service_error", (*v3.ResponseFlags).GetRateLimitServiceError, + dto.FaultCategoryOther, dto.OtherMediationError}, + + // Upstream timeouts. The flag names the cause precisely, so the + // sub-category is derived rather than guessed. + {"upstream_request_timeout", (*v3.ResponseFlags).GetUpstreamRequestTimeout, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionTimeout}, + {"stream_idle_timeout", (*v3.ResponseFlags).GetStreamIdleTimeout, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionTimeout}, + {"duration_timeout", (*v3.ResponseFlags).GetDurationTimeout, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionTimeout}, + {"upstream_max_stream_duration_reached", (*v3.ResponseFlags).GetUpstreamMaxStreamDurationReached, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionTimeout}, + + // Upstream deliberately withheld: no member passed health checking, or a + // circuit breaker shed the request to protect it. + {"no_healthy_upstream", (*v3.ResponseFlags).GetNoHealthyUpstream, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionSuspended}, + {"failed_local_healthcheck", (*v3.ResponseFlags).GetFailedLocalHealthcheck, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionSuspended}, + {"upstream_overflow", (*v3.ResponseFlags).GetUpstreamOverflow, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionSuspended}, + + // Upstream unreachable or the connection broke mid-flight. + {"dns_resolution_failure", (*v3.ResponseFlags).GetDnsResolutionFailure, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_connection_failure", (*v3.ResponseFlags).GetUpstreamConnectionFailure, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_connection_termination", (*v3.ResponseFlags).GetUpstreamConnectionTermination, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_remote_reset", (*v3.ResponseFlags).GetUpstreamRemoteReset, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_protocol_error", (*v3.ResponseFlags).GetUpstreamProtocolError, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_retry_limit_exceeded", (*v3.ResponseFlags).GetUpstreamRetryLimitExceeded, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + + // Gateway configuration: the request matched no route, cluster, or filter + // config. Nothing upstream was attempted. + {"no_route_found", (*v3.ResponseFlags).GetNoRouteFound, + dto.FaultCategoryOther, dto.OtherResourceNotFound}, + {"no_cluster_found", (*v3.ResponseFlags).GetNoClusterFound, + dto.FaultCategoryOther, dto.OtherResourceNotFound}, + {"no_filter_config_found", (*v3.ResponseFlags).GetNoFilterConfigFound, + dto.FaultCategoryOther, dto.OtherMediationError}, + // The gateway shed the request to protect itself. + {"overload_manager", (*v3.ResponseFlags).GetOverloadManager, + dto.FaultCategoryOther, dto.OtherMediationError}, + + // Client-side: the caller sent something unusable or went away. Kept as + // distinct error types because downstream_connection_termination is routine + // for streaming and LLM traffic (a cancelled response) and an operator will + // want to exclude it from a fault-rate alert. + {"invalid_envoy_request_headers", (*v3.ResponseFlags).GetInvalidEnvoyRequestHeaders, + dto.FaultCategoryOther, dto.OtherUnclassified}, + {"downstream_protocol_error", (*v3.ResponseFlags).GetDownstreamProtocolError, + dto.FaultCategoryOther, dto.OtherUnclassified}, + {"downstream_connection_termination", (*v3.ResponseFlags).GetDownstreamConnectionTermination, + dto.FaultCategoryOther, dto.OtherUnclassified}, + {"downstream_remote_reset", (*v3.ResponseFlags).GetDownstreamRemoteReset, + dto.FaultCategoryOther, dto.OtherUnclassified}, + // Last: Envoy reset the stream locally, which is often accompanied by a more + // specific flag above. + {"local_reset", (*v3.ResponseFlags).GetLocalReset, + dto.FaultCategoryOther, dto.OtherUnclassified}, +} + +// statusFault is the classification a gateway-synthesized status code implies. +type statusFault struct { + category dto.FaultCategory + subCategory dto.FaultSubCategory +} + +// statusFaults names the cause for the status codes the gateway itself produces +// for a known reason. A map is safe here where flagFaults needed a slice: these +// are exact lookups on one status code, so there is no overlap for iteration +// order to disturb. +// +// This is an interim table. It exists because the flag path cannot see these +// faults at all: the policy engine's own denials (api-key-auth's 401, +// basic-ratelimit's 429) reach the access log with no response flag set and an +// empty response_code_details, so classifyFault has nothing else to key on. When +// the fault flow lands and reports the specific cause, it supersedes this table +// and the sub-categories become exact (which limit was exceeded, which claim +// failed) rather than the generic value used here. +var statusFaults = map[int]statusFault{ + http.StatusUnauthorized: {dto.FaultCategoryAuth, dto.AuthenticationFailure}, + http.StatusForbidden: {dto.FaultCategoryAuth, dto.AuthenticationAuthorizationFailure}, + http.StatusNotFound: {dto.FaultCategoryOther, dto.OtherResourceNotFound}, + http.StatusMethodNotAllowed: {dto.FaultCategoryOther, dto.OtherMethodNotAllowed}, + http.StatusTooManyRequests: {dto.FaultCategoryThrottled, dto.ThrottlingOther}, +} + +// classifyFault derives the error view of a request from the Envoy access log. +// +// Response flags are the primary signal rather than the status code, because the +// status code cannot separate a gateway fault from a backend one: a 503 with +// NoHealthyUpstream never reached the backend, while a 503 with no flags and +// via_upstream is the backend's own answer. Those belong in different categories +// and would be indistinguishable from a status-range table. +// +// Precedence: a matching response flag, then — for a response the gateway +// synthesized — the status code, then a generic error for any other 4xx/5xx. +func classifyFault(logEntry *v3.HTTPAccessLogEntry) faultClassification { + flags := logEntry.GetCommonProperties().GetResponseFlags() + for _, candidate := range flagFaults { + if candidate.set(flags) { + return faultClassification{ErrorType: candidate.category, SubCategory: candidate.subCategory} + } + } + + response := logEntry.GetResponse() + if response == nil { + return faultClassification{} + } + status := int(response.GetResponseCode().GetValue()) + details := response.GetResponseCodeDetails() + + // Anything below 400 is not an error + if status < 400 { + return faultClassification{} + } + + // Only a response the gateway synthesized lets the status code name the + // cause. via_upstream means the backend chose this status, and then the + // specific sub-categories would assert something false: a backend enforcing + // its own auth returns 401 after the gateway's authentication already + // succeeded, and a backend with its own rate limit returns 429 without the + // gateway throttling anything. Attributing either to the gateway sends + // triage to the wrong component and inflates the gateway's own throttling + // counts. The origin stays legible to consumers regardless: the detail + // string is published as wso2.upstream.response.detail. + if details != responseCodeDetailsViaUpstream { + if fault, ok := statusFaults[status]; ok { + return faultClassification{ErrorType: fault.category, SubCategory: fault.subCategory} + } + } + + // An upstream-chosen status, or one the table does not name: still an error, + // in the category that claims nothing about the cause. + return faultClassification{ErrorType: dto.FaultCategoryOther, SubCategory: dto.OtherUnclassified} +} + +// isCacheHit reports whether Envoy served the response from its cache filter. +// ResponseFromCacheFilter is one of the response flags that does not describe a +// failure, which is why it is handled here and not in flagFaults. +func isCacheHit(logEntry *v3.HTTPAccessLogEntry) bool { + return logEntry.GetCommonProperties().GetResponseFlags().GetResponseFromCacheFilter() +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go new file mode 100644 index 0000000000..75fef91640 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package analytics + +import ( + "fmt" + "net/http" + "reflect" + "testing" + + v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" +) + +// faultEntry builds a minimal access-log entry: the flags Envoy set, plus the +// status and detail string the client saw. +func faultEntry(flags *v3.ResponseFlags, status uint32, details string) *v3.HTTPAccessLogEntry { + return &v3.HTTPAccessLogEntry{ + CommonProperties: &v3.AccessLogCommon{ResponseFlags: flags}, + Response: &v3.HTTPResponseProperties{ + ResponseCode: wrapperspb.UInt32(status), + ResponseCodeDetails: details, + }, + } +} + +// Every flag in the table must produce its own error.type and category. Driven +// off flagFaults itself, using reflection to set the one field under test, so a +// flag added to the table without a mapping cannot silently go untested. +func TestClassifyFault_EveryMappedFlag(t *testing.T) { + for _, mapped := range flagFaults { + t.Run(mapped.name, func(t *testing.T) { + flags := &v3.ResponseFlags{} + setFlagByErrorType(t, flags, mapped.name) + + got := classifyFault(faultEntry(flags, 503, "")) + // The category is what lands in ErrorType — the field existing Moesif + // consumers read. The flag name only identifies the table row. + assert.Equal(t, mapped.category, got.ErrorType) + assert.Equal(t, mapped.subCategory, got.SubCategory) + }) + } +} + +// setFlagByErrorType sets the ResponseFlags field whose snake_case name matches +// the table entry, proving the two agree. +func setFlagByErrorType(t *testing.T, flags *v3.ResponseFlags, errorType string) { + t.Helper() + target := snakeToPascal(errorType) + value := reflect.ValueOf(flags).Elem() + field := value.FieldByName(target) + require.True(t, field.IsValid(), "no ResponseFlags field named %q for error type %q", target, errorType) + switch field.Kind() { + case reflect.Bool: + field.SetBool(true) + case reflect.Ptr: + // UnauthorizedDetails is a message, not a bool. + field.Set(reflect.New(field.Type().Elem())) + default: + t.Fatalf("unexpected kind %s for %s", field.Kind(), target) + } +} + +func snakeToPascal(s string) string { + out := []byte{} + upper := true + for i := 0; i < len(s); i++ { + if s[i] == '_' { + upper = true + continue + } + c := s[i] + if upper && c >= 'a' && c <= 'z' { + c -= 32 + } + upper = false + out = append(out, c) + } + return string(out) +} + +// The three informational flags describe deliberate behaviour, not failure. A +// cache hit counted as a fault would be a silent, permanent error-rate inflation. +func TestClassifyFault_InformationalFlagsAreNotFaults(t *testing.T) { + cases := map[string]*v3.ResponseFlags{ + "cache hit": {ResponseFromCacheFilter: true}, + "delay injected": {DelayInjected: true}, + "fault injected": {FaultInjected: true}, + } + for name, flags := range cases { + t.Run(name, func(t *testing.T) { + got := classifyFault(faultEntry(flags, 200, responseCodeDetailsViaUpstream)) + assert.Empty(t, got.ErrorType) + assert.Empty(t, got.SubCategory) + }) + } +} + +// The distinction the status code cannot make: same 503, two different causes. +func TestClassifyFault_SameStatusDifferentCause(t *testing.T) { + gateway := classifyFault(faultEntry(&v3.ResponseFlags{NoHealthyUpstream: true}, 503, "no_healthy_upstream")) + assert.Equal(t, dto.FaultCategoryTargetConnectivity, gateway.ErrorType, + "a 503 the backend never saw is a connectivity fault") + + backend := classifyFault(faultEntry(&v3.ResponseFlags{}, 503, responseCodeDetailsViaUpstream)) + assert.Equal(t, dto.FaultCategoryOther, backend.ErrorType, + "a 503 the backend answered is not a connectivity fault") +} + +func TestClassifyFault_UpstreamResponses(t *testing.T) { + cases := []struct { + name string + status uint32 + wantType dto.FaultCategory + wantSub dto.FaultSubCategory + }{ + {"200 is not a fault", 200, "", ""}, + {"301 is not a fault", 301, "", ""}, + // The backend chose these, so they are recorded as errors but the + // gateway-specific sub-categories are withheld: the gateway neither + // authenticated, throttled, nor failed to route. + {"400 is a generic error", 400, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"401 is not attributed to gateway auth", 401, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"403 is not attributed to gateway authz", 403, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"404 is a generic error", 404, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"429 is not attributed to gateway throttling", 429, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"500 is a backend fault", 500, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"502 is a backend fault", 502, dto.FaultCategoryOther, dto.OtherUnclassified}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, responseCodeDetailsViaUpstream)) + assert.Equal(t, tc.wantType, got.ErrorType) + assert.Equal(t, tc.wantSub, got.SubCategory) + }) + } +} + +// A response Envoy or a filter synthesized without setting a flag — a policy +// denial via ext_proc, a direct response. The detail string is high-cardinality +// so error.type gets one stable value instead. +func TestClassifyFault_SynthesizedResponses(t *testing.T) { + cases := []struct { + details string + status uint32 + wantType dto.FaultCategory + }{ + // A status in statusFaults names its own cause when the gateway + // synthesized the response. + {"ext_authz_denied", 403, dto.FaultCategoryAuth}, + {"direct_response", 401, dto.FaultCategoryAuth}, + {"direct_response", 404, dto.FaultCategoryOther}, + // Not in the table: an error, with nothing claimed about the cause. + {"ext_proc_error_gRPC_error_13", 500, dto.FaultCategoryOther}, + // A synthesized redirect is not a failure. + {"direct_response", 302, ""}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("%s_%d", tc.details, tc.status), func(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, tc.details)) + assert.Equal(t, tc.wantType, got.ErrorType) + }) + } +} + +// Every entry in statusFaults must be reachable for a gateway-synthesized +// response. Driven off the table itself, so an entry added without a test +// cannot slip through. Empty details are used deliberately: that is exactly +// what the policy engine's own denials produce. +func TestClassifyFault_EveryMappedStatus(t *testing.T) { + require.NotEmpty(t, statusFaults) + for status, want := range statusFaults { + t.Run(fmt.Sprintf("status_%d", status), func(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, uint32(status), "")) + assert.Equal(t, want.category, got.ErrorType) + assert.Equal(t, want.subCategory, got.SubCategory) + }) + } +} + +// The gating: an identical status code classifies differently depending on who +// produced the response. Without this, a backend's own 401/429 would be +// attributed to the gateway's authentication or rate limiting. +func TestClassifyFault_StatusMappingIsGatedOnOrigin(t *testing.T) { + cases := []struct { + status uint32 + wantGateway dto.FaultCategory + wantSubGw dto.FaultSubCategory + }{ + {http.StatusUnauthorized, dto.FaultCategoryAuth, dto.AuthenticationFailure}, + {http.StatusForbidden, dto.FaultCategoryAuth, dto.AuthenticationAuthorizationFailure}, + {http.StatusNotFound, dto.FaultCategoryOther, dto.OtherResourceNotFound}, + {http.StatusMethodNotAllowed, dto.FaultCategoryOther, dto.OtherMethodNotAllowed}, + {http.StatusTooManyRequests, dto.FaultCategoryThrottled, dto.ThrottlingOther}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("status_%d", tc.status), func(t *testing.T) { + gw := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, "direct_response")) + assert.Equal(t, tc.wantGateway, gw.ErrorType, "gateway-synthesized names the cause") + assert.Equal(t, tc.wantSubGw, gw.SubCategory) + + up := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, responseCodeDetailsViaUpstream)) + assert.Equal(t, dto.FaultCategoryOther, up.ErrorType, + "an upstream-chosen status must not be attributed to the gateway") + assert.Equal(t, dto.OtherUnclassified, up.SubCategory) + }) + } +} + +// A response flag outranks the status table: a 429 carrying RateLimited is +// classified by the flag, and a 404 from NoRouteFound keeps its flag-derived +// sub-category rather than being re-derived from the status. +func TestClassifyFault_FlagOutranksStatus(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{RateLimited: true}, 429, "direct_response")) + assert.Equal(t, dto.FaultCategoryThrottled, got.ErrorType) + assert.Equal(t, dto.ThrottlingOther, got.SubCategory) + + got = classifyFault(faultEntry(&v3.ResponseFlags{NoRouteFound: true}, 404, "route_not_found")) + assert.Equal(t, dto.FaultCategoryOther, got.ErrorType) + assert.Equal(t, dto.OtherResourceNotFound, got.SubCategory) +} + +// The boundary is >= 400, not > 400: a 400 is an error, a 399 is not. +func TestClassifyFault_ErrorBoundaryIncludes400(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, 400, "direct_response")) + assert.Equal(t, dto.FaultCategoryOther, got.ErrorType, "400 is an error") + assert.Equal(t, dto.OtherUnclassified, got.SubCategory) + + for _, status := range []uint32{200, 204, 301, 304, 399} { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, status, "direct_response")) + assert.Empty(t, got.ErrorType, "status %d must not be a fault", status) + } +} + +// Several flags are routinely set on one request. The table is ordered, so the +// result must be deterministic — a map would return either one at random. +func TestClassifyFault_MultipleFlagsAreDeterministic(t *testing.T) { + flags := &v3.ResponseFlags{ + UpstreamRequestTimeout: true, + UpstreamRetryLimitExceeded: true, + LocalReset: true, + } + for i := 0; i < 50; i++ { + got := classifyFault(faultEntry(flags, 504, "")) + require.Equal(t, dto.FaultCategoryTargetConnectivity, got.ErrorType, + "the most specific flag must win on every call") + require.Equal(t, dto.TargetConnectivityConnectionTimeout, got.SubCategory, + "upstream_request_timeout maps to CONNECTION_TIMEOUT, not the generic OTHER") + } +} + +// A rate-limit *service* failure is gateway infrastructure, not the client being +// throttled: counting it as THROTTLED would misattribute an outage to callers. +func TestClassifyFault_RateLimitServiceErrorIsNotThrottling(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{RateLimitServiceError: true}, 500, "")) + assert.Equal(t, dto.FaultCategoryOther, got.ErrorType) + assert.Equal(t, dto.OtherMediationError, got.SubCategory) +} + +// Nothing may panic on a sparse entry: the ALS message is built by Envoy, and +// every level of it is optional. +func TestClassifyFault_NilSafety(t *testing.T) { + cases := map[string]*v3.HTTPAccessLogEntry{ + "empty entry": {}, + "no common props": {Response: &v3.HTTPResponseProperties{}}, + "no response": {CommonProperties: &v3.AccessLogCommon{}}, + "no flags": {CommonProperties: &v3.AccessLogCommon{}, Response: &v3.HTTPResponseProperties{}}, + "no response code": {CommonProperties: &v3.AccessLogCommon{ResponseFlags: &v3.ResponseFlags{}}, Response: &v3.HTTPResponseProperties{ResponseCodeDetails: responseCodeDetailsViaUpstream}}, + } + for name, entry := range cases { + t.Run(name, func(t *testing.T) { + got := classifyFault(entry) + assert.Empty(t, got.ErrorType) + }) + } +} + +func TestIsCacheHit(t *testing.T) { + assert.True(t, isCacheHit(faultEntry(&v3.ResponseFlags{ResponseFromCacheFilter: true}, 200, ""))) + assert.False(t, isCacheHit(faultEntry(&v3.ResponseFlags{}, 200, ""))) + assert.False(t, isCacheHit(&v3.HTTPAccessLogEntry{}), "a sparse entry must not panic") +} + +// The classifier is only useful if prepareAnalyticEvent actually writes its +// output onto the event every publisher then reads. +func TestPrepareAnalyticEvent_WritesFaultClassification(t *testing.T) { + analytics := NewAnalytics(&config.Config{}) + + t.Run("gateway fault", func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{APIIDKey: "api-1"}) + logEntry.CommonProperties.ResponseFlags = &v3.ResponseFlags{UpstreamRequestTimeout: true} + logEntry.Response = &v3.HTTPResponseProperties{ResponseCode: wrapperspb.UInt32(504)} + + event := analytics.prepareAnalyticEvent(logEntry) + require.NotNil(t, event) + // The category lands in ErrorType — the field existing consumers read. + assert.Equal(t, string(dto.FaultCategoryTargetConnectivity), event.ErrorType) + require.NotNil(t, event.Error) + assert.Equal(t, dto.TargetConnectivityConnectionTimeout, event.Error.ErrorMessage) + // The client-visible status, populated before the classification runs. + assert.Equal(t, 504, event.Error.ErrorCode) + }) + + t.Run("clean upstream response leaves the fields unset", func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{APIIDKey: "api-1"}) + logEntry.CommonProperties.ResponseFlags = &v3.ResponseFlags{} + logEntry.Response = &v3.HTTPResponseProperties{ + ResponseCode: wrapperspb.UInt32(200), + ResponseCodeDetails: responseCodeDetailsViaUpstream, + } + + event := analytics.prepareAnalyticEvent(logEntry) + require.NotNil(t, event) + assert.Empty(t, event.ErrorType) + assert.Nil(t, event.Error, "a successful request must not carry an error object") + }) + + // ResponseFromCacheFilter replaces what was a hardcoded false. + t.Run("cache hit reaches Target.ResponseCacheHit", func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{APIIDKey: "api-1"}) + logEntry.CommonProperties.ResponseFlags = &v3.ResponseFlags{ResponseFromCacheFilter: true} + logEntry.Response = &v3.HTTPResponseProperties{ResponseCode: wrapperspb.UInt32(200)} + + event := analytics.prepareAnalyticEvent(logEntry) + require.NotNil(t, event) + require.NotNil(t, event.Target) + assert.True(t, event.Target.ResponseCacheHit) + assert.Empty(t, event.ErrorType, "a cache hit is not a fault") + }) +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go index 74f99e102d..7302be2cfa 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go @@ -324,6 +324,23 @@ func (m *Moesif) Publish(event *dto.Event) { metadataMap["responseMediationLatency"] = event.Latencies.ResponseMediationLatency } + // Fault classification, derived in analytics.classifyFault from the Envoy + // response flags. Three keys only, matching what API Manager's Moesif + // integration already reads: errorType carries the fault category (AUTH / + // TARGET_CONNECTIVITY / THROTTLED / OTHER), errorMessage the sub-category. + // All three are omitted when the request was not a gateway fault. + if event.ErrorType != "" { + metadataMap["errorType"] = event.ErrorType + } + if event.Error != nil { + if event.Error.ErrorCode != 0 { + metadataMap["errorCode"] = event.Error.ErrorCode + } + if event.Error.ErrorMessage != "" { + metadataMap["errorMessage"] = string(event.Error.ErrorMessage) + } + } + // commonName if commonName, ok := event.Properties["commonName"]; ok && commonName != nil { metadataMap["commonName"] = commonName diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go index a447fa990b..90589bf042 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go @@ -360,6 +360,60 @@ func TestPublish_McpAPIType(t *testing.T) { assert.Equal(t, "search", mcpAnalytics["toolName"]) } +// The fault taxonomy reaches Moesif, not only the OTLP publisher: both read the +// same classification off the canonical event. Exactly three keys, matching what +// API Manager's Moesif integration already reads. +func TestPublish_WithFaultClassification(t *testing.T) { + moesif := createTestMoesifWithoutAPI() + + event := createBaseEvent() + event.ErrorType = string(dto.FaultCategoryTargetConnectivity) + event.Error = &dto.Error{ErrorCode: 504, ErrorMessage: dto.TargetConnectivityConnectionTimeout} + + moesif.Publish(event) + + assert.Len(t, moesif.events, 1) + metadata := getMetadata(moesif.events[0]) + assert.Equal(t, "TARGET_CONNECTIVITY", metadata["errorType"]) + assert.Equal(t, 504, metadata["errorCode"]) + assert.Equal(t, "CONNECTION_TIMEOUT", metadata["errorMessage"]) + // The category/event-category enums belong to the in-development fault flow + // and must not be published from here. + for _, absent := range []string{"eventCategory", "faultCategory", "faultSubCategory"} { + assert.NotContains(t, metadata, absent) + } +} + +// A request that was not a gateway fault carries none of the three keys, so a +// consumer can filter on presence. +func TestPublish_SuccessOmitsFaultFields(t *testing.T) { + moesif := createTestMoesifWithoutAPI() + moesif.Publish(createBaseEvent()) + + assert.Len(t, moesif.events, 1) + metadata := getMetadata(moesif.events[0]) + for _, key := range []string{"errorType", "errorCode", "errorMessage"} { + assert.NotContains(t, metadata, key, "%s must be absent on a successful event", key) + } +} + +// Every value errorType can take is one of the four categories API Manager +// expects — never an Envoy flag name or a status code. +func TestPublish_ErrorTypeIsAlwaysAFaultCategory(t *testing.T) { + for _, category := range []dto.FaultCategory{ + dto.FaultCategoryAuth, dto.FaultCategoryThrottled, + dto.FaultCategoryTargetConnectivity, dto.FaultCategoryOther, + } { + moesif := createTestMoesifWithoutAPI() + event := createBaseEvent() + event.ErrorType = string(category) + moesif.Publish(event) + + metadata := getMetadata(moesif.events[0]) + assert.Equal(t, string(category), metadata["errorType"]) + } +} + func TestPublish_WithPayloads(t *testing.T) { moesif := createTestMoesifWithoutAPI() diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go new file mode 100644 index 0000000000..c3b029a379 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -0,0 +1,1326 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "net" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/constants" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/metrics" +) + +// Identifiers with more than one reader. Individual attribute names are written +// as literals at their single call site: sharing them with the tests would let a +// typo satisfy its own assertion. +const ( + // otelAttrNamespace prefixes every attribute OpenTelemetry does not define. + otelAttrNamespace = "wso2" + // otelScopeName is the InstrumentationScope downstream pipelines route on. It + // must match the collector's scope filter; a mismatch fails silently. + otelScopeName = otelAttrNamespace + ".analytics" + // otelEventName is emitted both as the record's eventName and as an + // event.name attribute — collectors before ~v0.117 drop the former. + otelEventName = otelAttrNamespace + ".api.transaction" + + otelSeverityNumberInfo = 9 + otelSeverityTextInfo = "INFO" + + // otelCloseFlushTimeout bounds the shutdown flush when the caller's context + // carries no deadline. + otelCloseFlushTimeout = 5 * time.Second + // Bound retry waits to 30s to prevent an export worker from stalling and to + // match the endpoint Retry-After limit. + otelMaxRetryBackoff = 30 * time.Second + // otelPublisherName is this publisher's value for the `publisher` metric + // label. Kept local so metric call sites need no config import. + otelPublisherName = "otel" + // otelMaxResponseBytes caps how much of the endpoint's response is read. A + // 2xx body carries partialSuccess and a failure body carries an error + // message; neither may be allowed to grow the heap. + otelMaxResponseBytes int64 = 4 << 10 + // otelMaxHeaderAttributes caps how many header attributes one record may + // carry, per direction. Header names become attribute keys, so an over-broad + // allowlist grows the record's schema, not just its size. + otelMaxHeaderAttributes = 32 +) + +// ns qualifies an attribute name with the WSO2 namespace. +func ns(name string) string { return otelAttrNamespace + "." + name } + +// OTel exports analytics events to an OpenTelemetry collector as OTLP log +// records ("OTel Events": log records carrying an event.name), over OTLP/HTTP +// with a JSON body. +// +// The logs signal is used rather than traces or metrics because it is the only +// OTLP signal that represents one discrete transaction with per-consumer +// attributes intact: metrics aggregate the transaction away and impose a +// cardinality ceiling, and traces are sampled by design. +// +// The OTLP wire format is built directly rather than through the OpenTelemetry Go +// SDK, whose logs signal is still pre-1.0 (sdk/log v0.x). The OTLP/HTTP JSON +// encoding it would produce is stable, so only the transport would change. +// +// Publish never blocks and never performs I/O on the caller's goroutine: the ALS +// ingest path calls it, and blocking there backpressures Envoy's access-log +// stream. Records go to a bounded queue drained by one worker; a full queue drops +// and counts rather than growing. +type OTel struct { + cfg config.OTelPublisherConfig + client *http.Client + // endpoint with credential-bearing parts stripped, for logs + logEndpoint string + + queue chan *otelLogRecord + + stop chan struct{} + workerDone chan struct{} + closeOnce sync.Once + closeErr error + + droppedMu sync.Mutex + dropped int + // dropOldest is resolved once at construction rather than per record. + dropOldest bool + // gzip is resolved once at construction from cfg.Compression. + gzip bool + // retryAbortDepth is the queue depth at which a retrying batch gives up so + // the worker can resume draining. 0 disables the check. See export. + retryAbortDepth int +} + +// NewOTel creates the OTLP-logs publisher and starts its exporting worker. +// cfg is assumed validated by config.validateOTelPublisherConfig; the TLS +// material is loaded again here, so this constructor fails closed rather than +// starting a publisher that can never reach its endpoint. +func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { + if cfg == nil { + return nil, fmt.Errorf("config is nil") + } + + tlsCfg, err := buildOTelTLSConfig(cfg.TLS) + if err != nil { + return nil, err + } + + o := &OTel{ + cfg: *cfg, + logEndpoint: endpointForLog(cfg.Endpoint), + client: &http.Client{ + Timeout: cfg.Timeout, + Transport: &http.Transport{ + TLSClientConfig: tlsCfg, + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 90 * time.Second, + }, + // Never auto-follow a redirect: the target is chosen by the endpoint, + // not the operator, and following it would send analytics records to a + // destination nobody configured. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + queue: make(chan *otelLogRecord, cfg.QueueCapacity), + stop: make(chan struct{}), + workerDone: make(chan struct{}), + dropOldest: strings.EqualFold(strings.TrimSpace(cfg.OnQueueFull), config.QueueDropOldest), + gzip: strings.EqualFold(strings.TrimSpace(cfg.Compression), + config.OTelCompressionGzip), + retryAbortDepth: cfg.EffectiveRetryAbortDepth(), + } + o.initMetrics() + go o.run() + + // Headers are deliberately omitted: they carry credentials. + slog.Info("OTel analytics publisher started", + "endpoint", o.logEndpoint, "batchSize", cfg.BatchSize, + "flushInterval", cfg.FlushInterval, "queueCapacity", cfg.QueueCapacity, + "onQueueFull", cfg.OnQueueFull) + return o, nil +} + +// endpointForLog strips userinfo and the query string, either of which can carry +// an intake credential that would then be logged. +func endpointForLog(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "(unparseable endpoint)" + } + return u.Scheme + "://" + u.Host + u.Path +} + +// buildOTelTLSConfig assembles the client TLS configuration. +// +// X25519MLKEM768 leads CurvePreferences per the repository's post-quantum +// standard; the classical curves stay listed after it so an endpoint that does +// not yet offer the hybrid still completes a handshake. +func buildOTelTLSConfig(cfg config.OTelTLSConfig) (*tls.Config, error) { + out := &tls.Config{ + MinVersion: tls.VersionTLS12, + CurvePreferences: []tls.CurveID{ + tls.X25519MLKEM768, tls.X25519, tls.CurveP256, tls.CurveP384, + }, + InsecureSkipVerify: cfg.InsecureSkipVerify, // #nosec G402 -- off by default; opt-in warns at startup + } + + if cfg.CAFile != "" { + pem, err := os.ReadFile(cfg.CAFile) + if err != nil { + return nil, fmt.Errorf("tls: cannot read ca_file %q: %w", cfg.CAFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("tls: ca_file %q contains no usable PEM certificate", cfg.CAFile) + } + out.RootCAs = pool + } + + if (cfg.CertFile == "") != (cfg.KeyFile == "") { + return nil, fmt.Errorf("tls: cert_file and key_file must be set together for mTLS") + } + if cfg.CertFile != "" { + pair, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile) + if err != nil { + return nil, fmt.Errorf("tls: cannot load client certificate/key pair: %w", err) + } + out.Certificates = []tls.Certificate{pair} + } + return out, nil +} + +// Publish converts the event to an OTLP log record and enqueues it. +// +// Never blocks: analytics is strictly downstream of request handling, so a full +// queue costs a record and never a request. +func (o *OTel) Publish(event *dto.Event) { + if event == nil { + return + } + record := o.buildRecord(event) + + select { + case o.queue <- record: + mAnalyticsQueueDepth(otelPublisherName, len(o.queue)) + return + default: + } + + if o.dropOldest { + // Evict one old record and retry once. A single attempt is deliberate: a + // loop could spin while producers keep the queue full, turning a + // non-blocking Publish into an unbounded one. + select { + case <-o.queue: + o.countQueueDrop() + default: + } + select { + case o.queue <- record: + mAnalyticsQueueDepth(otelPublisherName, len(o.queue)) + return + default: + } + } + + o.countQueueDrop() +} + +// countQueueDrop records one record dropped for a full queue, warning on the +// first and then every hundredth so a sustained outage cannot flood the log. +func (o *OTel) countQueueDrop() { + mAnalyticsDropped(otelPublisherName, dropReasonQueueFull, 1) + count := o.countDrops(1) + if count == 1 || count%100 == 0 { + slog.Warn("OTel publisher queue full; dropping analytics event", + "droppedTotal", count, "queueCapacity", o.cfg.QueueCapacity, + "onQueueFull", o.cfg.OnQueueFull) + } +} + +// run drains the queue, exporting on a full batch or on the flush interval. +func (o *OTel) run() { + defer func() { + // The queue is not drained further after this point, so leaving the last + // non-zero depth published would read as a permanently backed-up queue. + mAnalyticsQueueDepth(otelPublisherName, 0) + close(o.workerDone) + }() + + ticker := time.NewTicker(o.cfg.FlushInterval) + defer ticker.Stop() + + batch := make([]*otelLogRecord, 0, o.cfg.BatchSize) + flush := func() { + if len(batch) == 0 { + return + } + o.export(batch) + batch = batch[:0] + } + + for { + select { + case record := <-o.queue: + mAnalyticsQueueDepth(otelPublisherName, len(o.queue)) + batch = append(batch, record) + if len(batch) >= o.cfg.BatchSize { + flush() + } + case <-ticker.C: + flush() + case <-o.stop: + // Drain what is still queued so a rolling update does not lose it. + for drained := true; drained; { + select { + case record := <-o.queue: + mAnalyticsQueueDepth(otelPublisherName, len(o.queue)) + batch = append(batch, record) + if len(batch) >= o.cfg.BatchSize { + flush() + } + default: + drained = false + } + } + flush() + return + } + } +} + +// Close stops the worker and flushes what is buffered, satisfying Closer. +func (o *OTel) Close(ctx context.Context) error { + o.closeOnce.Do(func() { + close(o.stop) + + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, otelCloseFlushTimeout) + defer cancel() + } + + select { + case <-o.workerDone: + case <-ctx.Done(): + o.closeErr = fmt.Errorf("otel publisher shutdown timed out: %w", ctx.Err()) + } + + o.droppedMu.Lock() + dropped := o.dropped + o.droppedMu.Unlock() + slog.Info("OTel analytics publisher stopped", "droppedTotal", dropped) + }) + return o.closeErr +} + +// export builds the OTLP payload for one batch and delivers it, retrying +// transport errors, 429 and 5xx with jittered exponential backoff. Any other 4xx +// means the endpoint rejected the payload's shape, so retrying would only amplify +// a permanent failure. +func (o *OTel) export(batch []*otelLogRecord) { + records := make([]otelLogRecord, 0, len(batch)) + for _, r := range batch { + records = append(records, *r) + } + + resource := newOTelAttrs(). + str("service.name", o.cfg.ServiceName). + str("service.version", o.cfg.ServiceVersion) + for k, v := range o.cfg.ResourceAttributes { + resource.str(k, v) + } + + body, err := json.Marshal(otelExportRequest{ + ResourceLogs: []otelResourceLogs{{ + Resource: otelResource{Attributes: resource.list()}, + ScopeLogs: []otelScopeLogs{{ + Scope: otelScope{Name: otelScopeName}, + LogRecords: records, + }}, + }}, + }) + if err != nil { + slog.Error("OTel publisher failed to marshal OTLP payload", "error", err, "records", len(records)) + o.dropRecords(dropReasonSerializeFailed, len(records)) + return + } + if o.gzip { + compressed, err := gzipBytes(body) + if err != nil { + slog.Error("OTel publisher failed to compress OTLP payload", "error", err, "records", len(records)) + o.dropRecords(dropReasonSerializeFailed, len(records)) + return + } + body = compressed + } + + // Covers every attempt and the waits between them: that total is what holds + // the worker, and therefore what lets the queue fill behind it. + start := time.Now() + defer func() { mAnalyticsExportDuration(otelPublisherName, time.Since(start).Seconds()) }() + + var lastErr error + // Delay before the NEXT attempt. A Retry-After replaces our own backoff + // rather than adding to it, so the endpoint's own pacing is what applies. + var nextDelay time.Duration + for attempt := 0; attempt <= o.cfg.MaxRetries; attempt++ { + if attempt > 0 { + // Head-of-line check, before committing to another wait. One worker + // exports, so nothing drains the queue while this batch retries. Past + // the abort depth, retrying to save this batch costs more newer records + // to queue-full than it rescues — so abandon it and resume draining. + if depth := len(o.queue); o.retryAbortDepth > 0 && depth >= o.retryAbortDepth { + o.dropRecords(dropReasonBackpressure, len(records)) + slog.Error("OTel publisher abandoning batch retries to resume draining; the endpoint "+ + "is reachable but too slow to keep up", + "records", len(records), "attempts", attempt, + "queueDepth", depth, "queueCapacity", o.cfg.QueueCapacity, "error", lastErr) + return + } + if !o.sleep(nextDelay, attempt) { + break // shutting down: stop retrying rather than hold shutdown open + } + } + + retryAfter, rejected, err := o.post(body, len(records)) + if err == nil { + // Records refused inside a 2xx are already counted as dropped, so + // publishing the whole batch would count them twice and let + // published+dropped exceed the number of events that ever existed. + mAnalyticsPublished(otelPublisherName, len(records)-rejected) + return + } + lastErr = err + + var perm *otelPermanentExportError + if errors.As(err, &perm) { + break // 4xx other than 429 — retrying cannot help + } + nextDelay = retryAfter // 0 unless the endpoint asked for a specific delay + } + + o.dropRecords(dropReasonSendFailed, len(records)) + slog.Error("OTel publisher failed to export analytics batch; dropping records", + "records", len(records), "attempts", o.cfg.MaxRetries+1, + "endpoint", o.logEndpoint, "error", lastErr) +} + +// otelPermanentExportError marks a response that must not be retried. +type otelPermanentExportError struct{ status int } + +func (e *otelPermanentExportError) Error() string { + return fmt.Sprintf("endpoint rejected the batch with status %d", e.status) +} + +// post performs one export attempt. It returns the endpoint's requested +// Retry-After when it supplies one so the caller can honor it over its own +// backoff, and how many records the endpoint refused inside a 2xx so the caller +// does not count those as published. +func (o *OTel) post(body []byte, records int) (time.Duration, int, error) { + ctx, cancel := context.WithTimeout(context.Background(), o.cfg.Timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, o.cfg.Endpoint, bytes.NewReader(body)) + if err != nil { + return 0, 0, fmt.Errorf("building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + if o.gzip { + req.Header.Set("Content-Encoding", "gzip") + } + for k, v := range o.cfg.Headers { + req.Header.Set(k, v) + } + + resp, err := o.client.Do(req) + if err != nil { + mAnalyticsExportError(otelPublisherName, errCodeTransport, 1) + // The error can embed the endpoint URL but never the payload, so no + // request data can leak into the application log here. + return 0, 0, fmt.Errorf("posting batch: %w", err) + } + defer resp.Body.Close() + + // Read a bounded prefix: enough for the partialSuccess field or an error + // message, and capped so a hostile endpoint cannot grow the heap. + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, otelMaxResponseBytes)) + _, _ = io.Copy(io.Discard, resp.Body) // drain the rest so the connection is reusable + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return 0, o.recordPartialSuccess(respBody, records), nil + } + mAnalyticsExportError(otelPublisherName, strconv.Itoa(resp.StatusCode), 1) + + if resp.StatusCode == http.StatusTooManyRequests { + return parseRetryAfter(resp.Header.Get("Retry-After")), 0, + fmt.Errorf("endpoint is rate limiting (429)") + } + if resp.StatusCode >= 500 { + return 0, 0, fmt.Errorf("endpoint returned status %d: %s", resp.StatusCode, otelResponseExcerpt(respBody)) + } + return 0, 0, &otelPermanentExportError{status: resp.StatusCode} +} + +// recordPartialSuccess counts the records the endpoint accepted the request for +// but rejected, and returns that count so the caller can exclude them from the +// published tally. Without this a 200 carrying rejectedLogRecords looks like a +// clean export, and the records are silently gone. +func (o *OTel) recordPartialSuccess(respBody []byte, records int) int { + if len(respBody) == 0 { + return 0 + } + var parsed otelExportResponse + if err := json.Unmarshal(respBody, &parsed); err != nil { + return 0 // a non-JSON 2xx body is not an error; nothing to report + } + // Proto3 JSON encodes int64 as a string, but some receivers emit a bare + // number, so the field is json.Number to accept either. + reported, err := parsed.PartialSuccess.RejectedLogRecords.Int64() + if err != nil || reported <= 0 { + return 0 + } + // The count is the endpoint's claim, and it cannot exceed what was sent. An + // over-report has to be clamped rather than trusted: the caller subtracts + // this from the batch size, and a Prometheus counter panics on a negative + // Add — which in the export worker would take the process down. + rejected := int(min(reported, int64(records))) + if reported > int64(records) { + slog.Warn("OTel endpoint reported more rejected records than were sent; clamping", + "reported", reported, "records", records) + } + o.dropRecords(dropReasonRejected, rejected) + slog.Error("OTel endpoint accepted the export but rejected records", + "rejected", rejected, "records", records, + "endpointMessage", parsed.PartialSuccess.ErrorMessage) + return rejected +} + +// sleep waits out the backoff before a retry, returning false if shutdown was +// requested first. delay is the endpoint's Retry-After when it supplied one, +// otherwise the jittered exponential backoff for this attempt. +func (o *OTel) sleep(delay time.Duration, attempt int) bool { + if delay <= 0 { + delay = o.backoff(attempt) + } + if delay <= 0 { + return true + } + t := time.NewTimer(delay) + defer t.Stop() + select { + case <-t.C: + return true + case <-o.stop: + return false + } +} + +// backoff returns the delay before the given retry attempt (1-based), growing +// exponentially with full jitter. Jitter matters because every replica retries +// against the same endpoint after a shared outage; without it they reconverge +// into a synchronized herd on the first recovery. +func (o *OTel) backoff(attempt int) time.Duration { + base := o.cfg.RetryBackoff + if base <= 0 { + base = time.Second + } + // Cap the exponent so a large max_retries cannot overflow the shift. + shift := attempt - 1 + if shift > 10 { + shift = 10 + } + // Cap BEFORE jittering, so the jitter still spans the full range below the + // ceiling. Capping afterwards would land every attempt at the cap exactly, + // re-synchronising the replicas the jitter exists to spread out. + // + // The <= 0 arm also covers the shift overflowing int64 into a negative + // duration, which needs an absurd retry_backoff (~104 days) but would + // otherwise skip the wait altogether rather than lengthen it. + delay := base << shift + if delay <= 0 || delay > otelMaxRetryBackoff { + delay = otelMaxRetryBackoff + } + if half := delay / 2; half > 0 { + delay = half + time.Duration(rand.Int64N(int64(half))) + } + return delay +} + +// dropRecords counts n records lost for the given reason, on both the local +// total and the labelled metric. +func (o *OTel) dropRecords(reason string, n int) { + if n <= 0 { + return + } + mAnalyticsDropped(otelPublisherName, reason, n) + o.countDrops(n) +} + +// countDrops adds n to the dropped total and returns the new total. It does not +// log: every caller has something more specific to say than "a record was lost". +func (o *OTel) countDrops(n int) int { + if n <= 0 { + return 0 + } + o.droppedMu.Lock() + o.dropped += n + total := o.dropped + o.droppedMu.Unlock() + return total +} + +// Metric helpers. +// +// Every analytics-publisher metric goes through these rather than touching the +// package vars directly. The vars are nil until metrics.Init() runs — main() +// calls it long before any publisher exists, but a constructor must not depend +// on that ordering, and guarding only in the constructor while the export path +// dereferences freely turns a startup panic into a first-request panic. + +func mAnalyticsPublished(publisher string, n int) { + if metrics.AnalyticsPublishedTotal != nil { + metrics.AnalyticsPublishedTotal.WithLabelValues(publisher).Add(float64(n)) + } +} + +func mAnalyticsDropped(publisher, reason string, n int) { + if metrics.AnalyticsDroppedTotal != nil { + metrics.AnalyticsDroppedTotal.WithLabelValues(publisher, reason).Add(float64(n)) + } +} + +func mAnalyticsQueueDepth(publisher string, depth int) { + if metrics.AnalyticsQueueDepth != nil { + metrics.AnalyticsQueueDepth.WithLabelValues(publisher).Set(float64(depth)) + } +} + +func mAnalyticsQueueCapacity(publisher string, capacity int) { + if metrics.AnalyticsQueueCapacity != nil { + metrics.AnalyticsQueueCapacity.WithLabelValues(publisher).Set(float64(capacity)) + } +} + +func mAnalyticsExportDuration(publisher string, seconds float64) { + if metrics.AnalyticsExportDurationSeconds != nil { + metrics.AnalyticsExportDurationSeconds.WithLabelValues(publisher).Observe(seconds) + } +} + +func mAnalyticsExportError(publisher, code string, n int) { + if metrics.AnalyticsExportErrorsTotal != nil { + metrics.AnalyticsExportErrorsTotal.WithLabelValues(publisher, code).Add(float64(n)) + } +} + +// initOTelMetrics materializes this publisher's counters at zero. +// +// A labelled Prometheus counter does not exist in the scrape until it is first +// incremented, so on a healthy gateway analytics_dropped_total is simply absent. +// That makes a dashboard panel read "No data" rather than 0, and leaves an +// operator unable to tell "nothing was dropped" from "the metrics path is +// broken" — an unacceptable ambiguity for the one series that makes silent +// analytics loss visible. +// +// Export-error codes are deliberately not pre-created: the label carries the +// HTTP status, which is unbounded, and materializing every possible status would +// be worse than the gap it closes. +func (o *OTel) initMetrics() { + mAnalyticsPublished(otelPublisherName, 0) + for _, reason := range []string{ + dropReasonQueueFull, dropReasonSendFailed, dropReasonBackpressure, + dropReasonRejected, dropReasonSerializeFailed, + } { + mAnalyticsDropped(otelPublisherName, reason, 0) + } + mAnalyticsExportError(otelPublisherName, errCodeTransport, 0) + mAnalyticsQueueCapacity(otelPublisherName, o.cfg.QueueCapacity) + mAnalyticsQueueDepth(otelPublisherName, 0) +} + +// otelResponseExcerpt renders a bounded, single-line excerpt of an endpoint's +// error body for the log. The body is the endpoint's own text, never ours. +func otelResponseExcerpt(body []byte) string { + const maxExcerpt = 256 + excerpt := strings.TrimSpace(string(body)) + if len(excerpt) > maxExcerpt { + excerpt = excerpt[:maxExcerpt] + "..." + } + return strings.ReplaceAll(excerpt, "\n", " ") +} + +// gzipBytes compresses the payload for Content-Encoding: gzip. +func gzipBytes(body []byte) ([]byte, error) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(body); err != nil { + zw.Close() + return nil, err + } + if err := zw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// buildRecord maps the canonical analytics event onto one OTLP log record. +// +// Naming follows stable OpenTelemetry conventions where they exist, the GenAI and +// MCP conventions (both Development-stability, and now maintained in +// open-telemetry/semantic-conventions-genai) for AI fields, and the wso2.* +// namespace where OpenTelemetry defines nothing — API-product concepts and cost, +// chiefly. +func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { + attrs := newOTelAttrs() + attrs.str("event.name", otelEventName) + + // Keep http.route as the route template and url.path as the literal client-requested path. + route := "" + if event.Operation != nil { + route = event.Operation.APIResourceTemplate + attrs.str("http.request.method", event.Operation.APIMethod) + attrs.str("http.route", route) + } + if route == "" && event.API != nil { + route = event.API.APIContext + } + // Query strings are omitted to avoid exposing tokens or API keys to third-party analytics. + attrs.anyStr("url.path", event.Properties[constants.RequestPathPropertyKey]) + attrs.i64NonZero("http.response.status_code", int64(event.ProxyResponseCode)) + attrs.str("client.address", event.UserIP) + attrs.str("user_agent.original", event.UserAgentHeader) + attrs.anyInt("http.request.body.size", event.Properties["requestSize"]) + attrs.anyInt("http.response.body.size", event.Properties["responseSize"]) + attrs.anyStr(ns("response.content_type"), event.Properties["responseContentType"]) + + // API identity. + if event.API != nil { + attrs.str(ns("api.id"), event.API.APIID) + attrs.str(ns("api.name"), event.API.APIName) + attrs.str(ns("api.version"), event.API.APIVersion) + attrs.str(ns("api.context"), event.API.APIContext) + attrs.str(ns("api.type"), event.API.APIType) + attrs.str(ns("api.subtype"), event.API.SubType) + attrs.str(ns("project.id"), event.API.ProjectID) + attrs.str(ns("organization.id"), event.API.OrganizationID) + attrs.str(ns("environment.id"), event.API.EnvironmentID) + } + + // Consumer identity. + attrs.anyStr("user.id", event.Properties[dto.PropKeyAuthUserID]) + attrs.str("user.name", event.UserName) + if event.Application != nil { + attrs.str(ns("application.id"), event.Application.ApplicationID) + attrs.str(ns("application.name"), event.Application.ApplicationName) + attrs.str(ns("application.owner"), event.Application.ApplicationOwner) + attrs.str(ns("application.key_type"), event.Application.KeyType) + } + if event.Subscription != nil { + attrs.str(ns("subscription.id"), event.Subscription.BillingSubscriptionID) + attrs.str(ns("subscription.customer.id"), event.Subscription.BillingCustomerID) + attrs.str(ns("subscription.status"), event.Subscription.Status) + attrs.str(ns("subscription.plan"), event.Subscription.PlanName) + } + if event.MetaInfo != nil { + attrs.str(ns("correlation.id"), event.MetaInfo.CorrelationID) + attrs.str(ns("gateway.type"), event.MetaInfo.GatewayType) + attrs.str(ns("region.id"), event.MetaInfo.RegionID) + } + + // Latency. OpenTelemetry has no log attribute for these. + if event.Latencies != nil { + attrs.i64(ns("latency.response_ms"), event.Latencies.ResponseLatency) + attrs.i64(ns("latency.backend_ms"), event.Latencies.BackendLatency) + attrs.i64(ns("latency.request_mediation_ms"), event.Latencies.RequestMediationLatency) + attrs.i64(ns("latency.response_mediation_ms"), event.Latencies.ResponseMediationLatency) + attrs.i64(ns("latency.duration_ms"), event.Latencies.Duration) + } + + // Upstream outcome. Destination is authority+path, so only its authority + // half is a server address. + if event.Target != nil { + attrs.str(ns("upstream.destination"), event.Target.Destination) + if authority, _, found := strings.Cut(event.Target.Destination, "/"); found || authority != "" { + if host, port, err := net.SplitHostPort(authority); err == nil { + attrs.str("server.address", host) + if p, convErr := strconv.Atoi(port); convErr == nil { + attrs.i64NonZero("server.port", int64(p)) + } + } else { + attrs.str("server.address", authority) + } + } + attrs.i64NonZero(ns("upstream.response.status_code"), int64(event.Target.TargetResponseCode)) + attrs.str(ns("upstream.response.detail"), event.Target.ResponseCodeDetail) + attrs.b(ns("cache.hit"), event.Target.ResponseCacheHit) + } + + // Faults. error.type is the one stable error attribute, and it carries the + // fault category derived in analytics.classifyFault — the same value existing + // Moesif consumers read from errorType. + attrs.str("error.type", event.ErrorType) + if event.Error != nil { + attrs.i64NonZero(ns("error.code"), int64(event.Error.ErrorCode)) + attrs.str(ns("error.message"), string(event.Error.ErrorMessage)) + } + + // Payloads, only present when body capture is enabled on the collector. + attrs.anyStr(ns("request.body"), event.Properties[dto.PropKeyRequestPayload]) + attrs.anyStr(ns("response.body"), event.Properties[dto.PropKeyResponsePayload]) + + appendHeaderAttributes(attrs, "http.request.header.", event.Properties[dto.PropKeyRequestHeaders]) + appendHeaderAttributes(attrs, "http.response.header.", event.Properties[dto.PropKeyResponseHeaders]) + + o.appendAIAttributes(event, attrs, route) + o.appendMCPAttributes(event, attrs) + + now := time.Now() + requestTime := event.RequestTimestamp + if requestTime.IsZero() { + requestTime = now + } + + body := "api.transaction" + if event.Operation != nil && route != "" { + body = event.Operation.APIMethod + " " + route + } + + return &otelLogRecord{ + TimeUnixNano: strconv.FormatInt(requestTime.UnixNano(), 10), + ObservedTimeUnixNano: strconv.FormatInt(now.UnixNano(), 10), + SeverityNumber: otelSeverityNumberInfo, + SeverityText: otelSeverityTextInfo, + EventName: otelEventName, + Body: otelAnyValue{StringValue: &body}, + Attributes: attrs.list(), + } +} + +// appendHeaderAttributes emits one attribute per header under the given prefix, +// per the HTTP conventions: the header name forms the key, and the value is +// always a string array because a header can repeat. +// +// The headers are already filtered by the operator’s allowlist policy. When +// the policy is absent, nothing is emitted. +// +// otelMaxHeaderAttributes prevents overly broad header allowlists from +// exceeding OTel’s attribute limit and causing silent truncation +func appendHeaderAttributes(attrs *otelAttrs, prefix string, raw interface{}) { + serialized, ok := raw.(string) + if !ok || serialized == "" { + return + } + var headers map[string]string + if err := json.Unmarshal([]byte(serialized), &headers); err != nil { + slog.Debug("OTel publisher could not parse analytics headers", "error", err, "prefix", prefix) + return + } + + // Sorted so a truncated record keeps the same headers across requests; map + // order would drop a different arbitrary subset every time. + names := make([]string, 0, len(headers)) + for name := range headers { + names = append(names, name) + } + sort.Strings(names) + + emitted := 0 + for _, name := range names { + // HTTP/2 pseudo-headers (:method, :path, :scheme, :authority, :status) + // are not headers: Envoy surfaces them alongside the real ones, they + // duplicate attributes already mapped from their own event fields + // (http.request.method, url.path, server.address, + // http.response.status_code), and ":path" can carry a query string. + // They also make an attribute key that starts with a colon. + if strings.HasPrefix(name, ":") { + continue + } + if emitted == otelMaxHeaderAttributes { + slog.Warn("OTel publisher truncated header attributes; narrow the analytics-header-filter allowlist", + "prefix", prefix, "emitted", emitted, "available", len(names)) + return + } + // The analytics policy joins a repeated header into one comma-separated + // string before it reaches the event, so the array carries that single + // value verbatim. + if value := headers[name]; value != "" { + attrs.strs(prefix+strings.ToLower(name), []string{value}) + emitted++ + } + } +} + +// appendAIAttributes maps the AI metadata the pipeline stashes in +// Event.Properties onto GenAI conventions. Cost has no GenAI equivalent. +func (o *OTel) appendAIAttributes(event *dto.Event, attrs *otelAttrs, route string) { + if event.Properties == nil { + return + } + + md, hasAIMetadata := event.Properties["aiMetadata"].(dto.AIMetadata) + if hasAIMetadata { + attrs.str("gen_ai.provider.name", otelGenAIProviderName(md.VendorName)) + attrs.str(ns("gen_ai.provider.template_name"), md.VendorName) + // aitoken:modelid is the response model when the provider returns one, + // falling back to the request model. + attrs.str("gen_ai.response.model", md.Model) + attrs.str(ns("gen_ai.provider.api_version"), md.VendorVersion) + switch cost := md.LLMCost.(type) { + case float64: + attrs.f64(ns("gen_ai.cost.total"), cost) + case string: + attrs.str(ns("gen_ai.cost.total"), cost) + } + + if op := otelGenAIOperationName(route); op != "" { + attrs.str("gen_ai.operation.name", op) + } + } + attrs.anyStr("gen_ai.request.model", event.Properties[constants.RequestModelPropertyKey]) + + if usage, ok := event.Properties["aiTokenUsage"].(dto.AITokenUsage); ok { + attrs.i64("gen_ai.usage.input_tokens", int64(usage.PromptToken)) + attrs.i64("gen_ai.usage.output_tokens", int64(usage.CompletionToken)) + attrs.i64(ns("gen_ai.usage.total_tokens"), int64(usage.TotalToken)) + } + + attrs.anyBool(ns("gen_ai.egress"), event.Properties["isEgress"]) + attrs.anyBool(ns("guardrail.hit"), event.Properties[constants.GuardrailHitMetadataKey]) + attrs.anyStr(ns("guardrail.name"), event.Properties[constants.GuardrailNameMetadataKey]) +} + +// appendMCPAttributes flattens Properties["mcpAnalytics"] onto MCP conventions. +// +// Known keys get a curated attribute name, almost always a standard one: +// jsonRpcMethod becomes mcp.method.name, not a mechanical transliteration. +// Anything NOT named below is then swept up under wso2.mcp.* +func (o *OTel) appendMCPAttributes(event *dto.Event, attrs *otelAttrs) { + mcp, ok := event.Properties["mcpAnalytics"].(map[string]interface{}) + if !ok { + return + } + + // Each source map has its own claimed-key set and its own take: reading a key + // marks it as having a curated attribute name, which excludes it from the + // sweep at the end — including a key deliberately not emitted. + mapped := map[string]bool{} + take := func(key string) interface{} { + mapped[key] = true + return mcp[key] + } + + attrs.anyStr("mcp.method.name", take("jsonRpcMethod")) + attrs.anyStr("mcp.session.id", take("sessionId")) + attrs.anyStr("jsonrpc.request.id", take("jsonRpcId")) + + // Emit tool/prompt names or resource URIs only for recognized capabilities; + // leave unrecognized fields for the sweep. + switch capability, _ := take("capability").(string); capability { + case "TOOL": + attrs.anyStr("gen_ai.tool.name", take("capabilityName")) + case "PROMPT": + attrs.anyStr("gen_ai.prompt.name", take("capabilityName")) + case "RESOURCE": + attrs.anyStr("mcp.resource.uri", take("resourceUri")) + } + + // A JSON-RPC error code is a string in rpc.response.status_code. + switch code := take("errorCode").(type) { + case int: + attrs.str("rpc.response.status_code", strconv.Itoa(code)) + case float64: + attrs.str("rpc.response.status_code", strconv.FormatInt(int64(code), 10)) + case string: + attrs.str("rpc.response.status_code", code) + } + if isError, ok := take("isError").(bool); ok && isError { + attrs.strIfEmpty("error.type", "mcp_error") + } + + // clientInfo and serverInfo arrive as sub-objects, and both carry "name" and + // "version" — a flat lookup could not tell a client's name from a server's. + client := otelNestedMap(take("clientInfo")) + clientMapped := map[string]bool{} + clientTake := func(key string) interface{} { + clientMapped[key] = true + return client[key] + } + attrs.anyStr(ns("mcp.client.name"), clientTake("name")) + attrs.anyStr(ns("mcp.client.version"), clientTake("version")) + attrs.anyStr(ns("mcp.client.requested_protocol_version"), clientTake("requestedProtocolVersion")) + + server := otelNestedMap(take("serverInfo")) + serverMapped := map[string]bool{} + serverTake := func(key string) interface{} { + serverMapped[key] = true + return server[key] + } + // The negotiated version, as opposed to the client's requested one above. + attrs.anyStr("mcp.protocol.version", serverTake("protocolVersion")) + attrs.anyStr(ns("mcp.server.name"), serverTake("name")) + attrs.anyStr(ns("mcp.server.version"), serverTake("version")) + + // Sweep every key with no curated name, so a key the analytics policy gains + // later appears under a plainly-custom name rather than being silently + // dropped until someone notices. + otelSweepUnclaimed(attrs, ns("mcp."), mcp, mapped) + otelSweepUnclaimed(attrs, ns("mcp.client."), client, clientMapped) + otelSweepUnclaimed(attrs, ns("mcp.server."), server, serverMapped) +} + +// otelNestedMap reads a sub-object out of an analytics property, returning nil +// when the value is absent or not an object. Reading a key from the nil result is +// safe, so no call site needs an absence check. +func otelNestedMap(value interface{}) map[string]interface{} { + nested, _ := value.(map[string]interface{}) + return nested +} + +// otelSweepUnclaimed emits every unclaimed key under prefix + the key in +// snake_case. Sorted so two identical records produce the same attribute order; +// a sub-object recurses with an extended prefix and nothing claimed. +func otelSweepUnclaimed(attrs *otelAttrs, prefix string, source map[string]interface{}, claimed map[string]bool) { + if len(source) == 0 { + return + } + keys := make([]string, 0, len(source)) + for key := range source { + if !claimed[key] { + keys = append(keys, key) + } + } + sort.Strings(keys) + + for _, key := range keys { + name := prefix + otelSnakeCase(key) + if nested, ok := source[key].(map[string]interface{}); ok { + otelSweepUnclaimed(attrs, name+".", nested, nil) + continue + } + attrs.anyScalar(name, source[key]) + } +} + +// otelSnakeCase converts a camelCase analytics key into the snake_case that +// OpenTelemetry attribute names use. +func otelSnakeCase(s string) string { + var out strings.Builder + out.Grow(len(s) + 4) + for i := 0; i < len(s); i++ { + c := s[i] + if c >= 'A' && c <= 'Z' { + if i > 0 && !(s[i-1] >= 'A' && s[i-1] <= 'Z') { + out.WriteByte('_') + } + out.WriteByte(c - 'A' + 'a') + continue + } + out.WriteByte(c) + } + return out.String() +} + +// otelGenAIProviderName maps a WSO2 LLM provider template name onto the +// gen_ai.provider.name enum. An unrecognised provider yields "" so the enum +// attribute is left unset rather than carrying a non-member value; the raw name +// is always kept in wso2.gen_ai.provider.template_name. +func otelGenAIProviderName(templateName string) string { + switch strings.ToLower(templateName) { + case "openai": + return "openai" + case "anthropic": + return "anthropic" + case "awsbedrock": + return "aws.bedrock" + case "azure-openai": + return "azure.ai.openai" + case "azureai-foundry": + return "azure.ai.inference" + case "gemini": + return "gcp.gemini" + case "mistralai": + return "mistral_ai" + default: + return "" + } +} + +// otelGenAIOperationName derives the gen_ai.operation.name enum value from the +// route. "" when the route is not a recognised inference endpoint. +func otelGenAIOperationName(route string) string { + r := strings.ToLower(route) + switch { + case strings.Contains(r, "/chat/completions"), strings.Contains(r, "/messages"): + return "chat" + case strings.Contains(r, "/embeddings"): + return "embeddings" + case strings.Contains(r, "/generatecontent"): + return "generate_content" + case strings.Contains(r, "/completions"): + return "text_completion" + default: + return "" + } +} + +// --- OTLP/HTTP JSON wire types ----------------------------------------------- +// +// The proto3 JSON mapping of the OTLP protobufs: field names are lowerCamelCase +// and 64-bit integers are encoded as JSON strings. + +type otelExportRequest struct { + ResourceLogs []otelResourceLogs `json:"resourceLogs"` +} + +type otelResourceLogs struct { + Resource otelResource `json:"resource"` + ScopeLogs []otelScopeLogs `json:"scopeLogs"` +} + +type otelResource struct { + Attributes []otelKeyValue `json:"attributes"` +} + +type otelScopeLogs struct { + Scope otelScope `json:"scope"` + LogRecords []otelLogRecord `json:"logRecords"` +} + +type otelScope struct { + Name string `json:"name"` +} + +type otelLogRecord struct { + TimeUnixNano string `json:"timeUnixNano"` + ObservedTimeUnixNano string `json:"observedTimeUnixNano"` + SeverityNumber int `json:"severityNumber"` + SeverityText string `json:"severityText"` + EventName string `json:"eventName,omitempty"` + Body otelAnyValue `json:"body"` + Attributes []otelKeyValue `json:"attributes"` +} + +// otelExportResponse is the OTLP ExportLogsServiceResponse. A 2xx can still +// report records the endpoint refused, which is otherwise indistinguishable from +// a clean export. +type otelExportResponse struct { + PartialSuccess otelPartialSuccess `json:"partialSuccess"` +} + +type otelPartialSuccess struct { + // json.Number because proto3 JSON encodes int64 as a string while some + // receivers emit a bare number; either must parse. + RejectedLogRecords json.Number `json:"rejectedLogRecords"` + ErrorMessage string `json:"errorMessage"` +} + +type otelKeyValue struct { + Key string `json:"key"` + Value otelAnyValue `json:"value"` +} + +// otelAnyValue is the OTLP AnyValue union; exactly one field is set. +type otelAnyValue struct { + StringValue *string `json:"stringValue,omitempty"` + IntValue *string `json:"intValue,omitempty"` + DoubleValue *float64 `json:"doubleValue,omitempty"` + BoolValue *bool `json:"boolValue,omitempty"` + ArrayValue *otelArrayValue `json:"arrayValue,omitempty"` +} + +// otelArrayValue is the OTLP ArrayValue: a list of AnyValue. Required for the +// header conventions, whose values are always string arrays because a header can +// legitimately repeat. +type otelArrayValue struct { + Values []otelAnyValue `json:"values"` +} + +// otelAttrs accumulates the attributes of one record. +// +// A string attribute is omitted when empty: OpenTelemetry discourages +// empty-string attributes and nothing in this mapping means anything by "". +// +// Zero and false values are preserved as valid measurements; only values +// explicitly marked with `i64NonZero` are omitted when zero means “no value.” +type otelAttrs struct { + kvs []otelKeyValue +} + +func newOTelAttrs() *otelAttrs { + return &otelAttrs{kvs: make([]otelKeyValue, 0, 48)} +} + +// str sets a string attribute, omitting it when empty. Deliberately unlike +// i64/f64/b below — see the type comment. +func (a *otelAttrs) str(key, value string) *otelAttrs { + if value == "" { + return a + } + v := value + a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{StringValue: &v}}) + return a +} + +// strs sets a string-array attribute, skipping the attribute entirely when there +// is nothing to put in it. +func (a *otelAttrs) strs(key string, values []string) *otelAttrs { + if len(values) == 0 { + return a + } + items := make([]otelAnyValue, 0, len(values)) + for _, value := range values { + v := value + items = append(items, otelAnyValue{StringValue: &v}) + } + a.kvs = append(a.kvs, otelKeyValue{ + Key: key, + Value: otelAnyValue{ArrayValue: &otelArrayValue{Values: items}}, + }) + return a +} + +// strIfEmpty sets key only when it is not already present. +func (a *otelAttrs) strIfEmpty(key, value string) *otelAttrs { + for _, kv := range a.kvs { + if kv.Key == key { + return a + } + } + return a.str(key, value) +} + +// i64 sets an integer attribute, including when the value is 0. +func (a *otelAttrs) i64(key string, value int64) *otelAttrs { + v := strconv.FormatInt(value, 10) + a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{IntValue: &v}}) + return a +} + +// i64NonZero sets an integer attribute only when it is non-zero, for the fields +// where 0 is a sentinel for "no value" rather than a measurement: there is no +// HTTP status 0, no TCP port 0, and no error code 0. Every other integer +// attribute uses i64 so a real zero survives. +func (a *otelAttrs) i64NonZero(key string, value int64) *otelAttrs { + if value == 0 { + return a + } + return a.i64(key, value) +} + +// f64 sets a floating-point attribute, including when the value is 0. A zero +// cost — a free model, a cached completion, a blocked request — is a fact. +func (a *otelAttrs) f64(key string, value float64) *otelAttrs { + v := value + a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{DoubleValue: &v}}) + return a +} + +// b sets a boolean attribute, including when false. A false cache.hit is a cache +// miss; omitting it would make a miss indistinguishable from an API with no cache +// filter at all, which is exactly what makes a hit-ratio query unanswerable. +func (a *otelAttrs) b(key string, value bool) *otelAttrs { + v := value + a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{BoolValue: &v}}) + return a +} + +// anyStr / anyInt / anyBool accept the interface{} values held in +// Event.Properties, whose concrete types vary by producer. +func (a *otelAttrs) anyStr(key string, value interface{}) *otelAttrs { + if s, ok := value.(string); ok { + return a.str(key, s) + } + return a +} + +func (a *otelAttrs) anyInt(key string, value interface{}) *otelAttrs { + switch v := value.(type) { + case int: + return a.i64(key, int64(v)) + case int32: + return a.i64(key, int64(v)) + case int64: + return a.i64(key, v) + case uint32: + return a.i64(key, int64(v)) + case uint64: + return a.i64(key, int64(v)) + case float64: + return a.i64(key, int64(v)) + } + return a +} + +func (a *otelAttrs) anyBool(key string, value interface{}) *otelAttrs { + switch v := value.(type) { + case bool: + return a.b(key, v) + case string: + if parsed, err := strconv.ParseBool(v); err == nil { + return a.b(key, parsed) + } + } + return a +} + +// anyScalar emits a value of unknown concrete type, choosing the OTLP value kind +// from it. Used by the unmapped-key sweep, where the type is whatever the +// analytics policy put in the map. +func (a *otelAttrs) anyScalar(key string, value interface{}) *otelAttrs { + switch v := value.(type) { + case string: + return a.str(key, v) + case bool: + return a.b(key, v) + case float64: + // JSON has one number type, so an integral value must not be reported as + // a double: a consumer summing "3.0" and 3 does not always get the same + // answer, and backends type the column from the first value they see. + if v == float64(int64(v)) { + return a.i64(key, int64(v)) + } + return a.f64(key, v) + case int, int32, int64, uint32, uint64: + return a.anyInt(key, v) + } + return a +} + +func (a *otelAttrs) list() []otelKeyValue { + return a.kvs +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go new file mode 100644 index 0000000000..8ad5757b87 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -0,0 +1,2166 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "compress/gzip" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/constants" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/metrics" +) + +// Attribute names are written as literals throughout this file: they are the +// wire contract, and sharing a constant with the publisher would let a typo +// satisfy its own assertion. + +func testOTelConfig(endpoint string) config.OTelPublisherConfig { + return config.OTelPublisherConfig{ + Endpoint: endpoint, + ServiceName: "gateway-runtime", + BatchSize: 100, + FlushInterval: 50 * time.Millisecond, + QueueCapacity: 100, + OnQueueFull: config.QueueDropNew, + Timeout: 2 * time.Second, + } +} + +func restEvent() *dto.Event { + api := &dto.ExtendedAPI{ + OrganizationID: "org-1", + ProjectID: "default", + EnvironmentID: "env-1", + APIContext: "/petstore", + } + api.APIID = "api-1" + api.APIName = "PetStore" + api.APIVersion = "v1.0" + api.APIType = "RestApi" + api.SubType = "RestApi" + + return &dto.Event{ + API: api, + Operation: &dto.Operation{APIMethod: "GET", APIResourceTemplate: "/petstore/pet/{petId}"}, + Target: &dto.Target{TargetResponseCode: 200, Destination: "backend:8080/pet/1", ResponseCodeDetail: "via_upstream", ResponseCacheHit: true}, + Application: &dto.Application{ApplicationID: "app-1", ApplicationName: "Web", ApplicationOwner: "alice", KeyType: "PRODUCTION"}, + Subscription: &dto.Subscription{BillingSubscriptionID: "sub-1", BillingCustomerID: "cust-1", Status: "ACTIVE", PlanName: "Gold"}, + Latencies: &dto.Latencies{ResponseLatency: 12, BackendLatency: 9, RequestMediationLatency: 2, ResponseMediationLatency: 1, Duration: 12}, + MetaInfo: &dto.MetaInfo{CorrelationID: "corr-1", GatewayType: "Envoy", RegionID: "us-east"}, + ProxyResponseCode: 200, + RequestTimestamp: time.Unix(1788320847, 0), + UserAgentHeader: "curl/8.7.1", + UserName: "alice", + UserIP: "10.0.0.5", + Properties: map[string]interface{}{ + dto.PropKeyAuthUserID: "user-1", + "requestSize": uint64(12), + // The concrete path, as analytics.go now supplies it: the route + // template is "/petstore/pet/{petId}", this is one request against it. + constants.RequestPathPropertyKey: "/petstore/pet/12345", + "responseSize": uint64(463), + "responseContentType": "application/json", + }, + } +} + +// attrMap flattens a record's attributes for assertion. Each OTLP AnyValue keeps +// exactly one field set, so the concrete type is asserted alongside the value. +func attrMap(t *testing.T, record *otelLogRecord) map[string]interface{} { + t.Helper() + out := map[string]interface{}{} + for _, kv := range record.Attributes { + if _, dup := out[kv.Key]; dup { + t.Fatalf("attribute %q emitted twice", kv.Key) + } + switch { + case kv.Value.StringValue != nil: + out[kv.Key] = *kv.Value.StringValue + case kv.Value.IntValue != nil: + out[kv.Key] = *kv.Value.IntValue + case kv.Value.DoubleValue != nil: + out[kv.Key] = *kv.Value.DoubleValue + case kv.Value.BoolValue != nil: + out[kv.Key] = *kv.Value.BoolValue + case kv.Value.ArrayValue != nil: + // Header attributes are string arrays; flatten to []string so a test + // can assert on them directly. + items := make([]string, 0, len(kv.Value.ArrayValue.Values)) + for _, item := range kv.Value.ArrayValue.Values { + if item.StringValue == nil { + t.Fatalf("attribute %q has a non-string array element", kv.Key) + } + items = append(items, *item.StringValue) + } + out[kv.Key] = items + default: + t.Fatalf("attribute %q has no value set", kv.Key) + } + } + return out +} + +func TestBuildRecordRestAPI(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + record := o.buildRecord(restEvent()) + + if record.EventName != "wso2.api.transaction" { + t.Errorf("eventName = %q, want wso2.api.transaction", record.EventName) + } + if record.SeverityNumber != 9 || record.SeverityText != "INFO" { + t.Errorf("severity = %d/%q, want 9/INFO", record.SeverityNumber, record.SeverityText) + } + // Nanoseconds, as a JSON string per the proto3 mapping. + if record.TimeUnixNano != "1788320847000000000" { + t.Errorf("timeUnixNano = %q", record.TimeUnixNano) + } + + got := attrMap(t, record) + want := map[string]interface{}{ + "event.name": "wso2.api.transaction", + "http.request.method": "GET", + "http.route": "/petstore/pet/{petId}", + // The template groups requests; the path identifies one. Never equal. + "url.path": "/petstore/pet/12345", + "http.response.status_code": "200", + "client.address": "10.0.0.5", + "user_agent.original": "curl/8.7.1", + "http.request.body.size": "12", + "http.response.body.size": "463", + "user.id": "user-1", + "user.name": "alice", + "error.type": nil, + "server.address": "backend", + "server.port": "8080", + "wso2.upstream.destination": "backend:8080/pet/1", + "wso2.upstream.response.status_code": "200", + "wso2.upstream.response.detail": "via_upstream", + "wso2.cache.hit": true, + "wso2.response.content_type": "application/json", + "wso2.api.id": "api-1", + "wso2.api.name": "PetStore", + "wso2.api.version": "v1.0", + "wso2.api.context": "/petstore", + "wso2.api.type": "RestApi", + "wso2.api.subtype": "RestApi", + "wso2.project.id": "default", + "wso2.organization.id": "org-1", + "wso2.environment.id": "env-1", + "wso2.application.id": "app-1", + "wso2.application.name": "Web", + "wso2.application.owner": "alice", + "wso2.application.key_type": "PRODUCTION", + "wso2.subscription.id": "sub-1", + "wso2.subscription.customer.id": "cust-1", + "wso2.subscription.status": "ACTIVE", + "wso2.subscription.plan": "Gold", + "wso2.correlation.id": "corr-1", + "wso2.gateway.type": "Envoy", + "wso2.region.id": "us-east", + "wso2.latency.response_ms": "12", + "wso2.latency.backend_ms": "9", + "wso2.latency.request_mediation_ms": "2", + "wso2.latency.response_mediation_ms": "1", + "wso2.latency.duration_ms": "12", + } + for key, expected := range want { + actual, present := got[key] + if expected == nil { + if present { + t.Errorf("%s should be absent, got %v", key, actual) + } + continue + } + if !present { + t.Errorf("%s missing", key) + continue + } + if actual != expected { + t.Errorf("%s = %v (%T), want %v (%T)", key, actual, actual, expected, expected) + } + } +} + +// The resource template already carries the API context; concatenating the two +// produced "/ctx/ctx/resource" in an earlier revision. +func TestBuildRecordDoesNotDuplicateContext(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(restEvent())) + if got["http.route"] != "/petstore/pet/{petId}" { + t.Errorf("http.route = %v, want /petstore/pet/{petId}", got["http.route"]) + } +} + +// No resource template means no route to report — but the client still asked for +// something, and url.path is the only record of what. This is the request that +// most needs a path: nothing matched, so triage has the template nowhere else. +// url.path deliberately does NOT fall back to the API context, which would put a +// value in the attribute that was never the requested path. +func TestBuildRecordWithNoRouteKeepsConcretePath(t *testing.T) { + event := restEvent() + event.Operation.APIResourceTemplate = "" + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if got["url.path"] != "/petstore/pet/12345" { + t.Errorf("url.path = %v, want /petstore/pet/12345", got["url.path"]) + } + if _, present := got["http.route"]; present { + t.Error("http.route should be absent when there is no resource template") + } +} + +// Absent rather than guessed: with no path on the event there is nothing +// truthful to put in url.path, and the API context is not the requested path. +func TestBuildRecordOmitsURLPathWhenUnavailable(t *testing.T) { + event := restEvent() + delete(event.Properties, constants.RequestPathPropertyKey) + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if actual, present := got["url.path"]; present { + t.Errorf("url.path = %v; it must be absent, not fall back to the context", actual) + } +} + +func TestBuildRecordFaults(t *testing.T) { + event := restEvent() + event.ProxyResponseCode = 401 + event.ErrorType = "AUTH" + event.Error = &dto.Error{ErrorCode: 900901, ErrorMessage: dto.AuthenticationFailure} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, expected := range map[string]interface{}{ + "error.type": "AUTH", + "wso2.error.code": "900901", + "wso2.error.message": "AUTHENTICATION_FAILURE", + "http.response.status_code": "401", + } { + if got[key] != expected { + t.Errorf("%s = %v, want %v", key, got[key], expected) + } + } +} + +// error.type carries the fault category derived by analytics.classifyFault — the +// same value existing Moesif consumers read from errorType. +func TestBuildRecordFaultCategoryInErrorType(t *testing.T) { + event := restEvent() + event.ErrorType = string(dto.FaultCategoryTargetConnectivity) + event.Error = &dto.Error{ErrorCode: 504, ErrorMessage: dto.TargetConnectivityConnectionTimeout} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, expected := range map[string]interface{}{ + "error.type": "TARGET_CONNECTIVITY", + "wso2.error.code": "504", + "wso2.error.message": "CONNECTION_TIMEOUT", + } { + if got[key] != expected { + t.Errorf("%s = %v, want %v", key, got[key], expected) + } + } + // The category/event-category enums belong to the in-development fault flow. + for _, absent := range []string{"wso2.event.category", "wso2.error.category", "wso2.error.sub_category"} { + if _, present := got[absent]; present { + t.Errorf("%s must not be emitted", absent) + } + } +} + +// A request that was not a gateway fault carries no error attributes at all — +// absence is what lets a consumer filter faults. +func TestBuildRecordSuccessOmitsErrorAttributes(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(restEvent())) + + for _, key := range []string{"error.type", "wso2.error.code", "wso2.error.message"} { + if _, present := got[key]; present { + t.Errorf("%s is present on a successful record", key) + } + } +} + +// gen_ai.operation.name is the only AI attribute derived from the route rather +// than from a property the AI pipeline wrote, and otelGenAIOperationName +// substring-matches "/messages" and "/completions". Without a gate, ordinary REST +// routes containing those words emit a GenAI attribute and pollute GenAI +// dashboards with traffic that never reached a model. +func TestBuildRecordNonAIRouteOmitsGenAIOperation(t *testing.T) { + for _, route := range []string{ + "/notify/messages", // contains "/messages" + "/billing/completions", // contains "/completions" + "/v1/chat/completions", // the real LLM shape, on a plain REST API + "/inbox/messages/{id}", + } { + t.Run(route, func(t *testing.T) { + event := restEvent() + event.API.APIType = "RestApi" + event.Operation.APIResourceTemplate = route + // No aiMetadata: nothing in this request went near a model. + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if _, present := got["gen_ai.operation.name"]; present { + t.Fatalf("gen_ai.operation.name emitted for non-AI route %q (value %v)", + route, got["gen_ai.operation.name"]) + } + for _, k := range []string{"gen_ai.provider.name", "gen_ai.request.model", + "gen_ai.response.model", "gen_ai.usage.input_tokens"} { + if _, present := got[k]; present { + t.Errorf("%s emitted for a non-AI request", k) + } + } + }) + } +} + +// The gate keys off aiMetadata, so an AI request still gets the attribute — and +// still gets it for each recognised operation shape. +func TestBuildRecordAIRouteKeepsGenAIOperation(t *testing.T) { + for route, want := range map[string]string{ + "/v1/chat/completions": "chat", + "/v1/messages": "chat", + "/v1/embeddings": "embeddings", + } { + t.Run(route, func(t *testing.T) { + event := restEvent() + event.API.APIType = "LlmProxy" + event.Operation.APIResourceTemplate = route + event.Properties["aiMetadata"] = dto.AIMetadata{VendorName: "openai", Model: "gpt-4o"} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["gen_ai.operation.name"] != want { + t.Fatalf("gen_ai.operation.name = %v, want %q", got["gen_ai.operation.name"], want) + } + }) + } +} + +func TestBuildRecordGenAI(t *testing.T) { + event := restEvent() + event.API.APIType = "LlmProxy" + event.Operation.APIResourceTemplate = "/ai/chat/completions" + event.Properties["aiMetadata"] = dto.AIMetadata{ + Model: "claude-opus-4", + VendorName: "awsbedrock", + VendorVersion: "2024-10-01", + LLMCost: 0.0421, + } + event.Properties["aiTokenUsage"] = dto.AITokenUsage{PromptToken: 1841, CompletionToken: 210, TotalToken: 2051} + event.Properties[constants.RequestModelPropertyKey] = "claude-opus-4-20260101" + event.Properties[constants.GuardrailHitMetadataKey] = true + event.Properties[constants.GuardrailNameMetadataKey] = "pii-masking" + event.Properties["isEgress"] = true + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, expected := range map[string]interface{}{ + // awsbedrock is not a gen_ai.provider.name enum member; aws.bedrock is. + "gen_ai.provider.name": "aws.bedrock", + "wso2.gen_ai.provider.template_name": "awsbedrock", + // aitoken:modelid resolves to the response model. + "gen_ai.response.model": "claude-opus-4", + "gen_ai.request.model": "claude-opus-4-20260101", + "gen_ai.operation.name": "chat", + "gen_ai.usage.input_tokens": "1841", + "gen_ai.usage.output_tokens": "210", + "wso2.gen_ai.usage.total_tokens": "2051", + "wso2.gen_ai.cost.total": 0.0421, + "wso2.gen_ai.provider.api_version": "2024-10-01", + "wso2.guardrail.hit": true, + "wso2.guardrail.name": "pii-masking", + "wso2.gen_ai.egress": true, + } { + if got[key] != expected { + t.Errorf("%s = %v (%T), want %v (%T)", key, got[key], got[key], expected, expected) + } + } +} + +func TestBuildRecordMCP(t *testing.T) { + event := restEvent() + event.API.APIType = "Mcp" + // Nested exactly as the analytics policy serializes it: clientInfo and + // serverInfo are sub-objects, and both carry "name" and "version". A flat + // fixture here is what let the nested-lookup bug pass for so long. + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "jsonRpcMethod": "tools/call", + "jsonRpcId": "7", + "sessionId": "sess-1", + "capability": "TOOL", + "capabilityName": "search_docs", + "errorCode": -32602, + "isError": true, + "clientInfo": map[string]interface{}{ + "name": "claude-desktop", + "version": "1.2.0", + "requestedProtocolVersion": "2025-03-26", + }, + "serverInfo": map[string]interface{}{ + "protocolVersion": "2025-06-18", + "name": "everything-server", + "version": "0.9.1", + }, + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, expected := range map[string]interface{}{ + "mcp.method.name": "tools/call", + "jsonrpc.request.id": "7", + "mcp.session.id": "sess-1", + "gen_ai.tool.name": "search_docs", + "rpc.response.status_code": "-32602", + "mcp.protocol.version": "2025-06-18", + "wso2.mcp.client.requested_protocol_version": "2025-03-26", + "wso2.mcp.client.name": "claude-desktop", + "wso2.mcp.client.version": "1.2.0", + "wso2.mcp.server.name": "everything-server", + "wso2.mcp.server.version": "0.9.1", + } { + if got[key] != expected { + t.Errorf("%s = %v, want %v", key, got[key], expected) + } + } + // The capability decides which attribute the name lands on. + for _, absent := range []string{"mcp.resource.uri", "gen_ai.prompt.name"} { + if _, present := got[absent]; present { + t.Errorf("%s should be absent for a TOOL capability", absent) + } + } + // event.ErrorType is empty here, so the MCP error supplies error.type. + if got["error.type"] != "mcp_error" { + t.Errorf("error.type = %v, want mcp_error", got["error.type"]) + } +} + +// A gateway-level ErrorType must not be overwritten by the MCP fallback. +func TestBuildRecordMCPKeepsGatewayErrorType(t *testing.T) { + event := restEvent() + event.ErrorType = "THROTTLED" + event.Properties["mcpAnalytics"] = map[string]interface{}{"isError": true} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if got["error.type"] != "THROTTLED" { + t.Errorf("error.type = %v, want THROTTLED", got["error.type"]) + } +} + +// Each capability's target goes on its own attribute, read from its own source +// field: tools and prompts are named at params.name, a resource is addressed by +// URI at params.uri. +func TestMCPCapabilityRouting(t *testing.T) { + cases := []struct { + capability string + mcp map[string]interface{} + wantKey string + wantValue string + }{ + {"TOOL", map[string]interface{}{"capabilityName": "search_docs"}, + "gen_ai.tool.name", "search_docs"}, + {"PROMPT", map[string]interface{}{"capabilityName": "summarize"}, + "gen_ai.prompt.name", "summarize"}, + {"RESOURCE", map[string]interface{}{"resourceUri": "file:///docs/readme.md"}, + "mcp.resource.uri", "file:///docs/readme.md"}, + } + for _, tc := range cases { + t.Run(tc.capability, func(t *testing.T) { + event := restEvent() + tc.mcp["capability"] = tc.capability + event.Properties["mcpAnalytics"] = tc.mcp + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if got[tc.wantKey] != tc.wantValue { + t.Errorf("%s = %v, want %v", tc.wantKey, got[tc.wantKey], tc.wantValue) + } + }) + } +} + +// mcp.resource.uri must come from resourceUri only. Reading capabilityName here +// is the bug this replaced: params.name does not exist on a resources/read +// request, so the attribute was always absent. +func TestMCPResourceURIIgnoresCapabilityName(t *testing.T) { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "capability": "RESOURCE", + "capabilityName": "should-not-be-used", + "resourceUri": "file:///docs/readme.md", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["mcp.resource.uri"] != "file:///docs/readme.md" { + t.Errorf("mcp.resource.uri = %v, want the resourceUri value", got["mcp.resource.uri"]) + } + // A resource has no name, so neither name attribute may appear. + for _, key := range []string{"gen_ai.tool.name", "gen_ai.prompt.name"} { + if _, present := got[key]; present { + t.Errorf("%s is present on a resource read", key) + } + } +} + +// End-to-end over the real HTTP path: the payload a collector receives must be +// valid OTLP-JSON with the routing scope and the configured headers. +func TestExportPayloadAndHeaders(t *testing.T) { + var ( + mu sync.Mutex + body []byte + header http.Header + ) + done := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + body, _ = io.ReadAll(r.Body) + header = r.Header.Clone() + mu.Unlock() + w.WriteHeader(http.StatusOK) + close(done) + })) + defer server.Close() + + cfg := testOTelConfig(server.URL + "/v1/logs") + cfg.ServiceVersion = "1.2.0" + cfg.Headers = map[string]string{"X-Api-Key": "secret"} + cfg.ResourceAttributes = map[string]string{"deployment.environment.name": "test"} + + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + publisher.Publish(restEvent()) + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("no export received") + } + if err := publisher.Close(context.Background()); err != nil { + t.Errorf("Close: %v", err) + } + + mu.Lock() + defer mu.Unlock() + + if got := header.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q", got) + } + if got := header.Get("X-Api-Key"); got != "secret" { + t.Errorf("configured header not sent, got %q", got) + } + + var payload otelExportRequest + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("payload is not valid JSON: %v", err) + } + if len(payload.ResourceLogs) != 1 || len(payload.ResourceLogs[0].ScopeLogs) != 1 { + t.Fatalf("unexpected payload shape: %s", body) + } + scope := payload.ResourceLogs[0].ScopeLogs[0] + if scope.Scope.Name != "wso2.analytics" { + t.Errorf("scope.name = %q, want wso2.analytics — the collector routes on this", scope.Scope.Name) + } + if len(scope.LogRecords) != 1 { + t.Fatalf("want 1 record, got %d", len(scope.LogRecords)) + } + + resource := map[string]string{} + for _, kv := range payload.ResourceLogs[0].Resource.Attributes { + if kv.Value.StringValue != nil { + resource[kv.Key] = *kv.Value.StringValue + } + } + for key, want := range map[string]string{ + "service.name": "gateway-runtime", + "service.version": "1.2.0", + "deployment.environment.name": "test", + } { + if resource[key] != want { + t.Errorf("resource %s = %q, want %q", key, resource[key], want) + } + } +} + +// correlatedEvent returns the REST fixture with a distinguishable correlation id, +// so a queue's surviving record can be identified. +func correlatedEvent(id string) *dto.Event { + event := restEvent() + event.MetaInfo.CorrelationID = id + return event +} + +// newTestOTel builds a publisher with no worker goroutine, mirroring the derived +// fields NewOTel computes. NewOTel cannot be used where a test drives export or +// the queue directly, because it starts the worker that would drain them. +func newTestOTel(t *testing.T, cfg config.OTelPublisherConfig) *OTel { + t.Helper() + return &OTel{ + cfg: cfg, + client: &http.Client{Timeout: cfg.Timeout}, + queue: make(chan *otelLogRecord, cfg.QueueCapacity), + stop: make(chan struct{}), + workerDone: make(chan struct{}), + dropOldest: strings.EqualFold(cfg.OnQueueFull, config.QueueDropOldest), + gzip: strings.EqualFold(cfg.Compression, config.OTelCompressionGzip), + retryAbortDepth: cfg.EffectiveRetryAbortDepth(), + } +} + +// newUndrainedOTel builds a publisher whose queue nothing consumes, so Publish +// sees it full. +func newUndrainedOTel(t *testing.T, capacity int, onQueueFull string) *OTel { + t.Helper() + cfg := testOTelConfig("http://127.0.0.1:1/v1/logs") + cfg.QueueCapacity = capacity + cfg.BatchSize = capacity + cfg.OnQueueFull = onQueueFull + return newTestOTel(t, cfg) +} + +func (o *OTel) droppedCount() int { + o.droppedMu.Lock() + defer o.droppedMu.Unlock() + return o.dropped +} + +// A full queue must drop rather than block the ALS ingest path — under either +// policy, and Publish must never block. +func TestPublishDropsWhenQueueFull(t *testing.T) { + for _, policy := range []string{config.QueueDropNew, config.QueueDropOldest} { + t.Run(policy, func(t *testing.T) { + o := newUndrainedOTel(t, 1, policy) + for i := 0; i < 10; i++ { + o.Publish(restEvent()) + } + + if dropped := o.droppedCount(); dropped != 9 { + t.Errorf("dropped = %d, want 9 (queue holds 1)", dropped) + } + if len(o.queue) != 1 { + t.Errorf("queue holds %d records, want 1", len(o.queue)) + } + }) + } +} + +// The policy decides *which* record survives, which is the whole point of the +// setting: drop_new keeps the oldest queued record, drop_oldest keeps the newest. +func TestPublishQueueFullKeepsPolicysRecord(t *testing.T) { + cases := []struct { + policy string + survivor string + }{ + {config.QueueDropNew, "first"}, + {config.QueueDropOldest, "last"}, + } + for _, tc := range cases { + t.Run(tc.policy, func(t *testing.T) { + o := newUndrainedOTel(t, 1, tc.policy) + o.Publish(correlatedEvent("first")) + o.Publish(correlatedEvent("middle")) + o.Publish(correlatedEvent("last")) + + if len(o.queue) != 1 { + t.Fatalf("queue holds %d records, want 1", len(o.queue)) + } + got := attrMap(t, <-o.queue)["wso2.correlation.id"] + if got != tc.survivor { + t.Errorf("surviving record = %v, want %q", got, tc.survivor) + } + }) + } +} + +// drop_oldest evicts exactly once per Publish. A retry loop would spin while +// producers keep the queue full, turning a non-blocking Publish into a blocking +// one; a single eviction bounds the work per call. +func TestPublishDropOldestEvictsOncePerCall(t *testing.T) { + o := newUndrainedOTel(t, 2, config.QueueDropOldest) + o.Publish(correlatedEvent("a")) + o.Publish(correlatedEvent("b")) + o.Publish(correlatedEvent("c")) // evicts "a", enqueues "c" + + if len(o.queue) != 2 { + t.Fatalf("queue holds %d records, want 2", len(o.queue)) + } + var got []interface{} + for len(o.queue) > 0 { + got = append(got, attrMap(t, <-o.queue)["wso2.correlation.id"]) + } + if len(got) != 2 || got[0] != "b" || got[1] != "c" { + t.Errorf("queue = %v, want [b c]", got) + } +} + +func TestCloseIsIdempotent(t *testing.T) { + cfg := testOTelConfig("http://127.0.0.1:1/v1/logs") + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + if err := publisher.Close(context.Background()); err != nil { + t.Errorf("first Close: %v", err) + } + if err := publisher.Close(context.Background()); err != nil { + t.Errorf("second Close: %v", err) + } +} + +func TestNewOTelNilConfig(t *testing.T) { + publisher, err := NewOTel(nil) + if err == nil { + t.Error("NewOTel(nil) should return an error") + } + if publisher != nil { + t.Error("NewOTel(nil) should return a nil publisher") + } +} + +// --- TLS ------------------------------------------------------------------- + +// writeSelfSignedPair writes a throwaway self-signed certificate and its key, +// usable both as a CA bundle and as an mTLS client pair. +func writeSelfSignedPair(t *testing.T) (certPath, keyPath string) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "otel-publisher-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + + dir := t.TempDir() + certPath = filepath.Join(dir, "cert.pem") + keyPath = filepath.Join(dir, "key.pem") + writePEM(t, certPath, "CERTIFICATE", der) + writePEM(t, keyPath, "EC PRIVATE KEY", keyDER) + return certPath, keyPath +} + +func writePEM(t *testing.T, path, blockType string, der []byte) { + t.Helper() + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der}), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func TestBuildOTelTLSConfigDefaults(t *testing.T) { + got, err := buildOTelTLSConfig(config.OTelTLSConfig{}) + if err != nil { + t.Fatalf("buildOTelTLSConfig: %v", err) + } + if got.MinVersion != tls.VersionTLS12 { + t.Errorf("MinVersion = %x, want TLS 1.2", got.MinVersion) + } + // The hybrid must be offered first, with classical curves retained after it + // so an endpoint without the hybrid still handshakes. + if got.CurvePreferences[0] != tls.X25519MLKEM768 { + t.Errorf("CurvePreferences[0] = %v, want X25519MLKEM768", got.CurvePreferences[0]) + } + if len(got.CurvePreferences) < 2 { + t.Error("no classical curve retained after the hybrid") + } + if got.InsecureSkipVerify { + t.Error("InsecureSkipVerify must default to false") + } + if got.RootCAs != nil || len(got.Certificates) != 0 { + t.Error("no TLS material configured, yet RootCAs/Certificates are set") + } +} + +func TestBuildOTelTLSConfigCAFile(t *testing.T) { + certPath, _ := writeSelfSignedPair(t) + got, err := buildOTelTLSConfig(config.OTelTLSConfig{CAFile: certPath}) + if err != nil { + t.Fatalf("buildOTelTLSConfig: %v", err) + } + if got.RootCAs == nil { + t.Error("RootCAs not populated from ca_file") + } +} + +func TestBuildOTelTLSConfigMTLSPair(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + got, err := buildOTelTLSConfig(config.OTelTLSConfig{CertFile: certPath, KeyFile: keyPath}) + if err != nil { + t.Fatalf("buildOTelTLSConfig: %v", err) + } + if len(got.Certificates) != 1 { + t.Errorf("Certificates = %d, want 1", len(got.Certificates)) + } +} + +func TestBuildOTelTLSConfigRejections(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + garbage := filepath.Join(t.TempDir(), "garbage.pem") + if err := os.WriteFile(garbage, []byte("not a certificate"), 0o600); err != nil { + t.Fatalf("write garbage: %v", err) + } + + cases := map[string]config.OTelTLSConfig{ + "missing ca file": {CAFile: filepath.Join(t.TempDir(), "absent.pem")}, + "unusable ca file": {CAFile: garbage}, + "cert without key": {CertFile: certPath}, + "key without cert": {KeyFile: keyPath}, + "mismatched pair": {CertFile: keyPath, KeyFile: certPath}, + } + for name, cfg := range cases { + t.Run(name, func(t *testing.T) { + if _, err := buildOTelTLSConfig(cfg); err == nil { + t.Error("expected an error, got nil") + } + }) + } +} + +// TestExportOverTLSWithCAFile proves the ca_file path actually works against a +// server whose certificate the system trust store does not know. +func TestExportOverTLSWithCAFile(t *testing.T) { + done := make(chan struct{}) + var once sync.Once + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + once.Do(func() { close(done) }) + })) + defer server.Close() + + caPath := filepath.Join(t.TempDir(), "server-ca.pem") + writePEM(t, caPath, "CERTIFICATE", server.Certificate().Raw) + + cfg := testOTelConfig(server.URL + "/v1/logs") + cfg.TLS = config.OTelTLSConfig{CAFile: caPath} + + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + defer publisher.Close(context.Background()) + publisher.Publish(restEvent()) + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("no export received over TLS") + } +} + +// TestNewOTelFailsClosedOnBadTLSMaterial: a publisher that can never reach its +// endpoint must not be constructed. +func TestNewOTelFailsClosedOnBadTLSMaterial(t *testing.T) { + cfg := testOTelConfig("https://collector.invalid/v1/logs") + cfg.TLS = config.OTelTLSConfig{CAFile: filepath.Join(t.TempDir(), "absent.pem")} + if _, err := NewOTel(&cfg); err == nil { + t.Error("expected NewOTel to fail on an unreadable ca_file") + } +} + +// --- retry and export-failure handling ------------------------------------- + +// scriptedEndpoint serves the given statuses in order, repeating the last one +// once the script is exhausted, and records every request it received. +type scriptedEndpoint struct { + mu sync.Mutex + statuses []int + requests []*http.Request + bodies [][]byte + headers http.Header +} + +func newScriptedEndpoint(t *testing.T, statuses ...int) (*scriptedEndpoint, string) { + t.Helper() + e := &scriptedEndpoint{statuses: statuses, headers: http.Header{}} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + e.mu.Lock() + attempt := len(e.requests) + e.requests = append(e.requests, r) + e.bodies = append(e.bodies, body) + status := e.statuses[len(e.statuses)-1] + if attempt < len(e.statuses) { + status = e.statuses[attempt] + } + e.mu.Unlock() + + for k, vs := range e.headers { + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(status) + })) + t.Cleanup(server.Close) + return e, server.URL + "/v1/logs" +} + +func (e *scriptedEndpoint) attempts() int { + e.mu.Lock() + defer e.mu.Unlock() + return len(e.requests) +} + +// retryConfig enables retries with a short backoff so tests stay fast. +func retryConfig(endpoint string, maxRetries int) config.OTelPublisherConfig { + cfg := testOTelConfig(endpoint) + cfg.MaxRetries = maxRetries + cfg.RetryBackoff = 5 * time.Millisecond + return cfg +} + +func (o *OTel) exportOne(t *testing.T) { + t.Helper() + o.exportN(t, 1) +} + +// exportN exports a batch of n identical records. Any test asserting a rejected +// count needs this: the endpoint cannot reject more records than were sent, so a +// single-record batch can only ever exercise a rejected count of 1. +func (o *OTel) exportN(t *testing.T, n int) { + t.Helper() + batch := make([]*otelLogRecord, 0, n) + for i := 0; i < n; i++ { + batch = append(batch, o.buildRecord(restEvent())) + } + o.export(batch) +} + +// A 5xx is transient: retry until it clears, and lose nothing when it does. +func TestExportRetriesOn5xxThenSucceeds(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503, 500, 200) + o := newTestOTel(t, retryConfig(url, 3)) + o.exportOne(t) + + if got := endpoint.attempts(); got != 3 { + t.Errorf("attempts = %d, want 3 (two failures then success)", got) + } + if dropped := o.droppedCount(); dropped != 0 { + t.Errorf("dropped = %d, want 0: the batch was delivered", dropped) + } +} + +// A 4xx other than 429 means the payload's shape was rejected. Retrying cannot +// fix that, and would multiply a permanent failure by the retry budget. +func TestExportDoesNotRetryPermanentRejection(t *testing.T) { + for _, status := range []int{400, 401, 404, 422} { + t.Run(http.StatusText(status), func(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, status) + o := newTestOTel(t, retryConfig(url, 3)) + o.exportOne(t) + + if got := endpoint.attempts(); got != 1 { + t.Errorf("attempts = %d, want 1 (no retry on %d)", got, status) + } + if dropped := o.droppedCount(); dropped != 1 { + t.Errorf("dropped = %d, want 1", dropped) + } + }) + } +} + +// 429 is retryable, and the endpoint's Retry-After replaces our own backoff +// rather than adding to it. +func TestExportHonoursRetryAfterOn429(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 429, 200) + endpoint.headers.Set("Retry-After", "1") + + cfg := retryConfig(url, 3) + cfg.RetryBackoff = time.Millisecond // far shorter than Retry-After + o := newTestOTel(t, cfg) + + start := time.Now() + o.exportOne(t) + elapsed := time.Since(start) + + if got := endpoint.attempts(); got != 2 { + t.Errorf("attempts = %d, want 2", got) + } + // The wait must come from Retry-After, not the 1ms backoff. + if elapsed < 900*time.Millisecond { + t.Errorf("elapsed = %s, want >= ~1s from Retry-After", elapsed) + } + if dropped := o.droppedCount(); dropped != 0 { + t.Errorf("dropped = %d, want 0", dropped) + } +} + +// Exhausting the budget drops the batch exactly once, counting every record. +func TestExportDropsBatchAfterBudgetExhausted(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503) + o := newTestOTel(t, retryConfig(url, 2)) + + batch := []*otelLogRecord{ + o.buildRecord(restEvent()), o.buildRecord(restEvent()), o.buildRecord(restEvent()), + } + o.export(batch) + + if got := endpoint.attempts(); got != 3 { + t.Errorf("attempts = %d, want 3 (initial + 2 retries)", got) + } + if dropped := o.droppedCount(); dropped != 3 { + t.Errorf("dropped = %d, want 3 (every record in the batch)", dropped) + } +} + +// One worker exports, so nothing drains the queue while a batch retries. Past +// the abort depth, retrying to save this batch costs more newer records than it +// rescues — so it must abandon its budget and return to draining. +func TestExportAbandonsRetriesUnderQueuePressure(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503) + cfg := retryConfig(url, 5) + cfg.RetryBackoff = time.Millisecond + cfg.QueueCapacity = 4 + cfg.RetryAbortQueueRatio = 0.5 // abort depth 2 + o := newTestOTel(t, cfg) + + if o.retryAbortDepth != 2 { + t.Fatalf("retryAbortDepth = %d, want 2", o.retryAbortDepth) + } + // Fill past the abort depth; nothing drains it. + for i := 0; i < 3; i++ { + o.queue <- o.buildRecord(restEvent()) + } + + o.exportOne(t) + + // The first attempt happens unconditionally; the depth check runs before the + // second, so exactly one attempt is made instead of the budgeted six. + if got := endpoint.attempts(); got != 1 { + t.Errorf("attempts = %d, want 1 (abandoned before the first retry)", got) + } + if dropped := o.droppedCount(); dropped != 1 { + t.Errorf("dropped = %d, want 1", dropped) + } +} + +// A ratio of 0 disables the check, so every batch gets its full budget. +func TestExportZeroAbortRatioUsesFullBudget(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503) + cfg := retryConfig(url, 2) + cfg.QueueCapacity = 2 + cfg.RetryAbortQueueRatio = 0 + o := newTestOTel(t, cfg) + + if o.retryAbortDepth != 0 { + t.Fatalf("retryAbortDepth = %d, want 0 (check disabled)", o.retryAbortDepth) + } + o.queue <- o.buildRecord(restEvent()) + o.queue <- o.buildRecord(restEvent()) + + o.exportOne(t) + if got := endpoint.attempts(); got != 3 { + t.Errorf("attempts = %d, want 3 despite a full queue", got) + } +} + +// A 2xx carrying partialSuccess is not a clean export: those records are gone, +// and must be counted rather than silently discarded. +func TestExportCountsPartialSuccessRejections(t *testing.T) { + cases := map[string]string{ + // Proto3 JSON encodes int64 as a string; some receivers emit a number. + "int64 as string": `{"partialSuccess":{"rejectedLogRecords":"2","errorMessage":"bad attribute"}}`, + "bare number": `{"partialSuccess":{"rejectedLogRecords":2,"errorMessage":"bad attribute"}}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(body)) + })) + defer server.Close() + + o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 3)) + before := scrapeMetrics(t) + o.exportN(t, 5) + + if dropped := o.droppedCount(); dropped != 2 { + t.Errorf("dropped = %d, want 2 from partialSuccess", dropped) + } + // The request succeeded, so 3 of the 5 were published — not all 5. + // Counting the whole batch would report 5 published and 2 dropped for + // 5 events that existed. + published := seriesKey("policy_engine_analytics_published_total") + if got := delta(t, before, scrapeMetrics(t), published); got != 3 { + t.Errorf("published delta = %v, want 3 (5 sent, 2 rejected)", got) + } + }) + } +} + +// An endpoint claiming more rejections than were sent is claiming something +// impossible. It has to be clamped rather than trusted: the published tally is +// the batch size minus this count, and a Prometheus counter panics on a negative +// Add — which on the export worker would take the process down. +func TestExportClampsOverReportedRejections(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"partialSuccess":{"rejectedLogRecords":"99999"}}`)) + })) + defer server.Close() + + o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 0)) + before := scrapeMetrics(t) + o.exportN(t, 2) // must not panic + + if dropped := o.droppedCount(); dropped != 2 { + t.Errorf("dropped = %d, want 2 clamped to the batch size", dropped) + } + published := seriesKey("policy_engine_analytics_published_total") + if got := delta(t, before, scrapeMetrics(t), published); got != 0 { + t.Errorf("published delta = %v, want 0 — the whole batch was rejected", got) + } +} + +// The ordinary success shapes must not be read as rejections. +func TestExportCleanSuccessBodiesCountNoDrops(t *testing.T) { + bodies := map[string]string{ + "empty": ``, + "empty object": `{}`, + "empty partial success": `{"partialSuccess":{}}`, + "explicit zero rejected": `{"partialSuccess":{"rejectedLogRecords":"0"}}`, + "not json": `OK`, + } + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(body)) + })) + defer server.Close() + + o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 0)) + o.exportOne(t) + if dropped := o.droppedCount(); dropped != 0 { + t.Errorf("dropped = %d, want 0", dropped) + } + }) + } +} + +// gzip must set Content-Encoding and produce a body the endpoint can inflate +// back into the same OTLP payload. +func TestExportGzipCompression(t *testing.T) { + type received struct { + encoding string + payload otelExportRequest + } + got := make(chan received, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + zr, err := gzip.NewReader(r.Body) + if err != nil { + t.Errorf("body is not gzip: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + defer zr.Close() + raw, err := io.ReadAll(zr) + if err != nil { + t.Errorf("inflate: %v", err) + } + var payload otelExportRequest + if err := json.Unmarshal(raw, &payload); err != nil { + t.Errorf("inflated body is not OTLP JSON: %v", err) + } + got <- received{encoding: r.Header.Get("Content-Encoding"), payload: payload} + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + cfg := retryConfig(server.URL+"/v1/logs", 0) + cfg.Compression = config.OTelCompressionGzip + o := newTestOTel(t, cfg) + o.exportOne(t) + + select { + case r := <-got: + if r.encoding != "gzip" { + t.Errorf("Content-Encoding = %q, want gzip", r.encoding) + } + if len(r.payload.ResourceLogs) != 1 || len(r.payload.ResourceLogs[0].ScopeLogs[0].LogRecords) != 1 { + t.Error("inflated payload did not carry the record") + } + case <-time.After(3 * time.Second): + t.Fatal("no export received") + } +} + +// Uncompressed is the default, and must not claim an encoding it did not apply. +func TestExportUncompressedByDefault(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 200) + o := newTestOTel(t, retryConfig(url, 0)) + o.exportOne(t) + + endpoint.mu.Lock() + defer endpoint.mu.Unlock() + if enc := endpoint.requests[0].Header.Get("Content-Encoding"); enc != "" { + t.Errorf("Content-Encoding = %q, want empty", enc) + } + if !json.Valid(endpoint.bodies[0]) { + t.Error("body is not plain JSON") + } +} + +// Shutdown must not wait out the remaining backoff: a retrying batch has to +// notice the stop signal instead of holding shutdown open. +func TestExportStopsRetryingOnShutdown(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503) + cfg := retryConfig(url, 100) + cfg.RetryBackoff = 30 * time.Second // long enough that waiting it out would fail the test + o := newTestOTel(t, cfg) + + done := make(chan struct{}) + go func() { + o.exportOne(t) + close(done) + }() + + // Wait for the first attempt to fail, then signal shutdown mid-backoff. + deadline := time.Now().Add(2 * time.Second) + for endpoint.attempts() == 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + close(o.stop) + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("export kept retrying through shutdown") + } + if got := endpoint.attempts(); got != 1 { + t.Errorf("attempts = %d, want 1 (stopped during the first backoff)", got) + } +} + +// --- self-observability ---------------------------------------------------- + +// The publishers package shares one process and one registry across tests, so +// counters accumulate. Every assertion below is therefore on a delta. + +// scrapeMetrics renders the registry exactly as the policy-engine's /metrics +// endpoint does, so these assertions also prove the series actually reach a +// scrape — a metric that was never registered increments happily and is simply +// absent from the endpoint. +func scrapeMetrics(t *testing.T) string { + t.Helper() + rec := httptest.NewRecorder() + promhttp.HandlerFor(metrics.Init(), promhttp.HandlerOpts{}). + ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("/metrics returned %d", rec.Code) + } + return rec.Body.String() +} + +// seriesKey builds the exposition-format identifier for one series. Prometheus +// renders label pairs sorted by label NAME, not in the order the vec declared +// them, so the pairs are sorted here to match: code="..." precedes +// publisher="otel", while reason="..." follows it. +func seriesKey(name string, labels ...string) string { + pairs := append([]string{`publisher="otel"`}, labels...) + sort.Strings(pairs) + return name + "{" + strings.Join(pairs, ",") + "}" +} + +// metricValue reads a series out of a scrape, reporting whether it was present +// at all — "absent" and "zero" are different answers and tests need both. +func metricValue(t *testing.T, scrape, key string) (float64, bool) { + t.Helper() + for _, line := range strings.Split(scrape, "\n") { + rest, ok := strings.CutPrefix(line, key+" ") + if !ok { + continue + } + value, err := strconv.ParseFloat(strings.TrimSpace(rest), 64) + if err != nil { + t.Fatalf("unparseable value for %s: %q", key, rest) + } + return value, true + } + return 0, false +} + +// delta reports how much a series moved, requiring it to exist afterwards. +func delta(t *testing.T, before string, after string, key string) float64 { + t.Helper() + old, _ := metricValue(t, before, key) + current, ok := metricValue(t, after, key) + if !ok { + t.Fatalf("series %s is absent from the scrape", key) + } + return current - old +} + +// A successful export must be counted, and its duration observed. +func TestMetricsSuccessfulExport(t *testing.T) { + _, url := newScriptedEndpoint(t, 200) + o := newTestOTel(t, retryConfig(url, 0)) + + before := scrapeMetrics(t) + o.export([]*otelLogRecord{o.buildRecord(restEvent()), o.buildRecord(restEvent())}) + after := scrapeMetrics(t) + + published := seriesKey("policy_engine_analytics_published_total") + if got := delta(t, before, after, published); got != 2 { + t.Errorf("published delta = %v, want 2", got) + } + duration := seriesKey("policy_engine_analytics_export_duration_seconds_count") + if got := delta(t, before, after, duration); got != 1 { + t.Errorf("duration observation delta = %v, want 1", got) + } +} + +// Each failure mode must land on its own reason, so an operator can tell them +// apart: a slow endpoint (backpressure) is a different problem from a broken one +// (send_failed) or a full queue. +func TestMetricsDropReasons(t *testing.T) { + t.Run("queue_full", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_dropped_total", `reason="`+dropReasonQueueFull+`"`) + before := scrapeMetrics(t) + + o := newUndrainedOTel(t, 1, config.QueueDropNew) + for i := 0; i < 3; i++ { + o.Publish(restEvent()) + } + + if got := delta(t, before, scrapeMetrics(t), key); got != 2 { + t.Errorf("queue_full delta = %v, want 2", got) + } + }) + + t.Run("send_failed", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_dropped_total", `reason="`+dropReasonSendFailed+`"`) + before := scrapeMetrics(t) + + _, url := newScriptedEndpoint(t, 503) + o := newTestOTel(t, retryConfig(url, 1)) + o.exportOne(t) + + if got := delta(t, before, scrapeMetrics(t), key); got != 1 { + t.Errorf("send_failed delta = %v, want 1", got) + } + }) + + t.Run("backpressure", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_dropped_total", `reason="`+dropReasonBackpressure+`"`) + before := scrapeMetrics(t) + + _, url := newScriptedEndpoint(t, 503) + cfg := retryConfig(url, 5) + cfg.QueueCapacity = 2 + cfg.RetryAbortQueueRatio = 0.5 + o := newTestOTel(t, cfg) + o.queue <- o.buildRecord(restEvent()) + o.exportOne(t) + + if got := delta(t, before, scrapeMetrics(t), key); got != 1 { + t.Errorf("backpressure delta = %v, want 1", got) + } + }) + + t.Run("rejected", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_dropped_total", `reason="`+dropReasonRejected+`"`) + before := scrapeMetrics(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"partialSuccess":{"rejectedLogRecords":"3"}}`)) + })) + defer server.Close() + + o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 0)) + o.exportN(t, 5) + + if got := delta(t, before, scrapeMetrics(t), key); got != 3 { + t.Errorf("rejected delta = %v, want 3", got) + } + }) +} + +// The error code distinguishes an unreachable endpoint from one that answered +// with a status, which are diagnosed differently. +func TestMetricsExportErrorCodes(t *testing.T) { + t.Run("transport", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_export_errors_total", `code="`+errCodeTransport+`"`) + before := scrapeMetrics(t) + + o := newTestOTel(t, retryConfig("http://127.0.0.1:1/v1/logs", 0)) + o.exportOne(t) + + if got := delta(t, before, scrapeMetrics(t), key); got != 1 { + t.Errorf("transport delta = %v, want 1", got) + } + }) + + t.Run("http status", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_export_errors_total", `code="503"`) + before := scrapeMetrics(t) + + _, url := newScriptedEndpoint(t, 503) + o := newTestOTel(t, retryConfig(url, 1)) + o.exportOne(t) + + // Both the initial attempt and the retry answered 503. + if got := delta(t, before, scrapeMetrics(t), key); got != 2 { + t.Errorf("503 delta = %v, want 2", got) + } + }) +} + +// Depth and capacity are published as a pair so an alert can express "the queue +// is 80% full" rather than an absolute depth that means nothing without it. +func TestMetricsQueueDepthAndCapacity(t *testing.T) { + _, url := newScriptedEndpoint(t, 200) + cfg := retryConfig(url, 0) + cfg.QueueCapacity = 8 + cfg.BatchSize = 8 + cfg.FlushInterval = time.Hour // park the worker so the queue holds + + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + defer publisher.Close(context.Background()) + + capacity, ok := metricValue(t, scrapeMetrics(t), seriesKey("policy_engine_analytics_queue_capacity")) + if !ok || capacity != 8 { + t.Errorf("capacity = %v (present=%v), want 8", capacity, ok) + } + + for i := 0; i < 3; i++ { + publisher.Publish(restEvent()) + } + // The worker consumes from the channel as records arrive, so depth is + // whatever is still buffered — assert it never exceeds capacity and was + // published at all rather than pinning an inherently racy exact value. + depth, ok := metricValue(t, scrapeMetrics(t), seriesKey("policy_engine_analytics_queue_depth")) + if !ok || depth < 0 || depth > 8 { + t.Errorf("depth = %v (present=%v), want within 0..8", depth, ok) + } +} + +// A labelled counter is absent from a scrape until first incremented, so a +// healthy gateway would show "No data" instead of 0 for the one series that +// makes silent analytics loss visible. +func TestMetricsPreInitializedAtZero(t *testing.T) { + _, url := newScriptedEndpoint(t, 200) + cfg := retryConfig(url, 0) + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + defer publisher.Close(context.Background()) + + scrape := scrapeMetrics(t) + keys := []string{ + seriesKey("policy_engine_analytics_published_total"), + seriesKey("policy_engine_analytics_queue_capacity"), + seriesKey("policy_engine_analytics_queue_depth"), + seriesKey("policy_engine_analytics_export_errors_total", `code="`+errCodeTransport+`"`), + } + for _, reason := range []string{ + dropReasonQueueFull, dropReasonSendFailed, dropReasonBackpressure, + dropReasonRejected, dropReasonSerializeFailed, + } { + keys = append(keys, seriesKey("policy_engine_analytics_dropped_total", `reason="`+reason+`"`)) + } + for _, key := range keys { + if _, ok := metricValue(t, scrape, key); !ok { + t.Errorf("%s is absent from the scrape; a healthy gateway must report a value, not No data", key) + } + } +} + +// --- header attributes ----------------------------------------------------- + +// headerEvent returns the REST fixture with the serialized header properties the +// analytics-header-filter policy produces. +func headerEvent(t *testing.T, request, response map[string]string) *dto.Event { + t.Helper() + event := restEvent() + for property, headers := range map[string]map[string]string{ + dto.PropKeyRequestHeaders: request, + dto.PropKeyResponseHeaders: response, + } { + if headers == nil { + continue + } + serialized, err := json.Marshal(headers) + if err != nil { + t.Fatalf("marshal headers: %v", err) + } + event.Properties[property] = string(serialized) + } + return event +} + +// One attribute per header, name lowercased into the key, value a string array — +// the shape the HTTP conventions require. +func TestBuildRecordHeaderAttributes(t *testing.T) { + event := headerEvent(t, + map[string]string{"Content-Type": "application/json", "X-Tenant": "acme"}, + map[string]string{"Cache-Control": "no-store"}) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, want := range map[string][]string{ + "http.request.header.content-type": {"application/json"}, + "http.request.header.x-tenant": {"acme"}, + "http.response.header.cache-control": {"no-store"}, + } { + values, ok := got[key].([]string) + if !ok { + t.Errorf("%s = %#v, want a string array", key, got[key]) + continue + } + if len(values) != len(want) || values[0] != want[0] { + t.Errorf("%s = %v, want %v", key, values, want) + } + } +} + +// No header-filter policy attached means no header properties on the event, and +// therefore no header attributes — never an empty or partial set. +func TestBuildRecordNoHeaderAttributesWhenAbsent(t *testing.T) { + cases := map[string]*dto.Event{ + "property absent": restEvent(), + "empty string": headerEventRaw(""), + "empty object": headerEventRaw("{}"), + "unparseable json": headerEventRaw("not json"), + "wrong type": headerEventWrongType(), + } + for name, event := range cases { + t.Run(name, func(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + for key := range got { + if strings.HasPrefix(key, "http.request.header.") || + strings.HasPrefix(key, "http.response.header.") { + t.Errorf("unexpected header attribute %q", key) + } + } + }) + } +} + +func headerEventRaw(serialized string) *dto.Event { + event := restEvent() + event.Properties[dto.PropKeyRequestHeaders] = serialized + return event +} + +func headerEventWrongType() *dto.Event { + event := restEvent() + event.Properties[dto.PropKeyRequestHeaders] = map[string]string{"not": "a string"} + return event +} + +// HTTP/2 pseudo-headers are not headers. Envoy surfaces them next to the real +// ones, and each duplicates an attribute already mapped from its own event +// field — while ":path" can carry a query string into an attribute key's value. +func TestBuildRecordHeaderAttributesSkipPseudoHeaders(t *testing.T) { + event := headerEvent(t, + map[string]string{ + ":method": "GET", ":path": "/otele2e/anything?token=secret", + ":scheme": "http", ":authority": "localhost:8080", + "x-tenant": "acme", + }, + map[string]string{":status": "200", "content-type": "application/json"}) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key := range got { + if strings.Contains(key, ".header.:") { + t.Errorf("pseudo-header emitted as %q", key) + } + } + // The real headers alongside them must still come through. + for _, want := range []string{"http.request.header.x-tenant", "http.response.header.content-type"} { + if _, present := got[want]; !present { + t.Errorf("%s is missing", want) + } + } +} + +// An over-broad allowlist must not grow a record's schema without bound: header +// names become attribute keys, and SDKs cap a record at 128 attributes. +func TestBuildRecordHeaderAttributesAreCapped(t *testing.T) { + headers := map[string]string{} + for i := 0; i < otelMaxHeaderAttributes*2; i++ { + headers[fmt.Sprintf("x-header-%03d", i)] = "value" + } + event := headerEvent(t, headers, nil) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + emitted := []string{} + for key := range got { + if strings.HasPrefix(key, "http.request.header.") { + emitted = append(emitted, key) + } + } + if len(emitted) != otelMaxHeaderAttributes { + t.Errorf("emitted %d header attributes, want the cap of %d", len(emitted), otelMaxHeaderAttributes) + } + // Sorted selection, so a truncated record keeps the same headers every time + // rather than an arbitrary subset that changes per request. + sort.Strings(emitted) + if emitted[0] != "http.request.header.x-header-000" { + t.Errorf("first emitted = %s, want the lowest-sorting name", emitted[0]) + } +} + +// The cap is per direction, so a wide request allowlist cannot starve the +// response headers. +func TestBuildRecordHeaderCapIsPerDirection(t *testing.T) { + request := map[string]string{} + for i := 0; i < otelMaxHeaderAttributes*2; i++ { + request[fmt.Sprintf("x-req-%03d", i)] = "value" + } + event := headerEvent(t, request, map[string]string{"Cache-Control": "no-store"}) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if _, present := got["http.response.header.cache-control"]; !present { + t.Error("response header dropped because request headers hit the cap") + } +} + +// A header with no value is skipped rather than emitted as an empty array, and +// must not consume cap budget. +func TestBuildRecordHeaderAttributesSkipEmptyValues(t *testing.T) { + event := headerEvent(t, map[string]string{"X-Present": "yes", "X-Empty": ""}, nil) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if _, present := got["http.request.header.x-empty"]; present { + t.Error("an empty header value produced an attribute") + } + if _, present := got["http.request.header.x-present"]; !present { + t.Error("x-present is missing") + } +} + +// The array must serialize as OTLP's ArrayValue shape, since that is the wire +// contract a collector parses. +func TestHeaderAttributeWireShape(t *testing.T) { + event := headerEvent(t, map[string]string{"Accept-Encoding": "gzip, br"}, nil) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + encoded, err := json.Marshal(o.buildRecord(event)) + if err != nil { + t.Fatalf("marshal record: %v", err) + } + want := `{"key":"http.request.header.accept-encoding","value":{"arrayValue":{"values":[{"stringValue":"gzip, br"}]}}}` + if !strings.Contains(string(encoded), want) { + t.Errorf("record does not contain the expected ArrayValue attribute.\nwant substring: %s\ngot: %s", want, encoded) + } +} + +// OTLP log records have traceId/spanId envelope fields. We deliberately do not +// emit them (design doc §2.10), and this pins that: adding the fields "for +// completeness" would send an all-zero id to every destination on every record. +// +// Note that a collector's own re-serialization may still show traceId:"" — its +// internal representation holds those as fixed-size values that are always +// present. That is the collector's output format, not our payload. +func TestRecordOmitsTraceEnvelopeFields(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + encoded, err := json.Marshal(o.buildRecord(restEvent())) + if err != nil { + t.Fatalf("marshal record: %v", err) + } + for _, field := range []string{"traceId", "spanId"} { + if strings.Contains(string(encoded), field) { + t.Errorf("record carries %q; it must be absent, not empty: %s", field, encoded) + } + } +} + +// A key the analytics policy gains later must appear in the export rather than +// vanish until someone notices. That is the whole point of the sweep: flattening +// to curated names would otherwise mean the publisher and the policy drift +// silently. +func TestMCPUnmappedKeysAreSweptUp(t *testing.T) { + event := restEvent() + event.API.APIType = "Mcp" + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "jsonRpcMethod": "tools/call", + // Hypothetical future additions, at both levels. + "toolInvocationCount": float64(3), + "cacheWasWarm": true, + "upstreamLatencyMs": 12.5, + "clientInfo": map[string]interface{}{ + "name": "claude-desktop", + "platformId": "darwin-arm64", + }, + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, want := range map[string]interface{}{ + // camelCase becomes snake_case under the custom namespace. + "wso2.mcp.tool_invocation_count": "3", // integral float -> IntValue, formatted as a string + "wso2.mcp.cache_was_warm": true, + "wso2.mcp.upstream_latency_ms": 12.5, + "wso2.mcp.client.platform_id": "darwin-arm64", + // The curated name still wins for a key that has one. + "mcp.method.name": "tools/call", + "wso2.mcp.client.name": "claude-desktop", + } { + if got[key] != want { + t.Errorf("%s = %#v, want %#v", key, got[key], want) + } + } + + // A key with a curated name must not also appear under the sweep prefix. + for _, absent := range []string{ + "wso2.mcp.json_rpc_method", "wso2.mcp.client_info", "wso2.mcp.client.name_", + } { + if _, present := got[absent]; present { + t.Errorf("%s was emitted twice / under the wrong name", absent) + } + } +} + +// `capability` is read but deliberately not emitted — which attribute is +// populated already says it. It must not reappear via the sweep. +func TestMCPCapabilityIsNotSwept(t *testing.T) { + event := restEvent() + event.API.APIType = "Mcp" + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "jsonRpcMethod": "tools/call", + "capability": "TOOL", + "capabilityName": "search_docs", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for _, absent := range []string{"wso2.mcp.capability", "wso2.mcp.capability_name"} { + if _, present := got[absent]; present { + t.Errorf("%s must not be swept up; it has a curated mapping", absent) + } + } + if got["gen_ai.tool.name"] != "search_docs" { + t.Errorf("gen_ai.tool.name = %v, want search_docs", got["gen_ai.tool.name"]) + } +} + +func TestOTelSnakeCase(t *testing.T) { + for input, want := range map[string]string{ + "jsonRpcMethod": "json_rpc_method", + "resourceUri": "resource_uri", + "isError": "is_error", + "requestedProtocolVersion": "requested_protocol_version", + "sessionId": "session_id", + "already_snake": "already_snake", + "name": "name", + "": "", + } { + if got := otelSnakeCase(input); got != want { + t.Errorf("otelSnakeCase(%q) = %q, want %q", input, got, want) + } + } +} + +// An integral JSON number must not be reported as a double: a backend types the +// column from the first value it sees, and a consumer summing 3.0 and 3 does not +// reliably get the same answer. +func TestAnyScalarNumberKinds(t *testing.T) { + attrs := newOTelAttrs() + attrs.anyScalar("whole", float64(3)) + attrs.anyScalar("fractional", 12.5) + attrs.anyScalar("text", "hello") + attrs.anyScalar("flag", true) + attrs.anyScalar("unsupported", []string{"nope"}) + + encoded, err := json.Marshal(attrs.list()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{ + `{"key":"whole","value":{"intValue":"3"}}`, + `{"key":"fractional","value":{"doubleValue":12.5}}`, + `{"key":"text","value":{"stringValue":"hello"}}`, + `{"key":"flag","value":{"boolValue":true}}`, + } { + if !strings.Contains(string(encoded), want) { + t.Errorf("missing %s in %s", want, encoded) + } + } + if strings.Contains(string(encoded), "unsupported") { + t.Error("an unsupported type produced an attribute") + } +} + +// --- Zero is a value, not an absence ----------------------------------------- +// +// A record has one way to say "this attribute does not apply to this request": +// leave it out. So a measured zero must be emitted, or "the guardrail blocked +// this request so it produced no output tokens" and "this is a REST call with no +// tokens at all" become the same record to a consumer — and every avg() over the +// field drops the zeros from its denominator instead of counting them. + +// The wire shape is what actually carries the distinction. omitempty on a +// pointer field tests only for nil, which is why a *string holding "0" and a +// *bool holding false still marshal. Asserted here because a later refactor to +// non-pointer fields would silently restore the bug this test exists to prevent. +func TestZeroValuedAttributesReachTheWire(t *testing.T) { + attrs := newOTelAttrs() + attrs.i64("zero.int", 0) + attrs.f64("zero.double", 0) + attrs.b("zero.bool", false) + + encoded, err := json.Marshal(attrs.list()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{ + `{"key":"zero.int","value":{"intValue":"0"}}`, + `{"key":"zero.double","value":{"doubleValue":0}}`, + `{"key":"zero.bool","value":{"boolValue":false}}`, + } { + if !strings.Contains(string(encoded), want) { + t.Errorf("missing %s in %s", want, encoded) + } + } +} + +// A guardrail-blocked completion: the prompt was tokenized, nothing was +// generated, nothing was billed. Reporting no output tokens is the whole point +// of the record. +func TestBuildRecordGenAIZeroUsageIsReported(t *testing.T) { + event := restEvent() + event.API.APIType = "LlmProxy" + event.Operation.APIResourceTemplate = "/ai/chat/completions" + event.Properties["aiMetadata"] = dto.AIMetadata{ + Model: "claude-opus-4", + VendorName: "anthropic", + LLMCost: float64(0), + } + event.Properties["aiTokenUsage"] = dto.AITokenUsage{PromptToken: 1841, CompletionToken: 0, TotalToken: 1841} + event.Properties[constants.GuardrailHitMetadataKey] = true + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, want := range map[string]interface{}{ + "gen_ai.usage.input_tokens": "1841", + "gen_ai.usage.output_tokens": "0", + "wso2.gen_ai.usage.total_tokens": "1841", + "wso2.gen_ai.cost.total": float64(0), + "wso2.guardrail.hit": true, + } { + actual, present := got[key] + if !present { + t.Errorf("%s is absent; a measured zero must be emitted, not omitted", key) + continue + } + if actual != want { + t.Errorf("%s = %v (%T), want %v (%T)", key, actual, actual, want, want) + } + } +} + +// A GET has no request body and a cache miss is not a cache hit. Both are +// measurements the record must carry: without them "empty body" is +// indistinguishable from "body size not measured", and a cache miss from an API +// with no cache filter at all — which is what makes a hit ratio uncomputable. +func TestBuildRecordZeroSizesAndFalseFlagsAreReported(t *testing.T) { + event := restEvent() + event.Properties["requestSize"] = uint64(0) + event.Properties["responseSize"] = uint64(0) + event.Target.ResponseCacheHit = false + event.Latencies = &dto.Latencies{ResponseLatency: 4, BackendLatency: 0, Duration: 4} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, want := range map[string]interface{}{ + "http.request.body.size": "0", + "http.response.body.size": "0", + "wso2.cache.hit": false, + // Served from cache or sub-millisecond: zero backend time, measured. + "wso2.latency.backend_ms": "0", + "wso2.latency.request_mediation_ms": "0", + "wso2.latency.response_mediation_ms": "0", + } { + actual, present := got[key] + if !present { + t.Errorf("%s is absent; a measured zero must be emitted, not omitted", key) + continue + } + if actual != want { + t.Errorf("%s = %v (%T), want %v (%T)", key, actual, actual, want, want) + } + } +} + +// The exceptions. For these four, 0 is a sentinel rather than a measurement — +// there is no HTTP status 0, no TCP port 0, no error code 0 — so they keep +// suppressing it via i64NonZero. +func TestBuildRecordSentinelZerosStayOmitted(t *testing.T) { + event := restEvent() + event.ProxyResponseCode = 0 + event.Target = &dto.Target{TargetResponseCode: 0, Destination: "backend:0/pet/1"} + event.Error = &dto.Error{ErrorCode: 0, ErrorMessage: dto.OtherUnclassified} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for _, key := range []string{ + "http.response.status_code", + "server.port", + "wso2.upstream.response.status_code", + "wso2.error.code", + } { + if actual, present := got[key]; present { + t.Errorf("%s = %v; 0 is a sentinel for this attribute and must be omitted", key, actual) + } + } +} + +// --- Retry backoff ceiling --------------------------------------------------- +// +// One worker exports, so nothing drains the queue while a batch sleeps between +// attempts. An uncapped exponential backoff therefore parks that worker for as +// long as retry_backoff and max_retries multiply out to, and the queue behind it +// fills meanwhile — the same failure retryAfterCap already prevents an endpoint +// from causing with its own Retry-After. + +func TestBackoffGrowsExponentiallyWithinJitterRange(t *testing.T) { + o := &OTel{cfg: config.OTelPublisherConfig{RetryBackoff: time.Second}} + for attempt, want := range map[int]time.Duration{1: time.Second, 2: 2 * time.Second, 3: 4 * time.Second} { + got := o.backoff(attempt) + // Full jitter: [want/2, want). + if got < want/2 || got >= want { + t.Errorf("backoff(%d) = %s, want within [%s, %s)", attempt, got, want/2, want) + } + } +} + +func TestBackoffIsCappedAndStillJittered(t *testing.T) { + // 10m base: attempt 3 alone would be 40m uncapped. + o := &OTel{cfg: config.OTelPublisherConfig{RetryBackoff: 10 * time.Minute}} + + distinct := map[time.Duration]bool{} + for i := 0; i < 200; i++ { + got := o.backoff(3) + if got >= otelMaxRetryBackoff { + t.Fatalf("backoff = %s, want under the %s ceiling", got, otelMaxRetryBackoff) + } + if got < otelMaxRetryBackoff/2 { + t.Fatalf("backoff = %s, want at least half the ceiling", got) + } + distinct[got] = true + } + // Capped before jittering, so the delays still spread below the ceiling. Were + // the cap applied after, every attempt would land on it exactly and the + // replicas the jitter exists to spread out would resynchronise. + if len(distinct) < 2 { + t.Errorf("capped backoff produced %d distinct value(s); jitter was lost to the cap", len(distinct)) + } +} + +// A shift large enough to overflow int64 yields a negative duration, which would +// skip the wait entirely rather than lengthen it. It needs an absurd +// retry_backoff to reach, but the result must still be a real wait. +func TestBackoffOverflowFallsBackToCeiling(t *testing.T) { + o := &OTel{cfg: config.OTelPublisherConfig{RetryBackoff: 200 * 24 * time.Hour}} + got := o.backoff(11) // shift clamps to 10; 200d << 10 overflows + if got <= 0 { + t.Fatalf("backoff = %s, want a positive wait", got) + } + if got >= otelMaxRetryBackoff { + t.Errorf("backoff = %s, want under the %s ceiling", got, otelMaxRetryBackoff) + } +} + +// A zero/unset retry_backoff must still produce a real wait rather than a +// busy retry loop. +func TestBackoffUnsetBaseUsesOneSecond(t *testing.T) { + o := &OTel{cfg: config.OTelPublisherConfig{RetryBackoff: 0}} + if got := o.backoff(1); got < 500*time.Millisecond || got >= time.Second { + t.Errorf("backoff(1) = %s, want within [500ms, 1s)", got) + } +} + +// endpointForLog must drop anything that can carry a credential. +func TestEndpointForLog(t *testing.T) { + for raw, want := range map[string]string{ + "https://collector:4318/v1/logs": "https://collector:4318/v1/logs", + "https://collector:4318/v1/logs?api-key=s3cr3t": "https://collector:4318/v1/logs", + "https://svc:pw@collector:4318/v1/logs": "https://collector:4318/v1/logs", + "https://svc:pw@collector:4318/v1/logs?token=s3cr3t": "https://collector:4318/v1/logs", + "http://otel-collector:4318/v1/logs": "http://otel-collector:4318/v1/logs", + "://bad url": "(unparseable endpoint)", + } { + if got := endpointForLog(raw); got != want { + t.Errorf("endpointForLog(%q) = %q, want %q", raw, got, want) + } + } +} + +// The logged endpoint must never carry the query string, whatever the config holds. +func TestNewOTelStoresRedactedEndpoint(t *testing.T) { + cfg := testOTelConfig("https://collector:4318/v1/logs?api-key=s3cr3t") + o, err := NewOTel(&cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = o.Close(context.Background()) }() + + if strings.Contains(o.logEndpoint, "s3cr3t") || strings.Contains(o.logEndpoint, "api-key") { + t.Fatalf("logEndpoint leaks the credential: %q", o.logEndpoint) + } + if o.cfg.Endpoint != "https://collector:4318/v1/logs?api-key=s3cr3t" { + t.Errorf("cfg.Endpoint must keep the full URL for the request, got %q", o.cfg.Endpoint) + } +} + +// An unrecognized capability emits no capability-target attribute, so +// capabilityName/resourceUri must stay unclaimed and reach the sweep rather than +// being dropped. Moesif already receives capabilityName for these methods. +func TestMCPUnrecognizedCapabilityFallsToSweep(t *testing.T) { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "jsonRpcMethod": "completion/complete", + "capability": "", // deriveMCPCapability returns "" for a non tools/prompts/resources prefix + "capabilityName": "greet", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["wso2.mcp.capability_name"] != "greet" { + t.Errorf("wso2.mcp.capability_name = %v, want greet", got["wso2.mcp.capability_name"]) + } + for _, k := range []string{"gen_ai.tool.name", "gen_ai.prompt.name", "mcp.resource.uri"} { + if _, present := got[k]; present { + t.Errorf("%s must not be emitted for an unrecognized capability", k) + } + } +} + +// The sweep must not duplicate a value a branch already emitted under its +// curated name. +func TestMCPRecognizedCapabilityIsNotAlsoSwept(t *testing.T) { + cases := []struct { + capability, key, curated string + }{ + {"TOOL", "capabilityName", "gen_ai.tool.name"}, + {"PROMPT", "capabilityName", "gen_ai.prompt.name"}, + {"RESOURCE", "resourceUri", "mcp.resource.uri"}, + } + for _, tc := range cases { + t.Run(tc.capability, func(t *testing.T) { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "capability": tc.capability, + tc.key: "value-x", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got[tc.curated] != "value-x" { + t.Errorf("%s = %v, want value-x", tc.curated, got[tc.curated]) + } + swept := ns("mcp." + otelSnakeCase(tc.key)) + if _, present := got[swept]; present { + t.Errorf("%s was swept as well as emitted under %s", swept, tc.curated) + } + }) + } +} + +// The unused sibling key stays unclaimed, so a request carrying both still +// reports the one its capability does not use. +func TestMCPUnusedSiblingKeyIsSwept(t *testing.T) { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "capability": "TOOL", + "capabilityName": "search_docs", + "resourceUri": "file:///unexpected.md", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["gen_ai.tool.name"] != "search_docs" { + t.Errorf("gen_ai.tool.name = %v, want search_docs", got["gen_ai.tool.name"]) + } + if got["wso2.mcp.resource_uri"] != "file:///unexpected.md" { + t.Errorf("wso2.mcp.resource_uri = %v, want the unused sibling to be swept", got["wso2.mcp.resource_uri"]) + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_factory.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_factory.go index 529270cf97..233030c056 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_factory.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_factory.go @@ -34,25 +34,35 @@ const ( sinkNameHTTP = config.TrafficLogSinkHTTP ) -// Reasons recorded on policy_engine_traffic_log_dropped_total. +// Reasons recorded on policy_engine_traffic_log_dropped_total and +// policy_engine_analytics_dropped_total. Shared so the two subsystems describe +// the same failure with the same label rather than each inventing a spelling. const ( - // dropReasonQueueFull: the HTTP sink's bounded queue had no room. + // dropReasonQueueFull: a bounded queue had no room for the incoming item. dropReasonQueueFull = "queue_full" - // dropReasonSendFailed: the HTTP sink exhausted its retries. + // dropReasonSendFailed: the sender exhausted its retry budget. dropReasonSendFailed = "send_failed" // dropReasonWriteFailed: a local write returned an error. dropReasonWriteFailed = "write_failed" // dropReasonRotateFailed: the file sink could not rotate, so the line that // triggered the rotation was not written. dropReasonRotateFailed = "rotate_failed" - // dropReasonBackpressure: the HTTP sink abandoned a batch's remaining retries - // because the queue was filling behind it. Distinct from send_failed so an - // operator can tell "the receiver is slow" from "the receiver is broken". + // dropReasonBackpressure: a batch's remaining retries were abandoned because + // the queue was filling behind it. Distinct from send_failed so an operator + // can tell "the destination is slow" from "the destination is broken". dropReasonBackpressure = "backpressure" + // dropReasonRejected: the destination accepted the request but refused some + // records (an OTLP partialSuccess). The request succeeded, so this is not an + // export error — but the records are gone just the same. + dropReasonRejected = "rejected" + // dropReasonSerializeFailed: the batch could not be encoded or compressed, so + // it was never sent. Retrying cannot help; the payload itself is the problem. + dropReasonSerializeFailed = "serialize_failed" ) -// Codes recorded on policy_engine_traffic_log_write_errors_total for non-HTTP -// failures. HTTP failures use the numeric status code instead. +// Codes recorded on policy_engine_traffic_log_write_errors_total and +// policy_engine_analytics_export_errors_total for non-HTTP failures. HTTP +// failures use the numeric status code instead. const ( errCodeWrite = "write" errCodeRotate = "rotate" diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http.go index 690eff18c0..dafdcf52f5 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http.go @@ -156,7 +156,7 @@ func newHTTPSink(cfg config.TrafficLogHTTPConfig) (*httpSink, error) { authHeaderValue: value, queue: make(chan []byte, cfg.QueueCapacity), dropOldest: strings.EqualFold(strings.TrimSpace(cfg.OnQueueFull), - config.TrafficLogQueueDropOldest), + config.QueueDropOldest), done: make(chan struct{}), stopped: make(chan struct{}), // Below this, retrying is free — nothing is being lost diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http_test.go index 0a863f26bb..e9658b3e20 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http_test.go @@ -99,7 +99,7 @@ func httpSinkCfg(endpoint string) config.TrafficLogHTTPConfig { BatchMaxBytes: 1 << 20, FlushInterval: 25 * time.Millisecond, QueueCapacity: 100, - OnQueueFull: config.TrafficLogQueueDropNew, + OnQueueFull: config.QueueDropNew, RequestTimeout: 2 * time.Second, MaxRetries: 0, RetryBackoff: time.Millisecond, diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 3ba67a6653..a4acf959a8 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -134,6 +134,97 @@ type AnalyticsConfig struct { // AnalyticsPublishersConfig holds configuration for all analytics publishers type AnalyticsPublishersConfig struct { Moesif MoesifPublisherConfig `koanf:"moesif"` + OTel OTelPublisherConfig `koanf:"otel"` +} + +// OTelPublisherConfig configures the OpenTelemetry analytics publisher, which +// exports each event as an OTLP log record over OTLP/HTTP. +type OTelPublisherConfig struct { + // Endpoint is the full OTLP/HTTP logs URL, including the /v1/logs path. + Endpoint string `koanf:"endpoint"` + // Headers are sent on every export request. Use for a vendor's OTLP intake + // that authenticates by header. Values are secrets and are never logged. + Headers map[string]string `koanf:"headers"` + // ServiceName / ServiceVersion populate the OTLP resource. + ServiceName string `koanf:"service_name"` + ServiceVersion string `koanf:"service_version"` + // ResourceAttributes are added to the OTLP resource, alongside service.*. + ResourceAttributes map[string]string `koanf:"resource_attributes"` + // BatchSize is the record count that triggers an export before FlushInterval. + BatchSize int `koanf:"batch_size"` + // FlushInterval bounds how long a record waits when traffic is too slow to + // fill a batch. + FlushInterval time.Duration `koanf:"flush_interval"` + // QueueCapacity bounds records held in memory when the endpoint is slow. + // Once full, OnQueueFull decides which record is dropped. + QueueCapacity int `koanf:"queue_capacity"` + // OnQueueFull is QueueDropNew (default) or QueueDropOldest. + OnQueueFull string `koanf:"on_queue_full"` + // Timeout bounds a single export attempt. + Timeout time.Duration `koanf:"timeout"` + // MaxRetries is the number of retry attempts after the initial one. Only + // transport errors, 429 and 5xx are retried; any other 4xx means the endpoint + // rejected the payload's shape, which retrying can only amplify. + MaxRetries int `koanf:"max_retries"` + // RetryBackoff is the base delay for exponential backoff, with full jitter + // applied per attempt so replicas retrying after a shared outage do not + // resynchronize into a thundering herd. + RetryBackoff time.Duration `koanf:"retry_backoff"` + // RetryAbortQueueRatio is the fraction of QueueCapacity at which a retrying + // batch abandons its remaining budget and returns to draining. One worker + // exports, so nothing drains the queue while a batch retries: past this + // depth, retrying to save one batch costs more records than it rescues. 0 + // disables the check and lets every batch use its full budget. + RetryAbortQueueRatio float64 `koanf:"retry_abort_queue_ratio"` + // Compression is "none" (default, the OTLP spec's own default) or "gzip". + Compression string `koanf:"compression"` + // AllowInsecureTransport permits a plaintext http:// endpoint. Off by default + AllowInsecureTransport bool `koanf:"allow_insecure_transport"` + // TLS configures the client side of an https endpoint. Ignored for http. + TLS OTelTLSConfig `koanf:"tls"` +} + +// Accepted values for analytics.publishers.otel.compression. +const ( + // OTelCompressionNone sends the OTLP-JSON payload uncompressed. + OTelCompressionNone = "none" + // OTelCompressionGzip sends it gzip-encoded with Content-Encoding: gzip. + OTelCompressionGzip = "gzip" +) + +// DefaultOTelRetryAbortQueueRatio is the fraction of the OTel publisher's queue at +// which a retrying batch gives up, matching the traffic-log sink's midpoint: high +// enough that an ordinary blip still gets its full retry budget, low enough that a +// hung endpoint cannot consume the whole queue before exporting resumes. +const DefaultOTelRetryAbortQueueRatio = 0.5 + +// EffectiveRetryAbortDepth returns the queue depth at which a retrying batch stops +// retrying. Zero means the check is disabled. +func (c OTelPublisherConfig) EffectiveRetryAbortDepth() int { + depth := int(float64(c.QueueCapacity) * c.RetryAbortQueueRatio) + if depth < 1 && c.RetryAbortQueueRatio > 0 { + depth = 1 + } + return depth +} + +// OTelTLSConfig configures TLS to the OTLP endpoint +// ([analytics.publishers.otel.tls]). Deliberately a separate type from +// TrafficLogHTTPTLSConfig despite the identical keys: the two config blocks are +// independent, and sharing one type would couple them. +type OTelTLSConfig struct { + // CAFile is a PEM bundle used to verify the endpoint's certificate. Empty + // means the system trust store, which is correct for a vendor's OTLP intake + // and usually wrong for an in-cluster collector fronted by a private CA. + CAFile string `koanf:"ca_file"` + // CertFile / KeyFile enable mTLS. Both must be set, or neither. + CertFile string `koanf:"cert_file"` + KeyFile string `koanf:"key_file"` + // InsecureSkipVerify disables endpoint certificate verification. Off by + // default; when on, startup logs a warning naming the endpoint, because + // analytics records carry request metadata and, when body capture is + // enabled, request and response bodies. + InsecureSkipVerify bool `koanf:"insecure_skip_verify"` } // Traffic-log sink names accepted in traffic_logging.outputs. @@ -162,12 +253,14 @@ const ( TrafficLogAuthHeader = "header" ) -// Behavior when the HTTP sink's queue is full (traffic_logging.http.on_queue_full). +// Behavior when a bounded publisher queue is full. Shared vocabulary: both +// traffic_logging.http.on_queue_full and analytics.publishers.otel.on_queue_full +// accept exactly these values, so the two must never diverge. const ( - // TrafficLogQueueDropNew discards the incoming line, preserving older ones. - TrafficLogQueueDropNew = "drop_new" - // TrafficLogQueueDropOldest evicts the oldest queued line to make room. - TrafficLogQueueDropOldest = "drop_oldest" + // QueueDropNew discards the incoming item, preserving older queued ones. + QueueDropNew = "drop_new" + // QueueDropOldest evicts the oldest queued item to make room for the new one. + QueueDropOldest = "drop_oldest" ) // TrafficLoggingConfig holds configuration for the traffic-logging feature, which @@ -1007,7 +1100,7 @@ func defaultTrafficLogHTTPConfig() TrafficLogHTTPConfig { // ride out a short receiver blip without letting a long outage grow the // heap without bound. QueueCapacity: 10000, - OnQueueFull: TrafficLogQueueDropNew, + OnQueueFull: QueueDropNew, RequestTimeout: 10 * time.Second, MaxRetries: 3, RetryBackoff: time.Second, @@ -1170,6 +1263,21 @@ func defaultConfig() *Config { BatchSize: 50, TimerWakeupSeconds: 3, }, + OTel: OTelPublisherConfig{ + Endpoint: "http://otel-collector:4318/v1/logs", + ServiceName: "gateway-runtime", + ServiceVersion: "", + BatchSize: 100, + FlushInterval: 5 * time.Second, + QueueCapacity: 10000, + OnQueueFull: QueueDropNew, + Timeout: 10 * time.Second, + MaxRetries: 3, + RetryBackoff: time.Second, + // Half: retry freely while the queue is shallow, stop once it fills. + RetryAbortQueueRatio: DefaultOTelRetryAbortQueueRatio, + Compression: OTelCompressionNone, + }, }, GRPCEventServerCfg: map[string]interface{}{ "server_port": 18090, @@ -1454,6 +1562,148 @@ func (c *Config) validateXDSConfig() error { return nil } +// validateOTelPublisherConfig validates [analytics.publishers.otel]. +func validateOTelPublisherConfig(cfg OTelPublisherConfig) error { + if cfg.Endpoint == "" { + return fmt.Errorf("analytics.publishers.otel.endpoint is required when otel is enabled") + } + u, err := url.Parse(cfg.Endpoint) + if err != nil || u.Host == "" { + return fmt.Errorf("analytics.publishers.otel.endpoint must be a valid URL (e.g. http://otel-collector:4318/v1/logs), got %q", cfg.Endpoint) + } + // Reject URL credentials to prevent endpoint leakage through logs and HTTP errors; + // use headers for authentication. + if u.User != nil { + return fmt.Errorf("analytics.publishers.otel.endpoint must not contain credentials in the "+ + "URL (user:password@%s); the endpoint is written to logs, so use "+ + "analytics.publishers.otel.headers to authenticate instead", u.Host) + } + switch u.Scheme { + case "https": + case "http": + if !cfg.AllowInsecureTransport { + return fmt.Errorf("analytics.publishers.otel.endpoint uses plaintext http:// but "+ + "analytics.publishers.otel.allow_insecure_transport is false; analytics records carry "+ + "API keys and consumer identity, so set allow_insecure_transport = true only for a "+ + "trusted local collector, got %q", cfg.Endpoint) + } + slog.Warn("analytics.publishers.otel endpoint is plaintext http://; analytics records are "+ + "transmitted unencrypted", "host", u.Host) + // Headers are the endpoint's intake credential. Over plaintext they are on + // the wire in clear text on every export, which the scheme warning alone + // does not convey. + if len(cfg.Headers) > 0 { + slog.Warn("analytics.publishers.otel.headers are sent over a plaintext http:// endpoint; "+ + "the credential they carry is exposed to anyone able to intercept this connection", + "host", u.Host, "headerCount", len(cfg.Headers)) + } + default: + return fmt.Errorf("analytics.publishers.otel.endpoint scheme must be https (or http with "+ + "allow_insecure_transport), got %q", u.Scheme) + } + if cfg.ServiceName == "" { + return fmt.Errorf("analytics.publishers.otel.service_name is required") + } + if cfg.BatchSize <= 0 { + return fmt.Errorf("analytics.publishers.otel.batch_size must be > 0, got %d", cfg.BatchSize) + } + if cfg.QueueCapacity <= 0 { + return fmt.Errorf("analytics.publishers.otel.queue_capacity must be > 0, got %d; an unbounded "+ + "queue in front of a bounded exporter is deferred unbounded memory growth", cfg.QueueCapacity) + } + // A queue smaller than a batch can never fill one, so every export would be + // interval-driven regardless of load. + if cfg.QueueCapacity < cfg.BatchSize { + return fmt.Errorf("analytics.publishers.otel.queue_capacity (%d) must be >= batch_size (%d)", cfg.QueueCapacity, cfg.BatchSize) + } + switch strings.ToLower(strings.TrimSpace(cfg.OnQueueFull)) { + case QueueDropNew, QueueDropOldest: + default: + return fmt.Errorf("analytics.publishers.otel.on_queue_full must be %q or %q, got %q", + QueueDropNew, QueueDropOldest, cfg.OnQueueFull) + } + if cfg.FlushInterval <= 0 { + return fmt.Errorf("analytics.publishers.otel.flush_interval must be > 0, got %s", cfg.FlushInterval) + } + if cfg.Timeout <= 0 { + return fmt.Errorf("analytics.publishers.otel.timeout must be > 0, got %s", cfg.Timeout) + } + if cfg.MaxRetries < 0 { + return fmt.Errorf("analytics.publishers.otel.max_retries must be >= 0, got %d", cfg.MaxRetries) + } + if cfg.MaxRetries > 0 && cfg.RetryBackoff <= 0 { + return fmt.Errorf("analytics.publishers.otel.retry_backoff must be positive when max_retries > 0, got %s", + cfg.RetryBackoff) + } + if cfg.RetryAbortQueueRatio < 0 || cfg.RetryAbortQueueRatio > 1 { + return fmt.Errorf("analytics.publishers.otel.retry_abort_queue_ratio must be between 0 and 1, got %v", + cfg.RetryAbortQueueRatio) + } + switch strings.ToLower(strings.TrimSpace(cfg.Compression)) { + case "", OTelCompressionNone, OTelCompressionGzip: + default: + return fmt.Errorf("analytics.publishers.otel.compression must be %q or %q, got %q", + OTelCompressionNone, OTelCompressionGzip, cfg.Compression) + } + if err := validateOTelTLS(cfg.TLS, u.Host); err != nil { + return fmt.Errorf("analytics.publishers.otel.tls: %w", err) + } + return nil +} + +// tlsKeyPermMask is the set of permission bits that must be clear on a TLS +// private key: anything readable by group or other. +const tlsKeyPermMask os.FileMode = 0o077 + +// verifyTLSKeyPerms fails when a TLS private key is readable by group or other +// (GO-AUTH-018). Shared so traffic_logging.http.tls can adopt the same check. +func verifyTLSKeyPerms(field, path string) error { + fi, err := os.Stat(path) + if err != nil { + return fmt.Errorf("cannot stat %s %q: %w", field, path, err) + } + if fi.IsDir() { + return fmt.Errorf("%s %q is a directory, not a private key file", field, path) + } + if perm := fi.Mode().Perm(); perm&tlsKeyPermMask != 0 { + return fmt.Errorf("%s %q has permissions %#o, which allow group/other access to a private "+ + "key; fix it with `chmod 600 %s` and restart", field, path, perm, path) + } + return nil +} + +// validateOTelTLS checks that any referenced TLS material exists and parses, so a +// bad path fails at startup rather than on the first export. +func validateOTelTLS(cfg OTelTLSConfig, host string) error { + if cfg.InsecureSkipVerify { + slog.Warn("analytics.publishers.otel.tls.insecure_skip_verify is true: the endpoint's "+ + "certificate is not verified, so analytics records are exposed to anyone able to "+ + "intercept this connection", "host", host) + } + if cfg.CAFile != "" { + pem, err := os.ReadFile(cfg.CAFile) + if err != nil { + return fmt.Errorf("cannot read ca_file %q: %w", cfg.CAFile, err) + } + if !x509.NewCertPool().AppendCertsFromPEM(pem) { + return fmt.Errorf("ca_file %q contains no usable PEM certificate", cfg.CAFile) + } + } + if (cfg.CertFile == "") != (cfg.KeyFile == "") { + return fmt.Errorf("cert_file and key_file must be set together for mTLS (one is set, the other is not)") + } + if cfg.CertFile != "" { + // Permissions first: a key anyone can read is a finding whether or not it parses. + if err := verifyTLSKeyPerms("key_file", cfg.KeyFile); err != nil { + return err + } + if _, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile); err != nil { + return fmt.Errorf("cannot load client certificate/key pair: %w", err) + } + } + return nil +} + // validateCollectorConfig migrates deprecated analytics capture aliases onto the // collector and enforces the collector prerequisite: a consumer (analytics or // traffic logging) requires the collector that feeds it. The collector has no @@ -1565,6 +1815,10 @@ func (c *Config) validateAnalyticsConfig() error { return fmt.Errorf("analytics.publishers.moesif.moesif_base_url must be a valid URL (e.g. https://api.moesif.net), got %q", moesifCfg.BaseURL) } } + case "otel": + if err := validateOTelPublisherConfig(c.Analytics.Publishers.OTel); err != nil { + return err + } default: return fmt.Errorf("unknown publisher type in enabled_publishers: %s", publisherName) } @@ -1870,10 +2124,10 @@ func validateTrafficLogHTTPConfig(cfg TrafficLogHTTPConfig) error { "bounded sender is deferred unbounded memory growth", cfg.QueueCapacity) } switch strings.ToLower(strings.TrimSpace(cfg.OnQueueFull)) { - case TrafficLogQueueDropNew, TrafficLogQueueDropOldest: + case QueueDropNew, QueueDropOldest: default: return fmt.Errorf("on_queue_full must be %q or %q, got %q", - TrafficLogQueueDropNew, TrafficLogQueueDropOldest, cfg.OnQueueFull) + QueueDropNew, QueueDropOldest, cfg.OnQueueFull) } if cfg.RequestTimeout <= 0 { return fmt.Errorf("request_timeout must be positive, got %s", cfg.RequestTimeout) diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go new file mode 100644 index 0000000000..0a248c11b9 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go @@ -0,0 +1,435 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package config + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// otelConfig returns a Config with analytics enabled and only the otel publisher +// selected, so Validate exercises validateOTelPublisherConfig. +func otelConfig(mutate func(*OTelPublisherConfig)) *Config { + cfg := defaultConfig() + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = []string{"otel"} + // The default endpoint is plaintext http://, which now requires an explicit + // opt-in. Granting it here keeps the transport question out of every case + // that is really about batching, retries or TLS material; the gate itself is + // covered by TestValidate_OTelPublisherPlaintextTransport, and a case can + // still switch it back off via mutate. + cfg.Analytics.Publishers.OTel.AllowInsecureTransport = true + mutate(&cfg.Analytics.Publishers.OTel) + return cfg +} + +// Credentials in the endpoint's userinfo (CWE-532). The endpoint reaches the log +// three ways — the startup line, the export-failure line, and the HTTP client's +// own error, which embeds the URL and is wrapped and logged in turn — so the +// password would be on disk three times over. +func TestValidate_OTelPublisherRejectsCredentialsInEndpoint(t *testing.T) { + const password = "sup3rs3cr3t" + + t.Run("user and password are refused", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://svc:" + password + "@collector.example.com:4318/v1/logs" + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain credentials") + // The rejection must not reproduce the leak it exists to prevent. + assert.NotContains(t, err.Error(), password, "the error echoed the password back") + }) + + t.Run("a bare username is refused too", func(t *testing.T) { + // url.Parse sets User for "user@host" with no password, and a username is + // still a credential half worth keeping out of logs. + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://svc@collector.example.com:4318/v1/logs" + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain credentials") + }) + + t.Run("plaintext endpoints are covered as well", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "http://svc:" + password + "@collector.example.com:4318/v1/logs" + o.AllowInsecureTransport = true + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain credentials") + assert.NotContains(t, err.Error(), password) + }) + + t.Run("headers remain the supported way to authenticate", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.Headers = map[string]string{"authorization": "Bearer " + password} + }) + assert.NoError(t, cfg.Validate()) + }) +} + +// The plaintext gate, matching traffic_logging.http.allow_insecure_transport: +// analytics records carry API keys and consumer identity, and every Headers +// value is an intake credential put on the wire on each export. +func TestValidate_OTelPublisherPlaintextTransport(t *testing.T) { + const plaintext = "http://collector.example.com:4318/v1/logs" + + t.Run("http without the flag is refused", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = plaintext + o.AllowInsecureTransport = false + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "allow_insecure_transport is false") + }) + + t.Run("http with the flag is allowed", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = plaintext + o.AllowInsecureTransport = true + }) + assert.NoError(t, cfg.Validate()) + }) + + t.Run("loopback is not exempt", func(t *testing.T) { + // The sibling sink grants no loopback exemption, so neither does this: + // "localhost" inside a container is not the operator's machine. + for _, host := range []string{"http://127.0.0.1:4318/v1/logs", "http://localhost:4318/v1/logs"} { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = host + o.AllowInsecureTransport = false + }) + err := cfg.Validate() + require.Error(t, err, "%s must still require the opt-in", host) + assert.Contains(t, err.Error(), "allow_insecure_transport is false") + } + }) + + t.Run("https needs no flag", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.AllowInsecureTransport = false + }) + assert.NoError(t, cfg.Validate()) + }) + + t.Run("credentialed plaintext is still gated", func(t *testing.T) { + // CWE-319: Headers authenticate to the intake, so plaintext exposes the + // credential itself. Covered by the same gate rather than a second rule. + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = plaintext + o.AllowInsecureTransport = false + o.Headers = map[string]string{"x-api-key": "s3cr3t"} + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "allow_insecure_transport is false") + assert.NotContains(t, err.Error(), "s3cr3t", "the error must not echo the credential") + }) + + t.Run("a non-http scheme names the opt-in", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.Endpoint = "ftp://collector:4318/v1/logs" }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "must be https (or http with allow_insecure_transport)") + }) +} + +// writeSelfSignedPair writes a throwaway self-signed certificate and its key. +func writeSelfSignedPair(t *testing.T) (certPath, keyPath string) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "otel-config-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + + dir := t.TempDir() + certPath = filepath.Join(dir, "cert.pem") + keyPath = filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(certPath, + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)) + require.NoError(t, os.WriteFile(keyPath, + pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600)) + return certPath, keyPath +} + +func TestValidate_OTelPublisher(t *testing.T) { + t.Run("defaults pass", func(t *testing.T) { + assert.NoError(t, otelConfig(func(*OTelPublisherConfig) {}).Validate()) + }) + + t.Run("https endpoint passes", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://otlp.vendor.example.com/v1/logs" + }) + assert.NoError(t, cfg.Validate()) + }) + + rejected := map[string]func(*OTelPublisherConfig){ + "missing endpoint": func(o *OTelPublisherConfig) { o.Endpoint = "" }, + "endpoint without host": func(o *OTelPublisherConfig) { o.Endpoint = "/v1/logs" }, + "non-http scheme": func(o *OTelPublisherConfig) { o.Endpoint = "grpc://collector:4317" }, + "missing service name": func(o *OTelPublisherConfig) { o.ServiceName = "" }, + "zero batch size": func(o *OTelPublisherConfig) { o.BatchSize = 0 }, + "zero queue capacity": func(o *OTelPublisherConfig) { o.QueueCapacity = 0 }, + // A queue smaller than a batch can never fill one. + "queue smaller than batch": func(o *OTelPublisherConfig) { o.QueueCapacity = 10; o.BatchSize = 100 }, + "zero flush interval": func(o *OTelPublisherConfig) { o.FlushInterval = 0 }, + "negative timeout": func(o *OTelPublisherConfig) { o.Timeout = -time.Second }, + } + for name, mutate := range rejected { + t.Run(name, func(t *testing.T) { + err := otelConfig(mutate).Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "analytics.publishers.otel") + }) + } + + // The two accepted values are shared with traffic_logging.http via + // QueueDropNew/QueueDropOldest, so both blocks must keep accepting both. + t.Run("both drop policies are accepted", func(t *testing.T) { + for _, policy := range []string{QueueDropNew, QueueDropOldest, " DROP_OLDEST "} { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.OnQueueFull = policy }) + assert.NoError(t, cfg.Validate(), "policy %q", policy) + } + }) + + // Silently defaulting an unrecognised value would give an operator who asked + // for drop_oldest the opposite behaviour. + t.Run("unknown drop policy is an error", func(t *testing.T) { + for _, policy := range []string{"", "drop", "evict_oldest", "drop_newest"} { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.OnQueueFull = policy }) + err := cfg.Validate() + require.Error(t, err, "policy %q", policy) + assert.Contains(t, err.Error(), "on_queue_full") + } + }) + + // An unknown publisher name must fail rather than be silently skipped. + t.Run("unknown publisher name is an error", func(t *testing.T) { + cfg := otelConfig(func(*OTelPublisherConfig) {}) + cfg.Analytics.EnabledPublishers = []string{"otlp"} + assert.Error(t, cfg.Validate()) + }) + + // Nothing under the block is validated while the publisher is not enabled. + t.Run("not enabled skips validation", func(t *testing.T) { + cfg := defaultConfig() + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = nil + cfg.Analytics.Publishers.OTel.Endpoint = "" + cfg.Analytics.Publishers.OTel.TLS = OTelTLSConfig{CAFile: "/nonexistent/ca.pem"} + assert.NoError(t, cfg.Validate()) + }) +} + +// TLS material is loaded at startup so a bad path fails there rather than on the +// first export, hours later. +func TestValidate_OTelPublisherTLS(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + + t.Run("ca file plus mTLS pair passes", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.internal:4318/v1/logs" + o.TLS = OTelTLSConfig{CAFile: certPath, CertFile: certPath, KeyFile: keyPath} + }) + assert.NoError(t, cfg.Validate()) + }) + + garbage := filepath.Join(t.TempDir(), "garbage.pem") + require.NoError(t, os.WriteFile(garbage, []byte("not a certificate"), 0o600)) + absent := filepath.Join(t.TempDir(), "absent.pem") + + rejected := map[string]OTelTLSConfig{ + "unreadable ca file": {CAFile: absent}, + "ca file with no PEM": {CAFile: garbage}, + "cert without key": {CertFile: certPath}, + "key without cert": {KeyFile: keyPath}, + "unloadable pair": {CertFile: keyPath, KeyFile: certPath}, + } + for name, tlsCfg := range rejected { + t.Run(name, func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.internal:4318/v1/logs" + o.TLS = tlsCfg + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "analytics.publishers.otel.tls") + }) + } + + // insecure_skip_verify only warns — it must not block startup, or an + // operator debugging a cert problem has no way through. + t.Run("insecure_skip_verify warns but passes", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.internal:4318/v1/logs" + o.TLS = OTelTLSConfig{InsecureSkipVerify: true} + }) + assert.NoError(t, cfg.Validate()) + }) +} + +func TestValidate_OTelPublisherRetry(t *testing.T) { + t.Run("defaults are retry-enabled", func(t *testing.T) { + cfg := defaultConfig().Analytics.Publishers.OTel + assert.Equal(t, 3, cfg.MaxRetries) + assert.Equal(t, time.Second, cfg.RetryBackoff) + assert.Equal(t, DefaultOTelRetryAbortQueueRatio, cfg.RetryAbortQueueRatio) + assert.Equal(t, OTelCompressionNone, cfg.Compression) + }) + + // Retries off is a legitimate choice, so 0 must pass while negative fails. + t.Run("zero retries is allowed", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.MaxRetries = 0; o.RetryBackoff = 0 }) + assert.NoError(t, cfg.Validate()) + }) + + rejected := map[string]func(*OTelPublisherConfig){ + "negative retries": func(o *OTelPublisherConfig) { o.MaxRetries = -1 }, + "retries without backoff": func(o *OTelPublisherConfig) { o.MaxRetries = 3; o.RetryBackoff = 0 }, + "negative backoff": func(o *OTelPublisherConfig) { o.MaxRetries = 3; o.RetryBackoff = -time.Second }, + "abort ratio above one": func(o *OTelPublisherConfig) { o.RetryAbortQueueRatio = 1.5 }, + "negative abort ratio": func(o *OTelPublisherConfig) { o.RetryAbortQueueRatio = -0.1 }, + "unknown compression": func(o *OTelPublisherConfig) { o.Compression = "zstd" }, + } + for name, mutate := range rejected { + t.Run(name, func(t *testing.T) { + err := otelConfig(mutate).Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "analytics.publishers.otel") + }) + } + + t.Run("compression values accepted", func(t *testing.T) { + for _, c := range []string{"", OTelCompressionNone, OTelCompressionGzip, " GZIP "} { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.Compression = c }) + assert.NoError(t, cfg.Validate(), "compression %q", c) + } + }) + + // The abort depth is a fraction of capacity, and must never round down to + // zero for a non-zero ratio: that would silently disable the check. + t.Run("abort depth", func(t *testing.T) { + cases := []struct { + capacity int + ratio float64 + want int + }{ + {10000, 0.5, 5000}, + {4, 0.5, 2}, + {1, 0.5, 1}, // rounds to 0, floored to 1 + {10000, 0, 0}, // explicitly disabled + } + for _, tc := range cases { + cfg := OTelPublisherConfig{QueueCapacity: tc.capacity, RetryAbortQueueRatio: tc.ratio} + assert.Equal(t, tc.want, cfg.EffectiveRetryAbortDepth(), + "capacity %d ratio %v", tc.capacity, tc.ratio) + } + }) +} + +// GO-AUTH-018: a group/other-readable client key must abort startup, not warn. +func TestValidate_OTelPublisherRejectsPermissiveKey(t *testing.T) { + for _, mode := range []os.FileMode{0o644, 0o640, 0o604, 0o660, 0o666, 0o777} { + t.Run(fmt.Sprintf("mode_%#o", mode), func(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + require.NoError(t, os.Chmod(keyPath, mode)) + + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.TLS.CertFile = certPath + o.TLS.KeyFile = keyPath + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "allow group/other access to a private key") + }) + } + + for _, mode := range []os.FileMode{0o600, 0o400} { + t.Run(fmt.Sprintf("owner_only_%#o_passes", mode), func(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + require.NoError(t, os.Chmod(keyPath, mode)) + + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.TLS.CertFile = certPath + o.TLS.KeyFile = keyPath + }) + assert.NoError(t, cfg.Validate()) + }) + } +} + +// Permissions are checked before the key is parsed, so a readable key is reported +// as a permissions problem rather than a parse failure. +func TestValidate_OTelPublisherKeyPermsCheckedBeforeParse(t *testing.T) { + certPath, _ := writeSelfSignedPair(t) + badKey := filepath.Join(t.TempDir(), "garbage.key") + require.NoError(t, os.WriteFile(badKey, []byte("not a key"), 0o644)) + + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.TLS.CertFile = certPath + o.TLS.KeyFile = badKey + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "allow group/other access to a private key") + assert.NotContains(t, err.Error(), "cannot load client certificate/key pair") +} + +func TestVerifyTLSKeyPerms(t *testing.T) { + _, keyPath := writeSelfSignedPair(t) + assert.NoError(t, verifyTLSKeyPerms("key_file", keyPath)) + + assert.ErrorContains(t, verifyTLSKeyPerms("key_file", filepath.Join(t.TempDir(), "absent.key")), + "cannot stat") + assert.ErrorContains(t, verifyTLSKeyPerms("key_file", t.TempDir()), "is a directory") +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go new file mode 100644 index 0000000000..9f57afeef9 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The struct-level validation tests build OTelPublisherConfig values in Go, which +// proves the rules but never exercises the koanf tags. A mistyped tag fails +// silently — the key is ignored and the default is used — so an operator sets +// on_queue_full or queue_capacity, sees no error, and gets default behaviour. +// These tests load real TOML through Load() and assert every field. + +func writeOTelTOML(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(p, []byte(body), 0o600)) + return p +} + +// Every OTel publisher key set to a non-default value, so a field that failed to +// bind shows up as its default rather than what the TOML asked for. +func TestLoad_OTelPublisher_EveryKeyBindsFromTOML(t *testing.T) { + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +endpoint = "https://collector.example.com:4318/v1/logs" +service_name = "plumbing-test" +service_version = "9.9.9" +batch_size = 7 +flush_interval = "3s" +queue_capacity = 42 +on_queue_full = "drop_oldest" +timeout = "11s" +max_retries = 5 +retry_backoff = "250ms" +retry_abort_queue_ratio = 0.25 +compression = "gzip" +allow_insecure_transport = true + +[analytics.publishers.otel.headers] +"x-api-key" = "secret-value" + +[analytics.publishers.otel.resource_attributes] +"deployment.environment" = "staging" +"service.namespace" = "gw" +`) + cfg, err := Load(path) + require.NoError(t, err) + + o := cfg.Analytics.Publishers.OTel + assert.Equal(t, "https://collector.example.com:4318/v1/logs", o.Endpoint, "endpoint") + assert.Equal(t, "plumbing-test", o.ServiceName, "service_name") + assert.Equal(t, "9.9.9", o.ServiceVersion, "service_version") + assert.Equal(t, 7, o.BatchSize, "batch_size") + assert.Equal(t, 3*time.Second, o.FlushInterval, "flush_interval") + assert.Equal(t, 42, o.QueueCapacity, "queue_capacity") + assert.Equal(t, QueueDropOldest, o.OnQueueFull, "on_queue_full") + assert.Equal(t, 11*time.Second, o.Timeout, "timeout") + assert.Equal(t, 5, o.MaxRetries, "max_retries") + assert.Equal(t, 250*time.Millisecond, o.RetryBackoff, "retry_backoff") + assert.InDelta(t, 0.25, o.RetryAbortQueueRatio, 1e-9, "retry_abort_queue_ratio") + assert.Equal(t, OTelCompressionGzip, o.Compression, "compression") + assert.True(t, o.AllowInsecureTransport, "allow_insecure_transport") + assert.Equal(t, map[string]string{"x-api-key": "secret-value"}, o.Headers, "headers") + assert.Equal(t, map[string]string{ + "deployment.environment": "staging", + "service.namespace": "gw", + }, o.ResourceAttributes, "resource_attributes") + + // The abort depth an operator actually gets from these two keys together. + assert.Equal(t, 10, o.EffectiveRetryAbortDepth(), "42 * 0.25 truncates to 10") +} + +// The TLS sub-table is its own koanf level, so it binds separately from the +// publisher keys above. +func TestLoad_OTelPublisher_TLSBindsFromTOML(t *testing.T) { + // writeSelfSignedPair (otel_publisher_test.go) emits a self-signed CA cert and + // its key; the same cert serves as ca_file and as the mTLS client cert here, + // which is all the validation needs — it checks the material parses, not that + // it chains to anything. + certPath, keyPath := writeSelfSignedPair(t) + caPath := certPath + + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +endpoint = "https://collector.example.com:4318/v1/logs" + +[analytics.publishers.otel.tls] +ca_file = "`+caPath+`" +cert_file = "`+certPath+`" +key_file = "`+keyPath+`" +insecure_skip_verify = true +`) + cfg, err := Load(path) + require.NoError(t, err) + + tlsCfg := cfg.Analytics.Publishers.OTel.TLS + assert.Equal(t, caPath, tlsCfg.CAFile, "ca_file") + assert.Equal(t, certPath, tlsCfg.CertFile, "cert_file") + assert.Equal(t, keyPath, tlsCfg.KeyFile, "key_file") + assert.True(t, tlsCfg.InsecureSkipVerify, "insecure_skip_verify") +} + +// Keys left out of the TOML must fall back to the shipped defaults rather than +// zero values, which would fail validation or silently disable batching. +func TestLoad_OTelPublisher_OmittedKeysKeepDefaults(t *testing.T) { + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +endpoint = "http://otel-collector:4318/v1/logs" +allow_insecure_transport = true +`) + cfg, err := Load(path) + require.NoError(t, err) + + o := cfg.Analytics.Publishers.OTel + assert.Equal(t, "gateway-runtime", o.ServiceName, "default service_name") + assert.Equal(t, 100, o.BatchSize, "default batch_size") + assert.Equal(t, 5*time.Second, o.FlushInterval, "default flush_interval") + assert.Equal(t, 10000, o.QueueCapacity, "default queue_capacity") + assert.Equal(t, QueueDropNew, o.OnQueueFull, "default on_queue_full") + assert.Equal(t, 10*time.Second, o.Timeout, "default timeout") + assert.Equal(t, 3, o.MaxRetries, "default max_retries") + assert.Equal(t, time.Second, o.RetryBackoff, "default retry_backoff") + assert.InDelta(t, DefaultOTelRetryAbortQueueRatio, o.RetryAbortQueueRatio, 1e-9, "default abort ratio") + assert.Equal(t, OTelCompressionNone, o.Compression, "default compression") +} + +// allow_insecure_transport must default to off. Asserted against an https +// endpoint, so the load succeeds for a reason other than the flag. +func TestLoad_OTelPublisher_InsecureTransportDefaultsOff(t *testing.T) { + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +endpoint = "https://collector.example.com:4318/v1/logs" +`) + cfg, err := Load(path) + require.NoError(t, err) + assert.False(t, cfg.Analytics.Publishers.OTel.AllowInsecureTransport, + "plaintext must be opt-in, never the default") +} + +// A bad value in the TOML must fail the load, not be silently coerced. This is +// what makes the process refuse to start rather than run with defaults. +func TestLoad_OTelPublisher_InvalidTOMLValuesFailClosed(t *testing.T) { + cases := []struct { + name string + body string + // wantErr is a substring the rejection must contain, so a case cannot + // pass on an unrelated error (a duplicate TOML key, say). + wantErr string + }{ + {"unknown drop policy", `on_queue_full = "sometimes"`, "on_queue_full must be"}, + {"queue smaller than batch", "queue_capacity = 5\nbatch_size = 50", "must be >= batch_size"}, + {"zero queue capacity", `queue_capacity = 0`, "queue_capacity must be > 0"}, + {"unknown compression", `compression = "snappy"`, "compression must be"}, + {"abort ratio above one", `retry_abort_queue_ratio = 1.5`, "retry_abort_queue_ratio must be between"}, + {"negative retries", `max_retries = -1`, "max_retries must be >= 0"}, + {"zero flush interval", `flush_interval = "0s"`, "flush_interval must be > 0"}, + {"zero timeout", `timeout = "0s"`, "timeout must be > 0"}, + {"retries without backoff", "max_retries = 2\nretry_backoff = \"0s\"", "retry_backoff must be positive"}, + {"non-http scheme", `endpoint = "ftp://collector:4318/v1/logs"`, "must be https (or http with allow_insecure_transport)"}, + {"plaintext without opt-in", `endpoint = "http://collector.example.com:4318/v1/logs"`, "allow_insecure_transport is false"}, + {"credentials in the endpoint URL", `endpoint = "https://svc:pw@collector.example.com:4318/v1/logs"`, "must not contain credentials"}, + {"missing ca file", `[analytics.publishers.otel.tls]` + "\n" + `ca_file = "/nonexistent/ca.pem"`, "cannot read ca_file"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + base := "endpoint = \"http://otel-collector:4318/v1/logs\"\n" + + "allow_insecure_transport = true\n" + if strings.Contains(tc.body, "endpoint =") { + base = "" // the case sets its own; a second one is a TOML parse error + } + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +`+base+tc.body+"\n") + _, err := Load(path) + require.Error(t, err, "invalid TOML value must fail the load") + assert.Contains(t, err.Error(), tc.wantErr, "must be rejected for the stated reason") + t.Logf("rejected with: %v", err) + }) + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go index 95a1d34977..0c61aa0b5d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go +++ b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go @@ -128,4 +128,6 @@ const ( GuardrailNameMetadataKey = "guardrailName" LLMCostMetadataKey = "x-llm-cost" LLMCostPropertyKey = "llmCost" + RequestModelPropertyKey = "requestModel" // Holds the model named in the request + RequestPathPropertyKey = "requestPath" // Holds the concrete request path, query string removed. ) diff --git a/gateway/gateway-runtime/policy-engine/internal/metrics/metrics.go b/gateway/gateway-runtime/policy-engine/internal/metrics/metrics.go index cc10dc58ec..7b36208092 100644 --- a/gateway/gateway-runtime/policy-engine/internal/metrics/metrics.go +++ b/gateway/gateway-runtime/policy-engine/internal/metrics/metrics.go @@ -70,6 +70,15 @@ var ( TrafficLogFlushDurationSecond HistogramVec TrafficLogWriteErrorsTotal CounterVec + // Analytics publisher metrics, labelled by publisher so a second publisher + // (Moesif, or a future one) reports on the same series rather than its own. + AnalyticsPublishedTotal CounterVec + AnalyticsDroppedTotal CounterVec + AnalyticsQueueDepth GaugeVec + AnalyticsQueueCapacity GaugeVec + AnalyticsExportDurationSeconds HistogramVec + AnalyticsExportErrorsTotal CounterVec + // ResolutionFailuresTotal counts requests whose logical operation could not be // resolved to a policy chain, labelled by resolver name and FailureKind. It // sits alongside RouteLookupFailuresTotal rather than replacing it: that one @@ -364,6 +373,75 @@ func initMetrics() { []string{"sink", "code"}, ) + // Analytics publisher metrics. An analytics event that never reaches its + // destination is invisible to the customer's own dashboards and billing + // views, so dropped_total is the series to alert on; the rest exist to + // diagnose it. Labelled by publisher rather than named per publisher, so + // "are we losing analytics?" is one query regardless of how many are enabled. + AnalyticsPublishedTotal = newCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "analytics_published_total", + Help: "Total number of analytics records successfully delivered, by publisher", + }, + []string{"publisher"}, + ) + + AnalyticsDroppedTotal = newCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "analytics_dropped_total", + Help: "Total number of analytics records dropped, by publisher and reason " + + "(queue_full, send_failed, backpressure, rejected, serialize_failed)", + }, + []string{"publisher", "reason"}, + ) + + AnalyticsQueueDepth = newGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "analytics_queue_depth", + Help: "Current number of analytics records queued for export, by publisher", + }, + []string{"publisher"}, + ) + + // Published so an alert can compare depth against capacity as a RATIO. A + // fixed depth threshold is meaningless on its own: 1000 is 10% of the + // default 10000 queue (fires far too early) and unreachable on a queue + // configured smaller than that (never fires at all). + AnalyticsQueueCapacity = newGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "analytics_queue_capacity", + Help: "Configured capacity of the analytics export queue, by publisher", + }, + []string{"publisher"}, + ) + + // Covers the whole delivery of one batch including retries and backoff, not a + // single request: that total is what holds the export worker, and therefore + // what lets the queue fill behind it. + AnalyticsExportDurationSeconds = newHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: "analytics_export_duration_seconds", + Help: "Duration of an analytics batch delivery including retries, by publisher", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0}, + }, + []string{"publisher"}, + ) + + AnalyticsExportErrorsTotal = newCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "analytics_export_errors_total", + Help: "Total number of analytics export errors, by publisher and code " + + "(HTTP status, or a short error class for non-HTTP failures)", + }, + []string{"publisher", "code"}, + ) + ResolutionFailuresTotal = newCounterVec( prometheus.CounterOpts{ Namespace: namespace, @@ -487,6 +565,14 @@ func initRegistry() { registerGaugeVec(TrafficLogQueueCapacity) registerHistogramVec(TrafficLogFlushDurationSecond) registerCounterVec(TrafficLogWriteErrorsTotal) + + registerCounterVec(AnalyticsPublishedTotal) + registerCounterVec(AnalyticsDroppedTotal) + registerGaugeVec(AnalyticsQueueDepth) + registerGaugeVec(AnalyticsQueueCapacity) + registerHistogramVec(AnalyticsExportDurationSeconds) + registerCounterVec(AnalyticsExportErrorsTotal) + registerCounterVec(ResolutionFailuresTotal) registerCounterVec(RouteResolutionIngestFailuresTotal) diff --git a/gateway/system-policies/analytics/analytics.go b/gateway/system-policies/analytics/analytics.go index d9f7b8e21a..2c606f7219 100644 --- a/gateway/system-policies/analytics/analytics.go +++ b/gateway/system-policies/analytics/analytics.go @@ -24,6 +24,7 @@ const ( CompletionTokenCountMetadataKey = "aitoken:completiontokencount" TotalTokenCountMetadataKey = "aitoken:totaltokencount" ModelIDMetadataKey = "aitoken:modelid" + RequestModelIDMetadataKey = "aitoken:requestmodelid" AIProviderNameMetadataKey = "ai:providername" AIProviderDisplayNameMetadataKey = "ai:providerdisplayname" ApplicationIDMetadataKey = "x-wso2-application-id" @@ -76,6 +77,7 @@ const ( var ( // JSON Path expressions to extract MCP analytics properties from response body JsonRpcMethodJsonPath = "$.method" + JsonRpcIDJsonPath = "$.id" McpCapabilityNameJsonPath = "$.params.name" McpResourceUriJsonPath = "$.params.uri" ProtocolVersionJsonPath = "$.params.protocolVersion" @@ -89,14 +91,27 @@ var ( JsonRpcErrorCodeJsonPath = "$.error.code" ) +// MCP capability kinds, derived from the JSON-RPC method prefix. Emitted on the +// analytics event, so the policy-engine publishers match against these values. +const ( + McpCapabilityTool = "TOOL" + McpCapabilityResource = "RESOURCE" + McpCapabilityPrompt = "PROMPT" +) + // AnalyticsPolicy implements the default analytics data collection process. type AnalyticsPolicy struct{} type McpRequestAnalyticsProperties struct { - JsonRpcMethod string `json:"jsonRpcMethod,omitempty"` - Capability string `json:"capability,omitempty"` - CapabilityName string `json:"capabilityName,omitempty"` - ClientInfo *McpClientInfo `json:"clientInfo,omitempty"` + JsonRpcMethod string `json:"jsonRpcMethod,omitempty"` + JsonRpcID string `json:"jsonRpcId,omitempty"` + Capability string `json:"capability,omitempty"` + // CapabilityName is the target's name, from params.name. Tools and prompts + // are named; resources are not + CapabilityName string `json:"capabilityName,omitempty"` + // ResourceUri is the target of a resources/* method, from params.uri + ResourceUri string `json:"resourceUri,omitempty"` + ClientInfo *McpClientInfo `json:"clientInfo,omitempty"` } type McpClientInfo struct { @@ -400,8 +415,23 @@ func (a *AnalyticsPolicy) OnRequestBody(_ context.Context, ctx *policy.RequestCo } props.JsonRpcMethod = extractString(JsonRpcMethodJsonPath) - props.CapabilityName = extractString(McpCapabilityNameJsonPath) + // A JSON-RPC id may be a string or a number. + if raw, err := utils.ExtractValueFromJsonpath(mcpPayload, JsonRpcIDJsonPath); err == nil && raw != nil { + switch id := raw.(type) { + case string: + props.JsonRpcID = id + case float64: + props.JsonRpcID = strconv.FormatInt(int64(id), 10) + } + } props.Capability = deriveMCPCapability(props.JsonRpcMethod) + // resources/* addresses its target by URI at params.uri; tools/* and + // prompts/* name theirs at params.name. + if props.Capability == McpCapabilityResource { + props.ResourceUri = extractString(McpResourceUriJsonPath) + } else { + props.CapabilityName = extractString(McpCapabilityNameJsonPath) + } clientInfo := McpClientInfo{ RequestedProtocolVersion: extractStringFromJsonpath(mcpPayload, ProtocolVersionJsonPath), @@ -882,6 +912,9 @@ func populateTokenAnalyticsMetadata(analyticsMetadata map[string]any, tokenInfo } else if tokenInfo.RequestModel != nil { analyticsMetadata[ModelIDMetadataKey] = *tokenInfo.RequestModel } + if tokenInfo.RequestModel != nil { + analyticsMetadata[RequestModelIDMetadataKey] = *tokenInfo.RequestModel + } if tokenInfo.ProviderName != nil { analyticsMetadata[AIProviderNameMetadataKey] = *tokenInfo.ProviderName } @@ -1202,11 +1235,11 @@ func extractBoolFromJsonpath(payload map[string]interface{}, path string) (bool, func deriveMCPCapability(method string) string { switch { case strings.HasPrefix(method, "tools/"): - return "TOOL" + return McpCapabilityTool case strings.HasPrefix(method, "resources/"): - return "RESOURCE" + return McpCapabilityResource case strings.HasPrefix(method, "prompts/"): - return "PROMPT" + return McpCapabilityPrompt default: return "" } diff --git a/gateway/system-policies/analytics/analytics_test.go b/gateway/system-policies/analytics/analytics_test.go index 1b4eaa0895..2f7e492460 100644 --- a/gateway/system-policies/analytics/analytics_test.go +++ b/gateway/system-policies/analytics/analytics_test.go @@ -416,3 +416,87 @@ func TestExtractMCPResponseAnalyticsProps_IsError(t *testing.T) { }) } } + +// A resources/* method addresses its target by URI at params.uri, while tools/* +// and prompts/* name theirs at params.name. Extracting only params.name left +// mcp.resource.uri permanently empty for every resource read. +func TestOnRequestBody_MCPCapabilityTarget(t *testing.T) { + cases := []struct { + name string + body string + wantName string + wantURI string + }{ + { + name: "tools/call carries a name", + body: `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search_docs"}}`, + wantName: "search_docs", + }, + { + name: "prompts/get carries a name", + body: `{"jsonrpc":"2.0","id":2,"method":"prompts/get","params":{"name":"summarize"}}`, + wantName: "summarize", + }, + { + name: "resources/read carries a uri", + body: `{"jsonrpc":"2.0","id":3,"method":"resources/read","params":{"uri":"file:///docs/readme.md"}}`, + wantURI: "file:///docs/readme.md", + }, + { + // resources/list takes no target at all; neither field may be invented. + name: "resources/list carries neither", + body: `{"jsonrpc":"2.0","id":4,"method":"resources/list","params":{}}`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + props := mcpRequestProps(t, tc.body) + if props.CapabilityName != tc.wantName { + t.Errorf("capabilityName = %q, want %q", props.CapabilityName, tc.wantName) + } + if props.ResourceUri != tc.wantURI { + t.Errorf("resourceUri = %q, want %q", props.ResourceUri, tc.wantURI) + } + }) + } +} + +// A resources/read request that also happens to carry params.name must not have +// it mistaken for the resource's identity. +func TestOnRequestBody_MCPResourceIgnoresName(t *testing.T) { + props := mcpRequestProps(t, + `{"jsonrpc":"2.0","id":5,"method":"resources/read",`+ + `"params":{"uri":"file:///a.md","name":"not-the-target"}}`) + + if props.ResourceUri != "file:///a.md" { + t.Errorf("resourceUri = %q, want file:///a.md", props.ResourceUri) + } + if props.CapabilityName != "" { + t.Errorf("capabilityName = %q, want empty for a resource", props.CapabilityName) + } +} + +// mcpRequestProps runs OnRequestBody over an MCP request body and returns the +// properties it emitted onto analytics metadata. +func mcpRequestProps(t *testing.T, body string) McpRequestAnalyticsProperties { + t.Helper() + action := (&AnalyticsPolicy{}).OnRequestBody(context.Background(), &policy.RequestContext{ + SharedContext: &policy.SharedContext{APIKind: policy.APIKindMCP}, + Body: &policy.Body{Content: []byte(body)}, + }, nil) + + mods, ok := action.(policy.UpstreamRequestModifications) + if !ok { + t.Fatalf("expected UpstreamRequestModifications, got %T", action) + } + raw, ok := mods.AnalyticsMetadata["mcp_request_properties"].(string) + if !ok { + t.Fatalf("mcp_request_properties absent or not a string: %#v", mods.AnalyticsMetadata) + } + var props McpRequestAnalyticsProperties + if err := json.Unmarshal([]byte(raw), &props); err != nil { + t.Fatalf("unmarshal mcp_request_properties: %v", err) + } + return props +} diff --git a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml index 26bb9fcb58..ca125e2dd7 100644 --- a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml +++ b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml @@ -370,12 +370,100 @@ data: {{- end }} {{- if .Values.gateway.config.analytics.publishers }} {{- range $name, $publisher := .Values.gateway.config.analytics.publishers }} + {{- if eq $name "otel" }} + {{- with $publisher }} + {{- /* headers carry the intake credential and config.toml renders into a ConfigMap, not a Secret — require an {{ env }}/{{ file }} token, as traffic_logging.http.auth does. */ -}} + {{- $allowPlain := false -}} + {{- if hasKey . "allow_plaintext_credentials" -}} + {{- $allowPlain = eq (toString .allow_plaintext_credentials) "true" -}} + {{- end -}} + {{- if not $allowPlain -}} + {{- $suffixed := "^[^{}]*\\{\\{-? *(env|file) [^{}]*\\}\\}$" -}} + {{- range $header, $value := .headers -}} + {{- if not (regexMatch $suffixed (trim (toString $value))) -}} + {{- fail (printf "gateway.config.analytics.publishers.otel.headers.%s holds a literal value, which would be written in plaintext into the gateway ConfigMap. Supply it as an interpolation token resolved at runtime instead, e.g. '{{ env \"APIP_GW_OTEL_INTAKE_TOKEN\" }}' or '{{ file \"/secrets/gateway-runtime/otel-token\" }}' (allowed source dirs: /etc/gateway-runtime, /secrets/gateway-runtime). Set gateway.config.analytics.publishers.otel.allow_plaintext_credentials=true only for local development." $header) -}} + {{- end -}} + {{- end -}} + {{- end }} + [analytics.publishers.otel] + {{- if .endpoint }} + endpoint = {{ .endpoint | quote }} + {{- end }} + {{- if kindIs "bool" .allow_insecure_transport }} + allow_insecure_transport = {{ .allow_insecure_transport }} + {{- end }} + {{- if .service_name }} + service_name = {{ .service_name | quote }} + {{- end }} + {{- if .service_version }} + service_version = {{ .service_version | quote }} + {{- end }} + {{- if not (kindIs "invalid" .batch_size) }} + batch_size = {{ .batch_size | int64 }} + {{- end }} + {{- if not (kindIs "invalid" .flush_interval) }} + flush_interval = {{ .flush_interval | quote }} + {{- end }} + {{- if not (kindIs "invalid" .queue_capacity) }} + queue_capacity = {{ .queue_capacity | int64 }} + {{- end }} + {{- if .on_queue_full }} + on_queue_full = {{ .on_queue_full | quote }} + {{- end }} + {{- if not (kindIs "invalid" .timeout) }} + timeout = {{ .timeout | quote }} + {{- end }} + {{- if not (kindIs "invalid" .max_retries) }} + max_retries = {{ .max_retries | int64 }} + {{- end }} + {{- if not (kindIs "invalid" .retry_backoff) }} + retry_backoff = {{ .retry_backoff | quote }} + {{- end }} + {{- if not (kindIs "invalid" .retry_abort_queue_ratio) }} + retry_abort_queue_ratio = {{ .retry_abort_queue_ratio }} + {{- end }} + {{- if .compression }} + compression = {{ .compression | quote }} + {{- end }} + {{- with .headers }} + + [analytics.publishers.otel.headers] + {{- range $key, $value := . }} + {{ $key | quote }} = {{ $value | quote }} + {{- end }} + {{- end }} + {{- with .resource_attributes }} + + [analytics.publishers.otel.resource_attributes] + {{- range $key, $value := . }} + {{ $key | quote }} = {{ $value | quote }} + {{- end }} + {{- end }} + {{- with .tls }} + + [analytics.publishers.otel.tls] + {{- if .ca_file }} + ca_file = {{ .ca_file | quote }} + {{- end }} + {{- if .cert_file }} + cert_file = {{ .cert_file | quote }} + {{- end }} + {{- if .key_file }} + key_file = {{ .key_file | quote }} + {{- end }} + {{- if kindIs "bool" .insecure_skip_verify }} + insecure_skip_verify = {{ .insecure_skip_verify }} + {{- end }} + {{- end }} + {{- end }} + {{- else }} [analytics.publishers.{{ $name }}] {{- range $key, $value := $publisher }} {{ $key }} = {{ kindIs "string" $value | ternary ($value | quote) $value }} {{- end }} {{- end }} {{- end }} + {{- end }} {{- if .Values.gateway.config.analytics.grpc_event_server }} [analytics.grpc_event_server] host = "127.0.0.1"