From 170f6bc62c7c2b855ed444af5dbb4d37b7582fc3 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Mon, 7 Sep 2026 14:03:36 +0530 Subject: [PATCH 01/20] Adding initial implementation for otel publisher --- gateway/build-manifest.yaml | 2 +- gateway/configs/config-template.toml | 59 ++ .../internal/analytics/analytics.go | 19 + .../internal/analytics/publishers/otel.go | 738 ++++++++++++++++++ .../analytics/publishers/otel_test.go | 681 ++++++++++++++++ .../policy-engine/internal/config/config.go | 126 +++ .../internal/config/otel_publisher_test.go | 172 ++++ .../internal/constants/constants.go | 1 + .../system-policies/analytics/analytics.go | 15 + 9 files changed, 1812 insertions(+), 1 deletion(-) create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go create mode 100644 gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go diff --git a/gateway/build-manifest.yaml b/gateway/build-manifest.yaml index ed4e647f5a..0a9c348929 100644 --- a/gateway/build-manifest.yaml +++ b/gateway/build-manifest.yaml @@ -1,7 +1,7 @@ version: v1 policies: - name: advanced-ratelimit - version: v1.1.2 + version: v1.2.0 gomodule: github.com/wso2/gateway-controllers/policies/advanced-ratelimit@v1 - name: analytics-header-filter version: v1.0.1 diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index df5fb32b39..d18388ad27 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -789,6 +789,18 @@ allow_payloads = false # during validation (with a warning). Prefer setting [collector] directly. send_request_body = false send_response_body = false +# Publishers each enabled event is delivered to. Any combination of: +# +# moesif maps the event onto Moesif's event model and posts it with the Moesif +# SDK. Configured under [analytics.publishers.moesif] below. +# otel exports the event to an OpenTelemetry collector as an OTLP log record +# (one record per transaction, event.name=wso2.api.transaction) over +# OTLP/HTTP. From the collector, any OTLP-capable backend — Datadog, +# Splunk, Elastic, Dynatrace, Grafana — is one exporter block away. +# Configured under [analytics.publishers.otel] below. +# +# Publishers are additive and independent: ["moesif", "otel"] delivers every event +# to both. An unknown name is a startup error, not a silent no-op. enabled_publishers = ["moesif"] [analytics.publishers.moesif] @@ -799,6 +811,53 @@ event_queue_size = 10000 batch_size = 50 timer_wakeup_seconds = 3 +[analytics.publishers.otel] +# Full OTLP/HTTP logs URL, including the /v1/logs path. Point it at a collector +# you run, or directly at a vendor's OTLP intake. +endpoint = "http://otel-collector:4318/v1/logs" +# service.name / service.version on the OTLP resource. +service_name = "policy-engine" +service_version = "" +# Record count that triggers an export before flush_interval elapses. +batch_size = 100 +# How long a record waits when traffic is too slow to fill a batch. +flush_interval = "5s" +# Records held in memory when the endpoint is slow. Must be >= batch_size. +# Once full, records are dropped and counted rather than growing unbounded. +queue_size = 10000 +# Bounds a single export attempt. +timeout = "10s" + +# ==== UNCOMMENT WHEN REQUIRED ================================ +# Headers sent on every export, for a vendor OTLP intake that authenticates by +# header. Values are secrets: they are never logged. +# [analytics.publishers.otel.headers] +# X-Moesif-Application-Id = '{{ env "MOESIF_APP_ID" "" }}' +# Extra OTLP resource attributes, e.g. deployment.environment.name. +# [analytics.publishers.otel.resource_attributes] +# "deployment.environment.name" = "prod" + +# TLS to the OTLP endpoint. Ignored when `endpoint` is http. +[analytics.publishers.otel.tls] +# PEM bundle used to verify the endpoint's certificate. Empty means the system +# trust store — correct for a vendor's OTLP intake, usually wrong for an +# in-cluster collector fronted by a private CA. +ca_file = "" +# Client certificate and key for mTLS, in PEM form. Both or neither: setting one +# without the other is a startup error rather than a silently unauthenticated +# connection. Both files are read and parsed at startup, so a wrong path or a +# mismatched pair fails immediately instead of at the first export. +# +# cert_file = "/secrets/gateway-runtime/otel-client.crt" +# key_file = "/secrets/gateway-runtime/otel-client.key" +cert_file = "" +key_file = "" +# Disables verification of the endpoint's certificate. Off by default; when on, +# startup logs a warning naming the endpoint, because analytics records carry +# request metadata and, with collector body capture enabled, payloads. +insecure_skip_verify = false + + # ============================================================================= # TRAFFIC LOGGING (consumer — enabling it activates the collector) # ============================================================================= diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 97ceda48a5..448c15ad16 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -63,6 +63,8 @@ const ( DefaultAnalyticsPublisher = "default" // MoesifAnalyticsPublisher represents the Moesif analytics publisher. MoesifAnalyticsPublisher = "moesif" + // OTelAnalyticsPublisher represents the OpenTelemetry analytics publisher + OTelAnalyticsPublisher = "otel" // HeaderKeys represents the header keys. RequestHeadersKey = "request_headers" @@ -82,6 +84,10 @@ const ( // AIProviderAPIVersionMetadataKey represents the AI provider API version metadata key. AIProviderAPIVersionMetadataKey string = "ai:providerversion" + // RequestModelIDMetadataKey represents the model named in the request + // (Separate from ModelIDMetadataKey (which resolves to the response model). + RequestModelIDMetadataKey string = "aitoken:requestmodelid" + // UserIDMetadataKey represents the user ID metadata key for analytics. UserIDMetadataKey string = "x-wso2-user-id" @@ -122,6 +128,16 @@ func NewAnalytics(cfg *config.Config) *Analytics { publishers = append(publishers, publisher) slog.Info("Moesif publisher added") } + case OTelAnalyticsPublisher: + publisher, err := analytics_publisher.NewOTel(&analyticsCfg.Publishers.OTel) + if err != nil { + // Fail closed on invalid TLS material to avoid a healthy-looking gateway + // silently exporting nothing. Validation already confirms the material loads. + slog.Error("Failed to initialize the OTel analytics publisher; refusing to start", "error", err) + panic(fmt.Sprintf("otel analytics publisher configuration is unusable: %v", err)) + } + publishers = append(publishers, publisher) + slog.Info("OTel publisher added") default: slog.Warn("Unknown publisher type", "type", publisherName) } @@ -527,6 +543,9 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E aiMetadata.LLMCost = parsedLLMCost } event.Properties["aiMetadata"] = aiMetadata + if requestModel := keyValuePairsFromMetadata[RequestModelIDMetadataKey]; requestModel != "" { + event.Properties[constants.RequestModelPropertyKey] = requestModel + } aiTokenUsage := dto.AITokenUsage{} // Prompt tokens diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go new file mode 100644 index 0000000000..210d19f6e7 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -0,0 +1,738 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/constants" +) + +// Identifiers with more than one reader. Individual attribute names are written +// as literals at their single call site: sharing them with the tests would let a +// typo satisfy its own assertion. +const ( + // otelAttrNamespace prefixes every attribute OpenTelemetry does not define. + otelAttrNamespace = "wso2" + // otelScopeName is the InstrumentationScope downstream pipelines route on. It + // must match the collector's scope filter; a mismatch fails silently. + otelScopeName = otelAttrNamespace + ".analytics" + // otelEventName is emitted both as the record's eventName and as an + // event.name attribute — collectors before ~v0.117 drop the former. + otelEventName = otelAttrNamespace + ".api.transaction" + + otelSeverityNumberInfo = 9 + otelSeverityTextInfo = "INFO" + + // otelCloseFlushTimeout bounds the shutdown flush when the caller's context + // carries no deadline. + otelCloseFlushTimeout = 5 * time.Second +) + +// ns qualifies an attribute name with the WSO2 namespace. +func ns(name string) string { return otelAttrNamespace + "." + name } + +// OTel exports analytics events to an OpenTelemetry collector as OTLP log +// records ("OTel Events": log records carrying an event.name), over OTLP/HTTP +// with a JSON body. +// +// The logs signal is used rather than traces or metrics because it is the only +// OTLP signal that represents one discrete transaction with per-consumer +// attributes intact: metrics aggregate the transaction away and impose a +// cardinality ceiling, and traces are sampled by design. +// +// The OTLP wire format is built directly rather than through the OpenTelemetry Go +// SDK, whose logs signal is still pre-1.0 (sdk/log v0.x). The OTLP/HTTP JSON +// encoding it would produce is stable, so only the transport would change. +// +// Publish never blocks and never performs I/O on the caller's goroutine: the ALS +// ingest path calls it, and blocking there backpressures Envoy's access-log +// stream. Records go to a bounded queue drained by one worker; a full queue drops +// and counts rather than growing. +type OTel struct { + cfg config.OTelPublisherConfig + client *http.Client + + queue chan *otelLogRecord + + stop chan struct{} + workerDone chan struct{} + closeOnce sync.Once + closeErr error + + droppedMu sync.Mutex + dropped int +} + +// NewOTel creates the OTLP-logs publisher and starts its exporting worker. +// cfg is assumed validated by config.validateOTelPublisherConfig; the TLS +// material is loaded again here, so this constructor fails closed rather than +// starting a publisher that can never reach its endpoint. +func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { + if cfg == nil { + return nil, fmt.Errorf("config is nil") + } + + tlsCfg, err := buildOTelTLSConfig(cfg.TLS) + if err != nil { + return nil, err + } + + o := &OTel{ + cfg: *cfg, + client: &http.Client{ + Timeout: cfg.Timeout, + Transport: &http.Transport{ + TLSClientConfig: tlsCfg, + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 90 * time.Second, + }, + // Never auto-follow a redirect: the target is chosen by the endpoint, + // not the operator, and following it would send analytics records to a + // destination nobody configured. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + queue: make(chan *otelLogRecord, cfg.QueueSize), + stop: make(chan struct{}), + workerDone: make(chan struct{}), + } + go o.run() + + if u, err := url.Parse(cfg.Endpoint); err == nil && u.Scheme == "http" && !isLoopbackHost(u.Hostname()) { + slog.Warn("OTel publisher is exporting analytics over plaintext HTTP to a non-loopback endpoint", + "endpoint", cfg.Endpoint) + } + // Headers are deliberately omitted: they carry credentials. + slog.Info("OTel analytics publisher started", + "endpoint", cfg.Endpoint, "batchSize", cfg.BatchSize, + "flushInterval", cfg.FlushInterval, "queueSize", cfg.QueueSize) + return o, nil +} + +// buildOTelTLSConfig assembles the client TLS configuration. +// +// X25519MLKEM768 leads CurvePreferences per the repository's post-quantum +// standard; the classical curves stay listed after it so an endpoint that does +// not yet offer the hybrid still completes a handshake. +func buildOTelTLSConfig(cfg config.OTelTLSConfig) (*tls.Config, error) { + out := &tls.Config{ + MinVersion: tls.VersionTLS12, + CurvePreferences: []tls.CurveID{ + tls.X25519MLKEM768, tls.X25519, tls.CurveP256, tls.CurveP384, + }, + InsecureSkipVerify: cfg.InsecureSkipVerify, // #nosec G402 -- off by default; opt-in warns at startup + } + + if cfg.CAFile != "" { + pem, err := os.ReadFile(cfg.CAFile) + if err != nil { + return nil, fmt.Errorf("tls: cannot read ca_file %q: %w", cfg.CAFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("tls: ca_file %q contains no usable PEM certificate", cfg.CAFile) + } + out.RootCAs = pool + } + + if (cfg.CertFile == "") != (cfg.KeyFile == "") { + return nil, fmt.Errorf("tls: cert_file and key_file must be set together for mTLS") + } + if cfg.CertFile != "" { + pair, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile) + if err != nil { + return nil, fmt.Errorf("tls: cannot load client certificate/key pair: %w", err) + } + out.Certificates = []tls.Certificate{pair} + } + return out, nil +} + +func isLoopbackHost(host string) bool { + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// Publish converts the event to an OTLP log record and enqueues it. +func (o *OTel) Publish(event *dto.Event) { + if event == nil { + return + } + select { + case o.queue <- o.buildRecord(event): + default: + // The collector is not keeping up. Dropping the newest record preserves + // the queued older ones; analytics is strictly downstream of request + // handling, so a drop must never surface to the client. + o.droppedMu.Lock() + o.dropped++ + count := o.dropped + o.droppedMu.Unlock() + if count == 1 || count%100 == 0 { + slog.Warn("OTel publisher queue full; dropping analytics event", + "droppedTotal", count, "queueSize", o.cfg.QueueSize) + } + } +} + +// run drains the queue, exporting on a full batch or on the flush interval. +func (o *OTel) run() { + defer close(o.workerDone) + + ticker := time.NewTicker(o.cfg.FlushInterval) + defer ticker.Stop() + + batch := make([]*otelLogRecord, 0, o.cfg.BatchSize) + flush := func() { + if len(batch) == 0 { + return + } + o.export(batch) + batch = batch[:0] + } + + for { + select { + case record := <-o.queue: + batch = append(batch, record) + if len(batch) >= o.cfg.BatchSize { + flush() + } + case <-ticker.C: + flush() + case <-o.stop: + // Drain what is still queued so a rolling update does not lose it. + for drained := true; drained; { + select { + case record := <-o.queue: + batch = append(batch, record) + if len(batch) >= o.cfg.BatchSize { + flush() + } + default: + drained = false + } + } + flush() + return + } + } +} + +// Close stops the worker and flushes what is buffered, satisfying Closer. +func (o *OTel) Close(ctx context.Context) error { + o.closeOnce.Do(func() { + close(o.stop) + + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, otelCloseFlushTimeout) + defer cancel() + } + + select { + case <-o.workerDone: + case <-ctx.Done(): + o.closeErr = fmt.Errorf("otel publisher shutdown timed out: %w", ctx.Err()) + } + + o.droppedMu.Lock() + dropped := o.dropped + o.droppedMu.Unlock() + slog.Info("OTel analytics publisher stopped", "droppedTotal", dropped) + }) + return o.closeErr +} + +// export POSTs one batch as a single OTLP/HTTP logs request. +func (o *OTel) export(batch []*otelLogRecord) { + records := make([]otelLogRecord, 0, len(batch)) + for _, r := range batch { + records = append(records, *r) + } + + resource := newOTelAttrs(). + str("service.name", o.cfg.ServiceName). + str("service.version", o.cfg.ServiceVersion) + for k, v := range o.cfg.ResourceAttributes { + resource.str(k, v) + } + + body, err := json.Marshal(otelExportRequest{ + ResourceLogs: []otelResourceLogs{{ + Resource: otelResource{Attributes: resource.list()}, + ScopeLogs: []otelScopeLogs{{ + Scope: otelScope{Name: otelScopeName}, + LogRecords: records, + }}, + }}, + }) + if err != nil { + slog.Error("OTel publisher failed to marshal OTLP payload", "error", err, "records", len(records)) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), o.cfg.Timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, o.cfg.Endpoint, bytes.NewReader(body)) + if err != nil { + slog.Error("OTel publisher failed to build OTLP request", "error", err) + return + } + req.Header.Set("Content-Type", "application/json") + for k, v := range o.cfg.Headers { + req.Header.Set(k, v) + } + + resp, err := o.client.Do(req) + if err != nil { + slog.Error("OTel publisher export failed", "error", err, "endpoint", o.cfg.Endpoint, "records", len(records)) + return + } + defer resp.Body.Close() + + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + slog.Error("OTel collector rejected the export", + "status", resp.StatusCode, "records", len(records), "response", string(respBody)) + return + } + slog.Debug("OTel publisher exported analytics events", "records", len(records), "status", resp.StatusCode) +} + +// buildRecord maps the canonical analytics event onto one OTLP log record. +// +// Naming follows stable OpenTelemetry conventions where they exist, the GenAI and +// MCP conventions (both Development-stability, and now maintained in +// open-telemetry/semantic-conventions-genai) for AI fields, and the wso2.* +// namespace where OpenTelemetry defines nothing — API-product concepts and cost, +// chiefly. See gateway/spec/analytics-otel-attribute-mapping.md. +func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { + attrs := newOTelAttrs() + attrs.str("event.name", otelEventName) + + // HTTP. APIResourceTemplate already carries the full path including the API + // context; the context is only a fallback when there is no template. + route := "" + if event.Operation != nil { + route = event.Operation.APIResourceTemplate + attrs.str("http.request.method", event.Operation.APIMethod) + attrs.str("http.route", route) + } + if route == "" && event.API != nil { + route = event.API.APIContext + } + attrs.str("url.path", route) + attrs.i64("http.response.status_code", int64(event.ProxyResponseCode)) + attrs.str("client.address", event.UserIP) + attrs.str("user_agent.original", event.UserAgentHeader) + attrs.anyInt("http.request.body.size", event.Properties["requestSize"]) + attrs.anyInt("http.response.body.size", event.Properties["responseSize"]) + attrs.anyStr(ns("response.content_type"), event.Properties["responseContentType"]) + + // API identity. + if event.API != nil { + attrs.str(ns("api.id"), event.API.APIID) + attrs.str(ns("api.name"), event.API.APIName) + attrs.str(ns("api.version"), event.API.APIVersion) + attrs.str(ns("api.context"), event.API.APIContext) + attrs.str(ns("api.type"), event.API.APIType) + attrs.str(ns("api.subtype"), event.API.SubType) + attrs.str(ns("project.id"), event.API.ProjectID) + attrs.str(ns("organization.id"), event.API.OrganizationID) + attrs.str(ns("environment.id"), event.API.EnvironmentID) + } + + // Consumer identity. + attrs.anyStr("user.id", event.Properties[dto.PropKeyAuthUserID]) + attrs.str("user.name", event.UserName) + if event.Application != nil { + attrs.str(ns("application.id"), event.Application.ApplicationID) + attrs.str(ns("application.name"), event.Application.ApplicationName) + attrs.str(ns("application.owner"), event.Application.ApplicationOwner) + attrs.str(ns("application.key_type"), event.Application.KeyType) + } + if event.Subscription != nil { + attrs.str(ns("subscription.id"), event.Subscription.BillingSubscriptionID) + attrs.str(ns("subscription.customer.id"), event.Subscription.BillingCustomerID) + attrs.str(ns("subscription.status"), event.Subscription.Status) + attrs.str(ns("subscription.plan"), event.Subscription.PlanName) + } + if event.MetaInfo != nil { + attrs.str(ns("correlation.id"), event.MetaInfo.CorrelationID) + attrs.str(ns("gateway.type"), event.MetaInfo.GatewayType) + attrs.str(ns("region.id"), event.MetaInfo.RegionID) + } + + // Latency. OpenTelemetry has no log attribute for these. + if event.Latencies != nil { + attrs.i64(ns("latency.response_ms"), event.Latencies.ResponseLatency) + attrs.i64(ns("latency.backend_ms"), event.Latencies.BackendLatency) + attrs.i64(ns("latency.request_mediation_ms"), event.Latencies.RequestMediationLatency) + attrs.i64(ns("latency.response_mediation_ms"), event.Latencies.ResponseMediationLatency) + attrs.i64(ns("latency.duration_ms"), event.Latencies.Duration) + } + + // Upstream outcome. Destination is authority+path, so only its authority + // half is a server address. + if event.Target != nil { + attrs.str(ns("upstream.destination"), event.Target.Destination) + if authority, _, found := strings.Cut(event.Target.Destination, "/"); found || authority != "" { + if host, port, err := net.SplitHostPort(authority); err == nil { + attrs.str("server.address", host) + if p, convErr := strconv.Atoi(port); convErr == nil { + attrs.i64("server.port", int64(p)) + } + } else { + attrs.str("server.address", authority) + } + } + attrs.i64(ns("upstream.response.status_code"), int64(event.Target.TargetResponseCode)) + attrs.str(ns("upstream.response.detail"), event.Target.ResponseCodeDetail) + attrs.b(ns("cache.hit"), event.Target.ResponseCacheHit) + } + + // Faults. error.type is the one stable error attribute. + attrs.str("error.type", event.ErrorType) + if event.Error != nil { + attrs.i64(ns("error.code"), int64(event.Error.ErrorCode)) + attrs.str(ns("error.sub_category"), string(event.Error.ErrorMessage)) + } + + // Payloads, only present when body capture is enabled on the collector. + attrs.anyStr(ns("request.body"), event.Properties[dto.PropKeyRequestPayload]) + attrs.anyStr(ns("response.body"), event.Properties[dto.PropKeyResponsePayload]) + + o.appendAIAttributes(event, attrs, route) + o.appendMCPAttributes(event, attrs) + + now := time.Now() + requestTime := event.RequestTimestamp + if requestTime.IsZero() { + requestTime = now + } + + body := "api.transaction" + if event.Operation != nil && route != "" { + body = event.Operation.APIMethod + " " + route + } + + return &otelLogRecord{ + TimeUnixNano: strconv.FormatInt(requestTime.UnixNano(), 10), + ObservedTimeUnixNano: strconv.FormatInt(now.UnixNano(), 10), + SeverityNumber: otelSeverityNumberInfo, + SeverityText: otelSeverityTextInfo, + EventName: otelEventName, + Body: otelAnyValue{StringValue: &body}, + Attributes: attrs.list(), + } +} + +// appendAIAttributes maps the AI metadata the pipeline stashes in +// Event.Properties onto GenAI conventions. Cost has no GenAI equivalent. +func (o *OTel) appendAIAttributes(event *dto.Event, attrs *otelAttrs, route string) { + if event.Properties == nil { + return + } + + if md, ok := event.Properties["aiMetadata"].(dto.AIMetadata); ok { + attrs.str("gen_ai.provider.name", otelGenAIProviderName(md.VendorName)) + attrs.str(ns("gen_ai.provider.template_name"), md.VendorName) + // aitoken:modelid is the response model when the provider returns one, + // falling back to the request model. + attrs.str("gen_ai.response.model", md.Model) + attrs.str(ns("gen_ai.provider.api_version"), md.VendorVersion) + switch cost := md.LLMCost.(type) { + case float64: + attrs.f64(ns("gen_ai.cost.total"), cost) + case string: + attrs.str(ns("gen_ai.cost.total"), cost) + } + } + attrs.anyStr("gen_ai.request.model", event.Properties[constants.RequestModelPropertyKey]) + + if usage, ok := event.Properties["aiTokenUsage"].(dto.AITokenUsage); ok { + attrs.i64("gen_ai.usage.input_tokens", int64(usage.PromptToken)) + attrs.i64("gen_ai.usage.output_tokens", int64(usage.CompletionToken)) + attrs.i64(ns("gen_ai.usage.total_tokens"), int64(usage.TotalToken)) + } + + if op := otelGenAIOperationName(route); op != "" { + attrs.str("gen_ai.operation.name", op) + } + + attrs.anyBool(ns("gen_ai.egress"), event.Properties["isEgress"]) + attrs.anyBool(ns("guardrail.hit"), event.Properties[constants.GuardrailHitMetadataKey]) + attrs.anyStr(ns("guardrail.name"), event.Properties[constants.GuardrailNameMetadataKey]) +} + +// appendMCPAttributes flattens Properties["mcpAnalytics"] onto MCP conventions. +// The capability determines which attribute the capability name belongs on. +func (o *OTel) appendMCPAttributes(event *dto.Event, attrs *otelAttrs) { + mcp, ok := event.Properties["mcpAnalytics"].(map[string]interface{}) + if !ok { + return + } + + attrs.anyStr("mcp.method.name", mcp["jsonRpcMethod"]) + attrs.anyStr("mcp.session.id", mcp["sessionId"]) + attrs.anyStr("jsonrpc.request.id", mcp["jsonRpcId"]) + + capabilityName, _ := mcp["capabilityName"].(string) + switch capability, _ := mcp["capability"].(string); capability { + case "TOOL": + attrs.str("gen_ai.tool.name", capabilityName) + case "RESOURCE": + attrs.str("mcp.resource.uri", capabilityName) + case "PROMPT": + attrs.str("gen_ai.prompt.name", capabilityName) + } + + // A JSON-RPC error code is a string in rpc.response.status_code. + switch code := mcp["errorCode"].(type) { + case int: + attrs.str("rpc.response.status_code", strconv.Itoa(code)) + case float64: + attrs.str("rpc.response.status_code", strconv.FormatInt(int64(code), 10)) + case string: + attrs.str("rpc.response.status_code", code) + } + if isError, ok := mcp["isError"].(bool); ok && isError { + attrs.strIfEmpty("error.type", "mcp_error") + } + + attrs.anyStr("mcp.protocol.version", mcp["protocolVersion"]) + attrs.anyStr(ns("mcp.client.requested_protocol_version"), mcp["requestedProtocolVersion"]) + attrs.anyStr(ns("mcp.client.name"), mcp["name"]) + attrs.anyStr(ns("mcp.client.version"), mcp["version"]) +} + +// otelGenAIProviderName maps a WSO2 LLM provider template name onto the +// gen_ai.provider.name enum. An unrecognised provider yields "" so the enum +// attribute is left unset rather than carrying a non-member value; the raw name +// is always kept in wso2.gen_ai.provider.template_name. +func otelGenAIProviderName(templateName string) string { + switch strings.ToLower(templateName) { + case "openai": + return "openai" + case "anthropic": + return "anthropic" + case "awsbedrock": + return "aws.bedrock" + case "azure-openai": + return "azure.ai.openai" + case "azureai-foundry": + return "azure.ai.inference" + case "gemini": + return "gcp.gemini" + case "mistralai": + return "mistral_ai" + default: + return "" + } +} + +// otelGenAIOperationName derives the gen_ai.operation.name enum value from the +// route. "" when the route is not a recognised inference endpoint. +func otelGenAIOperationName(route string) string { + r := strings.ToLower(route) + switch { + case strings.Contains(r, "/chat/completions"), strings.Contains(r, "/messages"): + return "chat" + case strings.Contains(r, "/embeddings"): + return "embeddings" + case strings.Contains(r, "/generatecontent"): + return "generate_content" + case strings.Contains(r, "/completions"): + return "text_completion" + default: + return "" + } +} + +// --- OTLP/HTTP JSON wire types ----------------------------------------------- +// +// The proto3 JSON mapping of the OTLP protobufs: field names are lowerCamelCase +// and 64-bit integers are encoded as JSON strings. + +type otelExportRequest struct { + ResourceLogs []otelResourceLogs `json:"resourceLogs"` +} + +type otelResourceLogs struct { + Resource otelResource `json:"resource"` + ScopeLogs []otelScopeLogs `json:"scopeLogs"` +} + +type otelResource struct { + Attributes []otelKeyValue `json:"attributes"` +} + +type otelScopeLogs struct { + Scope otelScope `json:"scope"` + LogRecords []otelLogRecord `json:"logRecords"` +} + +type otelScope struct { + Name string `json:"name"` +} + +type otelLogRecord struct { + TimeUnixNano string `json:"timeUnixNano"` + ObservedTimeUnixNano string `json:"observedTimeUnixNano"` + SeverityNumber int `json:"severityNumber"` + SeverityText string `json:"severityText"` + EventName string `json:"eventName,omitempty"` + Body otelAnyValue `json:"body"` + Attributes []otelKeyValue `json:"attributes"` +} + +type otelKeyValue struct { + Key string `json:"key"` + Value otelAnyValue `json:"value"` +} + +// otelAnyValue is the OTLP AnyValue union; exactly one field is set. +type otelAnyValue struct { + StringValue *string `json:"stringValue,omitempty"` + IntValue *string `json:"intValue,omitempty"` + DoubleValue *float64 `json:"doubleValue,omitempty"` + BoolValue *bool `json:"boolValue,omitempty"` +} + +// otelAttrs accumulates attributes, skipping empty ones so a record carries only +// what the event populated. +type otelAttrs struct { + kvs []otelKeyValue +} + +func newOTelAttrs() *otelAttrs { + return &otelAttrs{kvs: make([]otelKeyValue, 0, 48)} +} + +func (a *otelAttrs) str(key, value string) *otelAttrs { + if value == "" { + return a + } + v := value + a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{StringValue: &v}}) + return a +} + +// strIfEmpty sets key only when it is not already present. +func (a *otelAttrs) strIfEmpty(key, value string) *otelAttrs { + for _, kv := range a.kvs { + if kv.Key == key { + return a + } + } + return a.str(key, value) +} + +func (a *otelAttrs) i64(key string, value int64) *otelAttrs { + if value == 0 { + return a + } + v := strconv.FormatInt(value, 10) + a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{IntValue: &v}}) + return a +} + +func (a *otelAttrs) f64(key string, value float64) *otelAttrs { + if value == 0 { + return a + } + v := value + a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{DoubleValue: &v}}) + return a +} + +func (a *otelAttrs) b(key string, value bool) *otelAttrs { + if !value { + return a + } + v := value + a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{BoolValue: &v}}) + return a +} + +// anyStr / anyInt / anyBool accept the interface{} values held in +// Event.Properties, whose concrete types vary by producer. +func (a *otelAttrs) anyStr(key string, value interface{}) *otelAttrs { + if s, ok := value.(string); ok { + return a.str(key, s) + } + return a +} + +func (a *otelAttrs) anyInt(key string, value interface{}) *otelAttrs { + switch v := value.(type) { + case int: + return a.i64(key, int64(v)) + case int32: + return a.i64(key, int64(v)) + case int64: + return a.i64(key, v) + case uint32: + return a.i64(key, int64(v)) + case uint64: + return a.i64(key, int64(v)) + case float64: + return a.i64(key, int64(v)) + } + return a +} + +func (a *otelAttrs) anyBool(key string, value interface{}) *otelAttrs { + switch v := value.(type) { + case bool: + return a.b(key, v) + case string: + if parsed, err := strconv.ParseBool(v); err == nil { + return a.b(key, parsed) + } + } + return a +} + +func (a *otelAttrs) list() []otelKeyValue { + return a.kvs +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go new file mode 100644 index 0000000000..91c1b5f15e --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -0,0 +1,681 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "io" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/constants" +) + +// Attribute names are written as literals throughout this file: they are the +// wire contract, and sharing a constant with the publisher would let a typo +// satisfy its own assertion. + +func testOTelConfig(endpoint string) config.OTelPublisherConfig { + return config.OTelPublisherConfig{ + Endpoint: endpoint, + ServiceName: "policy-engine", + BatchSize: 100, + FlushInterval: 50 * time.Millisecond, + QueueSize: 100, + Timeout: 2 * time.Second, + } +} + +func restEvent() *dto.Event { + api := &dto.ExtendedAPI{ + OrganizationID: "org-1", + ProjectID: "default", + EnvironmentID: "env-1", + APIContext: "/petstore", + } + api.APIID = "api-1" + api.APIName = "PetStore" + api.APIVersion = "v1.0" + api.APIType = "RestApi" + api.SubType = "RestApi" + + return &dto.Event{ + API: api, + Operation: &dto.Operation{APIMethod: "GET", APIResourceTemplate: "/petstore/pet/{petId}"}, + Target: &dto.Target{TargetResponseCode: 200, Destination: "backend:8080/pet/1", ResponseCodeDetail: "via_upstream", ResponseCacheHit: true}, + Application: &dto.Application{ApplicationID: "app-1", ApplicationName: "Web", ApplicationOwner: "alice", KeyType: "PRODUCTION"}, + Subscription: &dto.Subscription{BillingSubscriptionID: "sub-1", BillingCustomerID: "cust-1", Status: "ACTIVE", PlanName: "Gold"}, + Latencies: &dto.Latencies{ResponseLatency: 12, BackendLatency: 9, RequestMediationLatency: 2, ResponseMediationLatency: 1, Duration: 12}, + MetaInfo: &dto.MetaInfo{CorrelationID: "corr-1", GatewayType: "Envoy", RegionID: "us-east"}, + ProxyResponseCode: 200, + RequestTimestamp: time.Unix(1788320847, 0), + UserAgentHeader: "curl/8.7.1", + UserName: "alice", + UserIP: "10.0.0.5", + Properties: map[string]interface{}{ + dto.PropKeyAuthUserID: "user-1", + "requestSize": uint64(12), + "responseSize": uint64(463), + "responseContentType": "application/json", + }, + } +} + +// attrMap flattens a record's attributes for assertion. Each OTLP AnyValue keeps +// exactly one field set, so the concrete type is asserted alongside the value. +func attrMap(t *testing.T, record *otelLogRecord) map[string]interface{} { + t.Helper() + out := map[string]interface{}{} + for _, kv := range record.Attributes { + if _, dup := out[kv.Key]; dup { + t.Fatalf("attribute %q emitted twice", kv.Key) + } + switch { + case kv.Value.StringValue != nil: + out[kv.Key] = *kv.Value.StringValue + case kv.Value.IntValue != nil: + out[kv.Key] = *kv.Value.IntValue + case kv.Value.DoubleValue != nil: + out[kv.Key] = *kv.Value.DoubleValue + case kv.Value.BoolValue != nil: + out[kv.Key] = *kv.Value.BoolValue + default: + t.Fatalf("attribute %q has no value set", kv.Key) + } + } + return out +} + +func TestBuildRecordRestAPI(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + record := o.buildRecord(restEvent()) + + if record.EventName != "wso2.api.transaction" { + t.Errorf("eventName = %q, want wso2.api.transaction", record.EventName) + } + if record.SeverityNumber != 9 || record.SeverityText != "INFO" { + t.Errorf("severity = %d/%q, want 9/INFO", record.SeverityNumber, record.SeverityText) + } + // Nanoseconds, as a JSON string per the proto3 mapping. + if record.TimeUnixNano != "1788320847000000000" { + t.Errorf("timeUnixNano = %q", record.TimeUnixNano) + } + + got := attrMap(t, record) + want := map[string]interface{}{ + "event.name": "wso2.api.transaction", + "http.request.method": "GET", + "http.route": "/petstore/pet/{petId}", + "url.path": "/petstore/pet/{petId}", + "http.response.status_code": "200", + "client.address": "10.0.0.5", + "user_agent.original": "curl/8.7.1", + "http.request.body.size": "12", + "http.response.body.size": "463", + "user.id": "user-1", + "user.name": "alice", + "error.type": nil, + "server.address": "backend", + "server.port": "8080", + "wso2.upstream.destination": "backend:8080/pet/1", + "wso2.upstream.response.status_code": "200", + "wso2.upstream.response.detail": "via_upstream", + "wso2.cache.hit": true, + "wso2.response.content_type": "application/json", + "wso2.api.id": "api-1", + "wso2.api.name": "PetStore", + "wso2.api.version": "v1.0", + "wso2.api.context": "/petstore", + "wso2.api.type": "RestApi", + "wso2.api.subtype": "RestApi", + "wso2.project.id": "default", + "wso2.organization.id": "org-1", + "wso2.environment.id": "env-1", + "wso2.application.id": "app-1", + "wso2.application.name": "Web", + "wso2.application.owner": "alice", + "wso2.application.key_type": "PRODUCTION", + "wso2.subscription.id": "sub-1", + "wso2.subscription.customer.id": "cust-1", + "wso2.subscription.status": "ACTIVE", + "wso2.subscription.plan": "Gold", + "wso2.correlation.id": "corr-1", + "wso2.gateway.type": "Envoy", + "wso2.region.id": "us-east", + "wso2.latency.response_ms": "12", + "wso2.latency.backend_ms": "9", + "wso2.latency.request_mediation_ms": "2", + "wso2.latency.response_mediation_ms": "1", + "wso2.latency.duration_ms": "12", + } + for key, expected := range want { + actual, present := got[key] + if expected == nil { + if present { + t.Errorf("%s should be absent, got %v", key, actual) + } + continue + } + if !present { + t.Errorf("%s missing", key) + continue + } + if actual != expected { + t.Errorf("%s = %v (%T), want %v (%T)", key, actual, actual, expected, expected) + } + } +} + +// The resource template already carries the API context; concatenating the two +// produced "/ctx/ctx/resource" in an earlier revision. +func TestBuildRecordDoesNotDuplicateContext(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(restEvent())) + if got["url.path"] != "/petstore/pet/{petId}" { + t.Errorf("url.path = %v, want /petstore/pet/{petId}", got["url.path"]) + } +} + +// With no resource template the API context is the fallback path. +func TestBuildRecordFallsBackToContext(t *testing.T) { + event := restEvent() + event.Operation.APIResourceTemplate = "" + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if got["url.path"] != "/petstore" { + t.Errorf("url.path = %v, want /petstore", got["url.path"]) + } + if _, present := got["http.route"]; present { + t.Error("http.route should be absent when there is no resource template") + } +} + +func TestBuildRecordFaults(t *testing.T) { + event := restEvent() + event.ProxyResponseCode = 401 + event.ErrorType = "AUTH" + event.Error = &dto.Error{ErrorCode: 900901, ErrorMessage: dto.AuthenticationFailure} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, expected := range map[string]interface{}{ + "error.type": "AUTH", + "wso2.error.code": "900901", + "wso2.error.sub_category": "AUTHENTICATION_FAILURE", + "http.response.status_code": "401", + } { + if got[key] != expected { + t.Errorf("%s = %v, want %v", key, got[key], expected) + } + } +} + +func TestBuildRecordGenAI(t *testing.T) { + event := restEvent() + event.API.APIType = "LlmProxy" + event.Operation.APIResourceTemplate = "/ai/chat/completions" + event.Properties["aiMetadata"] = dto.AIMetadata{ + Model: "claude-opus-4", + VendorName: "awsbedrock", + VendorVersion: "2024-10-01", + LLMCost: 0.0421, + } + event.Properties["aiTokenUsage"] = dto.AITokenUsage{PromptToken: 1841, CompletionToken: 210, TotalToken: 2051} + event.Properties[constants.RequestModelPropertyKey] = "claude-opus-4-20260101" + event.Properties[constants.GuardrailHitMetadataKey] = true + event.Properties[constants.GuardrailNameMetadataKey] = "pii-masking" + event.Properties["isEgress"] = true + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, expected := range map[string]interface{}{ + // awsbedrock is not a gen_ai.provider.name enum member; aws.bedrock is. + "gen_ai.provider.name": "aws.bedrock", + "wso2.gen_ai.provider.template_name": "awsbedrock", + // aitoken:modelid resolves to the response model. + "gen_ai.response.model": "claude-opus-4", + "gen_ai.request.model": "claude-opus-4-20260101", + "gen_ai.operation.name": "chat", + "gen_ai.usage.input_tokens": "1841", + "gen_ai.usage.output_tokens": "210", + "wso2.gen_ai.usage.total_tokens": "2051", + "wso2.gen_ai.cost.total": 0.0421, + "wso2.gen_ai.provider.api_version": "2024-10-01", + "wso2.guardrail.hit": true, + "wso2.guardrail.name": "pii-masking", + "wso2.gen_ai.egress": true, + } { + if got[key] != expected { + t.Errorf("%s = %v (%T), want %v (%T)", key, got[key], got[key], expected, expected) + } + } +} + +// An unrecognised provider leaves the enum attribute unset rather than carrying +// a non-member value, and keeps its identity in the wso2.* attribute. +func TestBuildRecordUnknownGenAIProvider(t *testing.T) { + event := restEvent() + event.Properties["aiMetadata"] = dto.AIMetadata{VendorName: "some-private-llm", Model: "m1"} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if _, present := got["gen_ai.provider.name"]; present { + t.Errorf("gen_ai.provider.name should be unset for an unknown provider, got %v", got["gen_ai.provider.name"]) + } + if got["wso2.gen_ai.provider.template_name"] != "some-private-llm" { + t.Errorf("template_name = %v", got["wso2.gen_ai.provider.template_name"]) + } +} + +func TestGenAIOperationName(t *testing.T) { + for route, want := range map[string]string{ + "/ai/chat/completions": "chat", + "/anthropic/messages": "chat", + "/ai/embeddings": "embeddings", + "/ai/completions": "text_completion", + "/petstore/pet/{id}": "", + } { + if got := otelGenAIOperationName(route); got != want { + t.Errorf("otelGenAIOperationName(%q) = %q, want %q", route, got, want) + } + } +} + +func TestBuildRecordMCP(t *testing.T) { + event := restEvent() + event.API.APIType = "Mcp" + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "jsonRpcMethod": "tools/call", + "jsonRpcId": "7", + "sessionId": "sess-1", + "capability": "TOOL", + "capabilityName": "search_docs", + "errorCode": -32602, + "isError": true, + "protocolVersion": "2025-06-18", + "requestedProtocolVersion": "2025-03-26", + "name": "claude-desktop", + "version": "1.2.0", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, expected := range map[string]interface{}{ + "mcp.method.name": "tools/call", + "jsonrpc.request.id": "7", + "mcp.session.id": "sess-1", + "gen_ai.tool.name": "search_docs", + "rpc.response.status_code": "-32602", + "mcp.protocol.version": "2025-06-18", + "wso2.mcp.client.requested_protocol_version": "2025-03-26", + "wso2.mcp.client.name": "claude-desktop", + "wso2.mcp.client.version": "1.2.0", + } { + if got[key] != expected { + t.Errorf("%s = %v, want %v", key, got[key], expected) + } + } + // The capability decides which attribute the name lands on. + for _, absent := range []string{"mcp.resource.uri", "gen_ai.prompt.name"} { + if _, present := got[absent]; present { + t.Errorf("%s should be absent for a TOOL capability", absent) + } + } + // event.ErrorType is empty here, so the MCP error supplies error.type. + if got["error.type"] != "mcp_error" { + t.Errorf("error.type = %v, want mcp_error", got["error.type"]) + } +} + +// A gateway-level ErrorType must not be overwritten by the MCP fallback. +func TestBuildRecordMCPKeepsGatewayErrorType(t *testing.T) { + event := restEvent() + event.ErrorType = "THROTTLED" + event.Properties["mcpAnalytics"] = map[string]interface{}{"isError": true} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if got["error.type"] != "THROTTLED" { + t.Errorf("error.type = %v, want THROTTLED", got["error.type"]) + } +} + +func TestMCPCapabilityRouting(t *testing.T) { + for capability, wantKey := range map[string]string{ + "RESOURCE": "mcp.resource.uri", + "PROMPT": "gen_ai.prompt.name", + "TOOL": "gen_ai.tool.name", + } { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "capability": capability, + "capabilityName": "target-1", + } + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if got[wantKey] != "target-1" { + t.Errorf("capability %s: %s = %v, want target-1", capability, wantKey, got[wantKey]) + } + } +} + +// End-to-end over the real HTTP path: the payload a collector receives must be +// valid OTLP-JSON with the routing scope and the configured headers. +func TestExportPayloadAndHeaders(t *testing.T) { + var ( + mu sync.Mutex + body []byte + header http.Header + ) + done := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + body, _ = io.ReadAll(r.Body) + header = r.Header.Clone() + mu.Unlock() + w.WriteHeader(http.StatusOK) + close(done) + })) + defer server.Close() + + cfg := testOTelConfig(server.URL + "/v1/logs") + cfg.ServiceVersion = "1.2.0" + cfg.Headers = map[string]string{"X-Api-Key": "secret"} + cfg.ResourceAttributes = map[string]string{"deployment.environment.name": "test"} + + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + publisher.Publish(restEvent()) + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("no export received") + } + if err := publisher.Close(context.Background()); err != nil { + t.Errorf("Close: %v", err) + } + + mu.Lock() + defer mu.Unlock() + + if got := header.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q", got) + } + if got := header.Get("X-Api-Key"); got != "secret" { + t.Errorf("configured header not sent, got %q", got) + } + + var payload otelExportRequest + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("payload is not valid JSON: %v", err) + } + if len(payload.ResourceLogs) != 1 || len(payload.ResourceLogs[0].ScopeLogs) != 1 { + t.Fatalf("unexpected payload shape: %s", body) + } + scope := payload.ResourceLogs[0].ScopeLogs[0] + if scope.Scope.Name != "wso2.analytics" { + t.Errorf("scope.name = %q, want wso2.analytics — the collector routes on this", scope.Scope.Name) + } + if len(scope.LogRecords) != 1 { + t.Fatalf("want 1 record, got %d", len(scope.LogRecords)) + } + + resource := map[string]string{} + for _, kv := range payload.ResourceLogs[0].Resource.Attributes { + if kv.Value.StringValue != nil { + resource[kv.Key] = *kv.Value.StringValue + } + } + for key, want := range map[string]string{ + "service.name": "policy-engine", + "service.version": "1.2.0", + "deployment.environment.name": "test", + } { + if resource[key] != want { + t.Errorf("resource %s = %q, want %q", key, resource[key], want) + } + } +} + +// A full queue must drop rather than block the ALS ingest path. +func TestPublishDropsWhenQueueFull(t *testing.T) { + cfg := testOTelConfig("http://127.0.0.1:1/v1/logs") + cfg.QueueSize = 1 + cfg.BatchSize = 1 + cfg.FlushInterval = time.Hour // keep the worker parked so the queue fills + + o := &OTel{ + cfg: cfg, + client: &http.Client{Timeout: cfg.Timeout}, + queue: make(chan *otelLogRecord, cfg.QueueSize), + stop: make(chan struct{}), + workerDone: make(chan struct{}), + } + // No worker started: nothing drains the queue. + for i := 0; i < 10; i++ { + o.Publish(restEvent()) + } + + o.droppedMu.Lock() + dropped := o.dropped + o.droppedMu.Unlock() + if dropped != 9 { + t.Errorf("dropped = %d, want 9 (queue holds 1)", dropped) + } +} + +func TestCloseIsIdempotent(t *testing.T) { + cfg := testOTelConfig("http://127.0.0.1:1/v1/logs") + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + if err := publisher.Close(context.Background()); err != nil { + t.Errorf("first Close: %v", err) + } + if err := publisher.Close(context.Background()); err != nil { + t.Errorf("second Close: %v", err) + } +} + +func TestNewOTelNilConfig(t *testing.T) { + publisher, err := NewOTel(nil) + if err == nil { + t.Error("NewOTel(nil) should return an error") + } + if publisher != nil { + t.Error("NewOTel(nil) should return a nil publisher") + } +} + +// --- TLS ------------------------------------------------------------------- + +// writeSelfSignedPair writes a throwaway self-signed certificate and its key, +// usable both as a CA bundle and as an mTLS client pair. +func writeSelfSignedPair(t *testing.T) (certPath, keyPath string) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "otel-publisher-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + + dir := t.TempDir() + certPath = filepath.Join(dir, "cert.pem") + keyPath = filepath.Join(dir, "key.pem") + writePEM(t, certPath, "CERTIFICATE", der) + writePEM(t, keyPath, "EC PRIVATE KEY", keyDER) + return certPath, keyPath +} + +func writePEM(t *testing.T, path, blockType string, der []byte) { + t.Helper() + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der}), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func TestBuildOTelTLSConfigDefaults(t *testing.T) { + got, err := buildOTelTLSConfig(config.OTelTLSConfig{}) + if err != nil { + t.Fatalf("buildOTelTLSConfig: %v", err) + } + if got.MinVersion != tls.VersionTLS12 { + t.Errorf("MinVersion = %x, want TLS 1.2", got.MinVersion) + } + // The hybrid must be offered first, with classical curves retained after it + // so an endpoint without the hybrid still handshakes. + if got.CurvePreferences[0] != tls.X25519MLKEM768 { + t.Errorf("CurvePreferences[0] = %v, want X25519MLKEM768", got.CurvePreferences[0]) + } + if len(got.CurvePreferences) < 2 { + t.Error("no classical curve retained after the hybrid") + } + if got.InsecureSkipVerify { + t.Error("InsecureSkipVerify must default to false") + } + if got.RootCAs != nil || len(got.Certificates) != 0 { + t.Error("no TLS material configured, yet RootCAs/Certificates are set") + } +} + +func TestBuildOTelTLSConfigCAFile(t *testing.T) { + certPath, _ := writeSelfSignedPair(t) + got, err := buildOTelTLSConfig(config.OTelTLSConfig{CAFile: certPath}) + if err != nil { + t.Fatalf("buildOTelTLSConfig: %v", err) + } + if got.RootCAs == nil { + t.Error("RootCAs not populated from ca_file") + } +} + +func TestBuildOTelTLSConfigMTLSPair(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + got, err := buildOTelTLSConfig(config.OTelTLSConfig{CertFile: certPath, KeyFile: keyPath}) + if err != nil { + t.Fatalf("buildOTelTLSConfig: %v", err) + } + if len(got.Certificates) != 1 { + t.Errorf("Certificates = %d, want 1", len(got.Certificates)) + } +} + +func TestBuildOTelTLSConfigRejections(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + garbage := filepath.Join(t.TempDir(), "garbage.pem") + if err := os.WriteFile(garbage, []byte("not a certificate"), 0o600); err != nil { + t.Fatalf("write garbage: %v", err) + } + + cases := map[string]config.OTelTLSConfig{ + "missing ca file": {CAFile: filepath.Join(t.TempDir(), "absent.pem")}, + "unusable ca file": {CAFile: garbage}, + "cert without key": {CertFile: certPath}, + "key without cert": {KeyFile: keyPath}, + "mismatched pair": {CertFile: keyPath, KeyFile: certPath}, + } + for name, cfg := range cases { + t.Run(name, func(t *testing.T) { + if _, err := buildOTelTLSConfig(cfg); err == nil { + t.Error("expected an error, got nil") + } + }) + } +} + +// TestExportOverTLSWithCAFile proves the ca_file path actually works against a +// server whose certificate the system trust store does not know. +func TestExportOverTLSWithCAFile(t *testing.T) { + done := make(chan struct{}) + var once sync.Once + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + once.Do(func() { close(done) }) + })) + defer server.Close() + + caPath := filepath.Join(t.TempDir(), "server-ca.pem") + writePEM(t, caPath, "CERTIFICATE", server.Certificate().Raw) + + cfg := testOTelConfig(server.URL + "/v1/logs") + cfg.TLS = config.OTelTLSConfig{CAFile: caPath} + + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + defer publisher.Close(context.Background()) + publisher.Publish(restEvent()) + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("no export received over TLS") + } +} + +// TestNewOTelFailsClosedOnBadTLSMaterial: a publisher that can never reach its +// endpoint must not be constructed. +func TestNewOTelFailsClosedOnBadTLSMaterial(t *testing.T) { + cfg := testOTelConfig("https://collector.invalid/v1/logs") + cfg.TLS = config.OTelTLSConfig{CAFile: filepath.Join(t.TempDir(), "absent.pem")} + if _, err := NewOTel(&cfg); err == nil { + t.Error("expected NewOTel to fail on an unreadable ca_file") + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 3ba67a6653..034574669d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -134,6 +134,53 @@ type AnalyticsConfig struct { // AnalyticsPublishersConfig holds configuration for all analytics publishers type AnalyticsPublishersConfig struct { Moesif MoesifPublisherConfig `koanf:"moesif"` + OTel OTelPublisherConfig `koanf:"otel"` +} + +// OTelPublisherConfig configures the OpenTelemetry analytics publisher, which +// exports each event as an OTLP log record over OTLP/HTTP. +type OTelPublisherConfig struct { + // Endpoint is the full OTLP/HTTP logs URL, including the /v1/logs path. + Endpoint string `koanf:"endpoint"` + // Headers are sent on every export request. Use for a vendor's OTLP intake + // that authenticates by header. Values are secrets and are never logged. + Headers map[string]string `koanf:"headers"` + // ServiceName / ServiceVersion populate the OTLP resource. + ServiceName string `koanf:"service_name"` + ServiceVersion string `koanf:"service_version"` + // ResourceAttributes are added to the OTLP resource, alongside service.*. + ResourceAttributes map[string]string `koanf:"resource_attributes"` + // BatchSize is the record count that triggers an export before FlushInterval. + BatchSize int `koanf:"batch_size"` + // FlushInterval bounds how long a record waits when traffic is too slow to + // fill a batch. + FlushInterval time.Duration `koanf:"flush_interval"` + // QueueSize bounds records held in memory when the endpoint is slow. Records + // are dropped and counted once it is full. + QueueSize int `koanf:"queue_size"` + // Timeout bounds a single export attempt. + Timeout time.Duration `koanf:"timeout"` + // TLS configures the client side of an https endpoint. Ignored for http. + TLS OTelTLSConfig `koanf:"tls"` +} + +// OTelTLSConfig configures TLS to the OTLP endpoint +// ([analytics.publishers.otel.tls]). Deliberately a separate type from +// TrafficLogHTTPTLSConfig despite the identical keys: the two config blocks are +// independent, and sharing one type would couple them. +type OTelTLSConfig struct { + // CAFile is a PEM bundle used to verify the endpoint's certificate. Empty + // means the system trust store, which is correct for a vendor's OTLP intake + // and usually wrong for an in-cluster collector fronted by a private CA. + CAFile string `koanf:"ca_file"` + // CertFile / KeyFile enable mTLS. Both must be set, or neither. + CertFile string `koanf:"cert_file"` + KeyFile string `koanf:"key_file"` + // InsecureSkipVerify disables endpoint certificate verification. Off by + // default; when on, startup logs a warning naming the endpoint, because + // analytics records carry request metadata and, when body capture is + // enabled, request and response bodies. + InsecureSkipVerify bool `koanf:"insecure_skip_verify"` } // Traffic-log sink names accepted in traffic_logging.outputs. @@ -1170,6 +1217,15 @@ func defaultConfig() *Config { BatchSize: 50, TimerWakeupSeconds: 3, }, + OTel: OTelPublisherConfig{ + Endpoint: "http://otel-collector:4318/v1/logs", + ServiceName: "policy-engine", + ServiceVersion: "", + BatchSize: 100, + FlushInterval: 5 * time.Second, + QueueSize: 10000, + Timeout: 10 * time.Second, + }, }, GRPCEventServerCfg: map[string]interface{}{ "server_port": 18090, @@ -1454,6 +1510,72 @@ func (c *Config) validateXDSConfig() error { return nil } +// validateOTelPublisherConfig validates [analytics.publishers.otel]. +func validateOTelPublisherConfig(cfg OTelPublisherConfig) error { + if cfg.Endpoint == "" { + return fmt.Errorf("analytics.publishers.otel.endpoint is required when otel is enabled") + } + u, err := url.Parse(cfg.Endpoint) + if err != nil || u.Host == "" { + return fmt.Errorf("analytics.publishers.otel.endpoint must be a valid URL (e.g. http://otel-collector:4318/v1/logs), got %q", cfg.Endpoint) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("analytics.publishers.otel.endpoint scheme must be http or https, got %q", u.Scheme) + } + if cfg.ServiceName == "" { + return fmt.Errorf("analytics.publishers.otel.service_name is required") + } + if cfg.BatchSize <= 0 { + return fmt.Errorf("analytics.publishers.otel.batch_size must be > 0, got %d", cfg.BatchSize) + } + if cfg.QueueSize <= 0 { + return fmt.Errorf("analytics.publishers.otel.queue_size must be > 0, got %d", cfg.QueueSize) + } + // A queue smaller than a batch can never fill one, so every export would be + // interval-driven regardless of load. + if cfg.QueueSize < cfg.BatchSize { + return fmt.Errorf("analytics.publishers.otel.queue_size (%d) must be >= batch_size (%d)", cfg.QueueSize, cfg.BatchSize) + } + if cfg.FlushInterval <= 0 { + return fmt.Errorf("analytics.publishers.otel.flush_interval must be > 0, got %s", cfg.FlushInterval) + } + if cfg.Timeout <= 0 { + return fmt.Errorf("analytics.publishers.otel.timeout must be > 0, got %s", cfg.Timeout) + } + if err := validateOTelTLS(cfg.TLS, u.Host); err != nil { + return fmt.Errorf("analytics.publishers.otel.tls: %w", err) + } + return nil +} + +// validateOTelTLS checks that any referenced TLS material exists and parses, so a +// bad path fails at startup rather than on the first export. +func validateOTelTLS(cfg OTelTLSConfig, host string) error { + if cfg.InsecureSkipVerify { + slog.Warn("analytics.publishers.otel.tls.insecure_skip_verify is true: the endpoint's "+ + "certificate is not verified, so analytics records are exposed to anyone able to "+ + "intercept this connection", "host", host) + } + if cfg.CAFile != "" { + pem, err := os.ReadFile(cfg.CAFile) + if err != nil { + return fmt.Errorf("cannot read ca_file %q: %w", cfg.CAFile, err) + } + if !x509.NewCertPool().AppendCertsFromPEM(pem) { + return fmt.Errorf("ca_file %q contains no usable PEM certificate", cfg.CAFile) + } + } + if (cfg.CertFile == "") != (cfg.KeyFile == "") { + return fmt.Errorf("cert_file and key_file must be set together for mTLS (one is set, the other is not)") + } + if cfg.CertFile != "" { + if _, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile); err != nil { + return fmt.Errorf("cannot load client certificate/key pair: %w", err) + } + } + return nil +} + // validateCollectorConfig migrates deprecated analytics capture aliases onto the // collector and enforces the collector prerequisite: a consumer (analytics or // traffic logging) requires the collector that feeds it. The collector has no @@ -1565,6 +1687,10 @@ func (c *Config) validateAnalyticsConfig() error { return fmt.Errorf("analytics.publishers.moesif.moesif_base_url must be a valid URL (e.g. https://api.moesif.net), got %q", moesifCfg.BaseURL) } } + case "otel": + if err := validateOTelPublisherConfig(c.Analytics.Publishers.OTel); err != nil { + return err + } default: return fmt.Errorf("unknown publisher type in enabled_publishers: %s", publisherName) } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go new file mode 100644 index 0000000000..7fc2b12143 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package config + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// otelConfig returns a Config with analytics enabled and only the otel publisher +// selected, so Validate exercises validateOTelPublisherConfig. +func otelConfig(mutate func(*OTelPublisherConfig)) *Config { + cfg := defaultConfig() + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = []string{"otel"} + mutate(&cfg.Analytics.Publishers.OTel) + return cfg +} + +// writeSelfSignedPair writes a throwaway self-signed certificate and its key. +func writeSelfSignedPair(t *testing.T) (certPath, keyPath string) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "otel-config-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + + dir := t.TempDir() + certPath = filepath.Join(dir, "cert.pem") + keyPath = filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(certPath, + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)) + require.NoError(t, os.WriteFile(keyPath, + pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600)) + return certPath, keyPath +} + +func TestValidate_OTelPublisher(t *testing.T) { + t.Run("defaults pass", func(t *testing.T) { + assert.NoError(t, otelConfig(func(*OTelPublisherConfig) {}).Validate()) + }) + + t.Run("https endpoint passes", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://otlp.vendor.example.com/v1/logs" + }) + assert.NoError(t, cfg.Validate()) + }) + + rejected := map[string]func(*OTelPublisherConfig){ + "missing endpoint": func(o *OTelPublisherConfig) { o.Endpoint = "" }, + "endpoint without host": func(o *OTelPublisherConfig) { o.Endpoint = "/v1/logs" }, + "non-http scheme": func(o *OTelPublisherConfig) { o.Endpoint = "grpc://collector:4317" }, + "missing service name": func(o *OTelPublisherConfig) { o.ServiceName = "" }, + "zero batch size": func(o *OTelPublisherConfig) { o.BatchSize = 0 }, + "zero queue size": func(o *OTelPublisherConfig) { o.QueueSize = 0 }, + // A queue smaller than a batch can never fill one. + "queue smaller than batch": func(o *OTelPublisherConfig) { o.QueueSize = 10; o.BatchSize = 100 }, + "zero flush interval": func(o *OTelPublisherConfig) { o.FlushInterval = 0 }, + "negative timeout": func(o *OTelPublisherConfig) { o.Timeout = -time.Second }, + } + for name, mutate := range rejected { + t.Run(name, func(t *testing.T) { + err := otelConfig(mutate).Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "analytics.publishers.otel") + }) + } + + // An unknown publisher name must fail rather than be silently skipped. + t.Run("unknown publisher name is an error", func(t *testing.T) { + cfg := otelConfig(func(*OTelPublisherConfig) {}) + cfg.Analytics.EnabledPublishers = []string{"otlp"} + assert.Error(t, cfg.Validate()) + }) + + // Nothing under the block is validated while the publisher is not enabled. + t.Run("not enabled skips validation", func(t *testing.T) { + cfg := defaultConfig() + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = nil + cfg.Analytics.Publishers.OTel.Endpoint = "" + cfg.Analytics.Publishers.OTel.TLS = OTelTLSConfig{CAFile: "/nonexistent/ca.pem"} + assert.NoError(t, cfg.Validate()) + }) +} + +// TLS material is loaded at startup so a bad path fails there rather than on the +// first export, hours later. +func TestValidate_OTelPublisherTLS(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + + t.Run("ca file plus mTLS pair passes", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.internal:4318/v1/logs" + o.TLS = OTelTLSConfig{CAFile: certPath, CertFile: certPath, KeyFile: keyPath} + }) + assert.NoError(t, cfg.Validate()) + }) + + garbage := filepath.Join(t.TempDir(), "garbage.pem") + require.NoError(t, os.WriteFile(garbage, []byte("not a certificate"), 0o600)) + absent := filepath.Join(t.TempDir(), "absent.pem") + + rejected := map[string]OTelTLSConfig{ + "unreadable ca file": {CAFile: absent}, + "ca file with no PEM": {CAFile: garbage}, + "cert without key": {CertFile: certPath}, + "key without cert": {KeyFile: keyPath}, + "unloadable pair": {CertFile: keyPath, KeyFile: certPath}, + } + for name, tlsCfg := range rejected { + t.Run(name, func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.internal:4318/v1/logs" + o.TLS = tlsCfg + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "analytics.publishers.otel.tls") + }) + } + + // insecure_skip_verify only warns — it must not block startup, or an + // operator debugging a cert problem has no way through. + t.Run("insecure_skip_verify warns but passes", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.internal:4318/v1/logs" + o.TLS = OTelTLSConfig{InsecureSkipVerify: true} + }) + assert.NoError(t, cfg.Validate()) + }) +} diff --git a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go index 95a1d34977..cb395bb2f5 100644 --- a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go +++ b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go @@ -128,4 +128,5 @@ const ( GuardrailNameMetadataKey = "guardrailName" LLMCostMetadataKey = "x-llm-cost" LLMCostPropertyKey = "llmCost" + RequestModelPropertyKey = "requestModel" // Holds the model named in the request ) diff --git a/gateway/system-policies/analytics/analytics.go b/gateway/system-policies/analytics/analytics.go index d9f7b8e21a..4ce3ab9e7c 100644 --- a/gateway/system-policies/analytics/analytics.go +++ b/gateway/system-policies/analytics/analytics.go @@ -24,6 +24,7 @@ const ( CompletionTokenCountMetadataKey = "aitoken:completiontokencount" TotalTokenCountMetadataKey = "aitoken:totaltokencount" ModelIDMetadataKey = "aitoken:modelid" + RequestModelIDMetadataKey = "aitoken:requestmodelid" AIProviderNameMetadataKey = "ai:providername" AIProviderDisplayNameMetadataKey = "ai:providerdisplayname" ApplicationIDMetadataKey = "x-wso2-application-id" @@ -76,6 +77,7 @@ const ( var ( // JSON Path expressions to extract MCP analytics properties from response body JsonRpcMethodJsonPath = "$.method" + JsonRpcIDJsonPath = "$.id" McpCapabilityNameJsonPath = "$.params.name" McpResourceUriJsonPath = "$.params.uri" ProtocolVersionJsonPath = "$.params.protocolVersion" @@ -94,6 +96,7 @@ type AnalyticsPolicy struct{} type McpRequestAnalyticsProperties struct { JsonRpcMethod string `json:"jsonRpcMethod,omitempty"` + JsonRpcID string `json:"jsonRpcId,omitempty"` Capability string `json:"capability,omitempty"` CapabilityName string `json:"capabilityName,omitempty"` ClientInfo *McpClientInfo `json:"clientInfo,omitempty"` @@ -400,6 +403,15 @@ func (a *AnalyticsPolicy) OnRequestBody(_ context.Context, ctx *policy.RequestCo } props.JsonRpcMethod = extractString(JsonRpcMethodJsonPath) + // A JSON-RPC id may be a string or a number. + if raw, err := utils.ExtractValueFromJsonpath(mcpPayload, JsonRpcIDJsonPath); err == nil && raw != nil { + switch id := raw.(type) { + case string: + props.JsonRpcID = id + case float64: + props.JsonRpcID = strconv.FormatInt(int64(id), 10) + } + } props.CapabilityName = extractString(McpCapabilityNameJsonPath) props.Capability = deriveMCPCapability(props.JsonRpcMethod) @@ -882,6 +894,9 @@ func populateTokenAnalyticsMetadata(analyticsMetadata map[string]any, tokenInfo } else if tokenInfo.RequestModel != nil { analyticsMetadata[ModelIDMetadataKey] = *tokenInfo.RequestModel } + if tokenInfo.RequestModel != nil { + analyticsMetadata[RequestModelIDMetadataKey] = *tokenInfo.RequestModel + } if tokenInfo.ProviderName != nil { analyticsMetadata[AIProviderNameMetadataKey] = *tokenInfo.ProviderName } From aeacaba0c9636c610351dda29d18f93da55d096b Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Mon, 7 Sep 2026 15:40:17 +0530 Subject: [PATCH 02/20] Adding queue drop policy and config vocabulary --- gateway/configs/config-template.toml | 14 ++- .../internal/analytics/publishers/otel.go | 57 ++++++--- .../analytics/publishers/otel_test.go | 108 +++++++++++++++--- .../analytics/publishers/sink_http.go | 2 +- .../analytics/publishers/sink_http_test.go | 2 +- .../policy-engine/internal/config/config.go | 44 ++++--- .../internal/config/otel_publisher_test.go | 24 +++- 7 files changed, 199 insertions(+), 52 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index d18388ad27..1edca835bd 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -823,8 +823,18 @@ batch_size = 100 # How long a record waits when traffic is too slow to fill a batch. flush_interval = "5s" # Records held in memory when the endpoint is slow. Must be >= batch_size. -# Once full, records are dropped and counted rather than growing unbounded. -queue_size = 10000 +# Once full, on_queue_full decides which record is dropped — the queue never +# grows unbounded. +queue_capacity = 10000 +# Which record is discarded once the queue is full. Analytics is strictly +# downstream of request handling, so one of the two always happens; the choice is +# whether a slow endpoint costs the newest records or the oldest. +# "drop_new" — discard the incoming record, preserving older queued ones. +# Keeps the earliest view of an incident. +# "drop_oldest" — evict the oldest queued record to make room. Keeps the most +# recent traffic, at the cost of the start of the backlog. +# Same two values as traffic_logging.http.on_queue_full. +on_queue_full = "drop_new" # Bounds a single export attempt. timeout = "10s" diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 210d19f6e7..78d94ab3a1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -94,6 +94,8 @@ type OTel struct { droppedMu sync.Mutex dropped int + // dropOldest is resolved once at construction rather than per record. + dropOldest bool } // NewOTel creates the OTLP-logs publisher and starts its exporting worker. @@ -126,9 +128,10 @@ func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { return http.ErrUseLastResponse }, }, - queue: make(chan *otelLogRecord, cfg.QueueSize), + queue: make(chan *otelLogRecord, cfg.QueueCapacity), stop: make(chan struct{}), workerDone: make(chan struct{}), + dropOldest: strings.EqualFold(strings.TrimSpace(cfg.OnQueueFull), config.QueueDropOldest), } go o.run() @@ -139,7 +142,8 @@ func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { // Headers are deliberately omitted: they carry credentials. slog.Info("OTel analytics publisher started", "endpoint", cfg.Endpoint, "batchSize", cfg.BatchSize, - "flushInterval", cfg.FlushInterval, "queueSize", cfg.QueueSize) + "flushInterval", cfg.FlushInterval, "queueCapacity", cfg.QueueCapacity, + "onQueueFull", cfg.OnQueueFull) return o, nil } @@ -191,24 +195,51 @@ func isLoopbackHost(host string) bool { } // Publish converts the event to an OTLP log record and enqueues it. +// +// Never blocks: analytics is strictly downstream of request handling, so a full +// queue costs a record and never a request. func (o *OTel) Publish(event *dto.Event) { if event == nil { return } + record := o.buildRecord(event) + select { - case o.queue <- o.buildRecord(event): + case o.queue <- record: + return default: - // The collector is not keeping up. Dropping the newest record preserves - // the queued older ones; analytics is strictly downstream of request - // handling, so a drop must never surface to the client. - o.droppedMu.Lock() - o.dropped++ - count := o.dropped - o.droppedMu.Unlock() - if count == 1 || count%100 == 0 { - slog.Warn("OTel publisher queue full; dropping analytics event", - "droppedTotal", count, "queueSize", o.cfg.QueueSize) + } + + if o.dropOldest { + // Evict one old record and retry once. A single attempt is deliberate: a + // loop could spin while producers keep the queue full, turning a + // non-blocking Publish into an unbounded one. + select { + case <-o.queue: + o.countDrop() + default: } + select { + case o.queue <- record: + return + default: + } + } + + o.countDrop() +} + +// countDrop records one dropped record, warning on the first and then every +// hundredth so a sustained outage cannot flood the log. +func (o *OTel) countDrop() { + o.droppedMu.Lock() + o.dropped++ + count := o.dropped + o.droppedMu.Unlock() + if count == 1 || count%100 == 0 { + slog.Warn("OTel publisher queue full; dropping analytics event", + "droppedTotal", count, "queueCapacity", o.cfg.QueueCapacity, + "onQueueFull", o.cfg.OnQueueFull) } } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 91c1b5f15e..45dd36c179 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -33,6 +33,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -52,7 +53,8 @@ func testOTelConfig(endpoint string) config.OTelPublisherConfig { ServiceName: "policy-engine", BatchSize: 100, FlushInterval: 50 * time.Millisecond, - QueueSize: 100, + QueueCapacity: 100, + OnQueueFull: config.QueueDropNew, Timeout: 2 * time.Second, } } @@ -475,30 +477,102 @@ func TestExportPayloadAndHeaders(t *testing.T) { } } -// A full queue must drop rather than block the ALS ingest path. -func TestPublishDropsWhenQueueFull(t *testing.T) { - cfg := testOTelConfig("http://127.0.0.1:1/v1/logs") - cfg.QueueSize = 1 - cfg.BatchSize = 1 - cfg.FlushInterval = time.Hour // keep the worker parked so the queue fills +// correlatedEvent returns the REST fixture with a distinguishable correlation id, +// so a queue's surviving record can be identified. +func correlatedEvent(id string) *dto.Event { + event := restEvent() + event.MetaInfo.CorrelationID = id + return event +} - o := &OTel{ +// newUndrainedOTel builds a publisher with no worker goroutine, so nothing +// consumes the queue and Publish sees it full. NewOTel cannot be used here: it +// starts the worker. +func newUndrainedOTel(t *testing.T, capacity int, onQueueFull string) *OTel { + t.Helper() + cfg := testOTelConfig("http://127.0.0.1:1/v1/logs") + cfg.QueueCapacity = capacity + cfg.BatchSize = capacity + cfg.OnQueueFull = onQueueFull + return &OTel{ cfg: cfg, client: &http.Client{Timeout: cfg.Timeout}, - queue: make(chan *otelLogRecord, cfg.QueueSize), + queue: make(chan *otelLogRecord, cfg.QueueCapacity), stop: make(chan struct{}), workerDone: make(chan struct{}), + dropOldest: strings.EqualFold(onQueueFull, config.QueueDropOldest), + } +} + +// A full queue must drop rather than block the ALS ingest path — under either +// policy, and Publish must never block. +func TestPublishDropsWhenQueueFull(t *testing.T) { + for _, policy := range []string{config.QueueDropNew, config.QueueDropOldest} { + t.Run(policy, func(t *testing.T) { + o := newUndrainedOTel(t, 1, policy) + for i := 0; i < 10; i++ { + o.Publish(restEvent()) + } + + o.droppedMu.Lock() + dropped := o.dropped + o.droppedMu.Unlock() + if dropped != 9 { + t.Errorf("dropped = %d, want 9 (queue holds 1)", dropped) + } + if len(o.queue) != 1 { + t.Errorf("queue holds %d records, want 1", len(o.queue)) + } + }) } - // No worker started: nothing drains the queue. - for i := 0; i < 10; i++ { - o.Publish(restEvent()) +} + +// The policy decides *which* record survives, which is the whole point of the +// setting: drop_new keeps the oldest queued record, drop_oldest keeps the newest. +func TestPublishQueueFullKeepsPolicysRecord(t *testing.T) { + cases := []struct { + policy string + survivor string + }{ + {config.QueueDropNew, "first"}, + {config.QueueDropOldest, "last"}, + } + for _, tc := range cases { + t.Run(tc.policy, func(t *testing.T) { + o := newUndrainedOTel(t, 1, tc.policy) + o.Publish(correlatedEvent("first")) + o.Publish(correlatedEvent("middle")) + o.Publish(correlatedEvent("last")) + + if len(o.queue) != 1 { + t.Fatalf("queue holds %d records, want 1", len(o.queue)) + } + got := attrMap(t, <-o.queue)["wso2.correlation.id"] + if got != tc.survivor { + t.Errorf("surviving record = %v, want %q", got, tc.survivor) + } + }) } +} - o.droppedMu.Lock() - dropped := o.dropped - o.droppedMu.Unlock() - if dropped != 9 { - t.Errorf("dropped = %d, want 9 (queue holds 1)", dropped) +// drop_oldest evicts exactly once per Publish. A retry loop would spin while +// producers keep the queue full, turning a non-blocking Publish into a blocking +// one; a single eviction bounds the work per call. +func TestPublishDropOldestEvictsOncePerCall(t *testing.T) { + o := newUndrainedOTel(t, 2, config.QueueDropOldest) + o.Publish(correlatedEvent("a")) + o.Publish(correlatedEvent("b")) + o.Publish(correlatedEvent("c")) // evicts "a", enqueues "c" + + if len(o.queue) != 2 { + t.Fatalf("queue holds %d records, want 2", len(o.queue)) + } + var got []interface{} + for len(o.queue) > 0 { + got = append(got, attrMap(t, <-o.queue)["wso2.correlation.id"]) + } + if len(got) != 2 || got[0] != "b" || got[1] != "c" { + t.Errorf("queue = %v, want [b c]", got) } } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http.go index 690eff18c0..dafdcf52f5 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http.go @@ -156,7 +156,7 @@ func newHTTPSink(cfg config.TrafficLogHTTPConfig) (*httpSink, error) { authHeaderValue: value, queue: make(chan []byte, cfg.QueueCapacity), dropOldest: strings.EqualFold(strings.TrimSpace(cfg.OnQueueFull), - config.TrafficLogQueueDropOldest), + config.QueueDropOldest), done: make(chan struct{}), stopped: make(chan struct{}), // Below this, retrying is free — nothing is being lost diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http_test.go index 0a863f26bb..e9658b3e20 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_http_test.go @@ -99,7 +99,7 @@ func httpSinkCfg(endpoint string) config.TrafficLogHTTPConfig { BatchMaxBytes: 1 << 20, FlushInterval: 25 * time.Millisecond, QueueCapacity: 100, - OnQueueFull: config.TrafficLogQueueDropNew, + OnQueueFull: config.QueueDropNew, RequestTimeout: 2 * time.Second, MaxRetries: 0, RetryBackoff: time.Millisecond, diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 034574669d..bd501dd9a4 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -155,9 +155,11 @@ type OTelPublisherConfig struct { // FlushInterval bounds how long a record waits when traffic is too slow to // fill a batch. FlushInterval time.Duration `koanf:"flush_interval"` - // QueueSize bounds records held in memory when the endpoint is slow. Records - // are dropped and counted once it is full. - QueueSize int `koanf:"queue_size"` + // QueueCapacity bounds records held in memory when the endpoint is slow. + // Once full, OnQueueFull decides which record is dropped. + QueueCapacity int `koanf:"queue_capacity"` + // OnQueueFull is QueueDropNew (default) or QueueDropOldest. + OnQueueFull string `koanf:"on_queue_full"` // Timeout bounds a single export attempt. Timeout time.Duration `koanf:"timeout"` // TLS configures the client side of an https endpoint. Ignored for http. @@ -209,12 +211,14 @@ const ( TrafficLogAuthHeader = "header" ) -// Behavior when the HTTP sink's queue is full (traffic_logging.http.on_queue_full). +// Behavior when a bounded publisher queue is full. Shared vocabulary: both +// traffic_logging.http.on_queue_full and analytics.publishers.otel.on_queue_full +// accept exactly these values, so the two must never diverge. const ( - // TrafficLogQueueDropNew discards the incoming line, preserving older ones. - TrafficLogQueueDropNew = "drop_new" - // TrafficLogQueueDropOldest evicts the oldest queued line to make room. - TrafficLogQueueDropOldest = "drop_oldest" + // QueueDropNew discards the incoming item, preserving older queued ones. + QueueDropNew = "drop_new" + // QueueDropOldest evicts the oldest queued item to make room for the new one. + QueueDropOldest = "drop_oldest" ) // TrafficLoggingConfig holds configuration for the traffic-logging feature, which @@ -1054,7 +1058,7 @@ func defaultTrafficLogHTTPConfig() TrafficLogHTTPConfig { // ride out a short receiver blip without letting a long outage grow the // heap without bound. QueueCapacity: 10000, - OnQueueFull: TrafficLogQueueDropNew, + OnQueueFull: QueueDropNew, RequestTimeout: 10 * time.Second, MaxRetries: 3, RetryBackoff: time.Second, @@ -1223,7 +1227,8 @@ func defaultConfig() *Config { ServiceVersion: "", BatchSize: 100, FlushInterval: 5 * time.Second, - QueueSize: 10000, + QueueCapacity: 10000, + OnQueueFull: QueueDropNew, Timeout: 10 * time.Second, }, }, @@ -1528,13 +1533,20 @@ func validateOTelPublisherConfig(cfg OTelPublisherConfig) error { if cfg.BatchSize <= 0 { return fmt.Errorf("analytics.publishers.otel.batch_size must be > 0, got %d", cfg.BatchSize) } - if cfg.QueueSize <= 0 { - return fmt.Errorf("analytics.publishers.otel.queue_size must be > 0, got %d", cfg.QueueSize) + if cfg.QueueCapacity <= 0 { + return fmt.Errorf("analytics.publishers.otel.queue_capacity must be > 0, got %d; an unbounded "+ + "queue in front of a bounded exporter is deferred unbounded memory growth", cfg.QueueCapacity) } // A queue smaller than a batch can never fill one, so every export would be // interval-driven regardless of load. - if cfg.QueueSize < cfg.BatchSize { - return fmt.Errorf("analytics.publishers.otel.queue_size (%d) must be >= batch_size (%d)", cfg.QueueSize, cfg.BatchSize) + if cfg.QueueCapacity < cfg.BatchSize { + return fmt.Errorf("analytics.publishers.otel.queue_capacity (%d) must be >= batch_size (%d)", cfg.QueueCapacity, cfg.BatchSize) + } + switch strings.ToLower(strings.TrimSpace(cfg.OnQueueFull)) { + case QueueDropNew, QueueDropOldest: + default: + return fmt.Errorf("analytics.publishers.otel.on_queue_full must be %q or %q, got %q", + QueueDropNew, QueueDropOldest, cfg.OnQueueFull) } if cfg.FlushInterval <= 0 { return fmt.Errorf("analytics.publishers.otel.flush_interval must be > 0, got %s", cfg.FlushInterval) @@ -1996,10 +2008,10 @@ func validateTrafficLogHTTPConfig(cfg TrafficLogHTTPConfig) error { "bounded sender is deferred unbounded memory growth", cfg.QueueCapacity) } switch strings.ToLower(strings.TrimSpace(cfg.OnQueueFull)) { - case TrafficLogQueueDropNew, TrafficLogQueueDropOldest: + case QueueDropNew, QueueDropOldest: default: return fmt.Errorf("on_queue_full must be %q or %q, got %q", - TrafficLogQueueDropNew, TrafficLogQueueDropOldest, cfg.OnQueueFull) + QueueDropNew, QueueDropOldest, cfg.OnQueueFull) } if cfg.RequestTimeout <= 0 { return fmt.Errorf("request_timeout must be positive, got %s", cfg.RequestTimeout) diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go index 7fc2b12143..cc0f68dc54 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go @@ -92,9 +92,9 @@ func TestValidate_OTelPublisher(t *testing.T) { "non-http scheme": func(o *OTelPublisherConfig) { o.Endpoint = "grpc://collector:4317" }, "missing service name": func(o *OTelPublisherConfig) { o.ServiceName = "" }, "zero batch size": func(o *OTelPublisherConfig) { o.BatchSize = 0 }, - "zero queue size": func(o *OTelPublisherConfig) { o.QueueSize = 0 }, + "zero queue capacity": func(o *OTelPublisherConfig) { o.QueueCapacity = 0 }, // A queue smaller than a batch can never fill one. - "queue smaller than batch": func(o *OTelPublisherConfig) { o.QueueSize = 10; o.BatchSize = 100 }, + "queue smaller than batch": func(o *OTelPublisherConfig) { o.QueueCapacity = 10; o.BatchSize = 100 }, "zero flush interval": func(o *OTelPublisherConfig) { o.FlushInterval = 0 }, "negative timeout": func(o *OTelPublisherConfig) { o.Timeout = -time.Second }, } @@ -106,6 +106,26 @@ func TestValidate_OTelPublisher(t *testing.T) { }) } + // The two accepted values are shared with traffic_logging.http via + // QueueDropNew/QueueDropOldest, so both blocks must keep accepting both. + t.Run("both drop policies are accepted", func(t *testing.T) { + for _, policy := range []string{QueueDropNew, QueueDropOldest, " DROP_OLDEST "} { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.OnQueueFull = policy }) + assert.NoError(t, cfg.Validate(), "policy %q", policy) + } + }) + + // Silently defaulting an unrecognised value would give an operator who asked + // for drop_oldest the opposite behaviour. + t.Run("unknown drop policy is an error", func(t *testing.T) { + for _, policy := range []string{"", "drop", "evict_oldest", "drop_newest"} { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.OnQueueFull = policy }) + err := cfg.Validate() + require.Error(t, err, "policy %q", policy) + assert.Contains(t, err.Error(), "on_queue_full") + } + }) + // An unknown publisher name must fail rather than be silently skipped. t.Run("unknown publisher name is an error", func(t *testing.T) { cfg := otelConfig(func(*OTelPublisherConfig) {}) From 178f724171182f766978085a6e7e31778704e17d Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Mon, 7 Sep 2026 15:57:34 +0530 Subject: [PATCH 03/20] Adding Retry and export-failure handling --- gateway/configs/config-template.toml | 18 + .../internal/analytics/publishers/otel.go | 244 +++++++++++- .../analytics/publishers/otel_test.go | 374 +++++++++++++++++- .../policy-engine/internal/config/config.go | 66 +++- .../internal/config/otel_publisher_test.go | 59 +++ 5 files changed, 726 insertions(+), 35 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 1edca835bd..74d6ca5bdf 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -837,6 +837,24 @@ queue_capacity = 10000 on_queue_full = "drop_new" # Bounds a single export attempt. timeout = "10s" +# Retry attempts after the initial one. Only transport errors, 429 and 5xx are +# retried — any other 4xx means the endpoint rejected the payload's shape, which +# retrying can only amplify. 0 disables retries. +max_retries = 3 +# Base delay for exponential backoff, with jitter applied per attempt so replicas +# retrying after a shared outage do not resynchronize into a thundering herd. A +# Retry-After from the endpoint replaces this rather than adding to it. +retry_backoff = "1s" +# Fraction of queue_capacity at which a retrying batch abandons its remaining +# budget and returns to draining. One worker exports, so nothing drains the queue +# while a batch retries: past this depth, retrying to save one batch costs more +# records than it rescues. 0 lets every batch use its full budget. +retry_abort_queue_ratio = 0.5 +# "none" (default, matching the OTLP specification's own default) or "gzip". gzip +# trades CPU on the export worker for a large egress reduction — these records are +# verbose JSON — and every OTLP/HTTP receiver is required to support it. Worth +# enabling when the endpoint is across a network you pay for. +compression = "none" # ==== UNCOMMENT WHEN REQUIRED ================================ # Headers sent on every export, for a vendor OTLP intake that authenticates by diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 78d94ab3a1..52af6395ad 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -19,13 +19,16 @@ package publishers import ( "bytes" + "compress/gzip" "context" "crypto/tls" "crypto/x509" "encoding/json" + "errors" "fmt" "io" "log/slog" + "math/rand/v2" "net" "net/http" "net/url" @@ -59,6 +62,10 @@ const ( // otelCloseFlushTimeout bounds the shutdown flush when the caller's context // carries no deadline. otelCloseFlushTimeout = 5 * time.Second + // otelMaxResponseBytes caps how much of the endpoint's response is read. A + // 2xx body carries partialSuccess and a failure body carries an error + // message; neither may be allowed to grow the heap. + otelMaxResponseBytes int64 = 4 << 10 ) // ns qualifies an attribute name with the WSO2 namespace. @@ -96,6 +103,11 @@ type OTel struct { dropped int // dropOldest is resolved once at construction rather than per record. dropOldest bool + // gzip is resolved once at construction from cfg.Compression. + gzip bool + // retryAbortDepth is the queue depth at which a retrying batch gives up so + // the worker can resume draining. 0 disables the check. See export. + retryAbortDepth int } // NewOTel creates the OTLP-logs publisher and starts its exporting worker. @@ -132,6 +144,9 @@ func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { stop: make(chan struct{}), workerDone: make(chan struct{}), dropOldest: strings.EqualFold(strings.TrimSpace(cfg.OnQueueFull), config.QueueDropOldest), + gzip: strings.EqualFold(strings.TrimSpace(cfg.Compression), + config.OTelCompressionGzip), + retryAbortDepth: cfg.EffectiveRetryAbortDepth(), } go o.run() @@ -216,7 +231,7 @@ func (o *OTel) Publish(event *dto.Event) { // non-blocking Publish into an unbounded one. select { case <-o.queue: - o.countDrop() + o.countQueueDrop() default: } select { @@ -226,16 +241,13 @@ func (o *OTel) Publish(event *dto.Event) { } } - o.countDrop() + o.countQueueDrop() } -// countDrop records one dropped record, warning on the first and then every -// hundredth so a sustained outage cannot flood the log. -func (o *OTel) countDrop() { - o.droppedMu.Lock() - o.dropped++ - count := o.dropped - o.droppedMu.Unlock() +// countQueueDrop records one record dropped for a full queue, warning on the +// first and then every hundredth so a sustained outage cannot flood the log. +func (o *OTel) countQueueDrop() { + count := o.countDrops(1) if count == 1 || count%100 == 0 { slog.Warn("OTel publisher queue full; dropping analytics event", "droppedTotal", count, "queueCapacity", o.cfg.QueueCapacity, @@ -312,7 +324,10 @@ func (o *OTel) Close(ctx context.Context) error { return o.closeErr } -// export POSTs one batch as a single OTLP/HTTP logs request. +// export builds the OTLP payload for one batch and delivers it, retrying +// transport errors, 429 and 5xx with jittered exponential backoff. Any other 4xx +// means the endpoint rejected the payload's shape, so retrying would only amplify +// a permanent failure. func (o *OTel) export(batch []*otelLogRecord) { records := make([]otelLogRecord, 0, len(batch)) for _, r := range batch { @@ -337,36 +352,213 @@ func (o *OTel) export(batch []*otelLogRecord) { }) if err != nil { slog.Error("OTel publisher failed to marshal OTLP payload", "error", err, "records", len(records)) + o.countDrops(len(records)) return } + if o.gzip { + compressed, err := gzipBytes(body) + if err != nil { + slog.Error("OTel publisher failed to compress OTLP payload", "error", err, "records", len(records)) + o.countDrops(len(records)) + return + } + body = compressed + } + + var lastErr error + // Delay before the NEXT attempt. A Retry-After replaces our own backoff + // rather than adding to it, so the endpoint's own pacing is what applies. + var nextDelay time.Duration + for attempt := 0; attempt <= o.cfg.MaxRetries; attempt++ { + if attempt > 0 { + // Head-of-line check, before committing to another wait. One worker + // exports, so nothing drains the queue while this batch retries. Past + // the abort depth, retrying to save this batch costs more newer records + // to queue-full than it rescues — so abandon it and resume draining. + if depth := len(o.queue); o.retryAbortDepth > 0 && depth >= o.retryAbortDepth { + o.countDrops(len(records)) + slog.Error("OTel publisher abandoning batch retries to resume draining; the endpoint "+ + "is reachable but too slow to keep up", + "records", len(records), "attempts", attempt, + "queueDepth", depth, "queueCapacity", o.cfg.QueueCapacity, "error", lastErr) + return + } + if !o.sleep(nextDelay, attempt) { + break // shutting down: stop retrying rather than hold shutdown open + } + } + + retryAfter, err := o.post(body, len(records)) + if err == nil { + return + } + lastErr = err + + var perm *otelPermanentExportError + if errors.As(err, &perm) { + break // 4xx other than 429 — retrying cannot help + } + nextDelay = retryAfter // 0 unless the endpoint asked for a specific delay + } + + o.countDrops(len(records)) + slog.Error("OTel publisher failed to export analytics batch; dropping records", + "records", len(records), "attempts", o.cfg.MaxRetries+1, + "endpoint", o.cfg.Endpoint, "error", lastErr) +} +// otelPermanentExportError marks a response that must not be retried. +type otelPermanentExportError struct{ status int } + +func (e *otelPermanentExportError) Error() string { + return fmt.Sprintf("endpoint rejected the batch with status %d", e.status) +} + +// post performs one export attempt, returning the endpoint's requested +// Retry-After when it supplies one so the caller can honor it over its own +// backoff. +func (o *OTel) post(body []byte, records int) (time.Duration, error) { ctx, cancel := context.WithTimeout(context.Background(), o.cfg.Timeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPost, o.cfg.Endpoint, bytes.NewReader(body)) if err != nil { - slog.Error("OTel publisher failed to build OTLP request", "error", err) - return + return 0, fmt.Errorf("building request: %w", err) } req.Header.Set("Content-Type", "application/json") + if o.gzip { + req.Header.Set("Content-Encoding", "gzip") + } for k, v := range o.cfg.Headers { req.Header.Set(k, v) } resp, err := o.client.Do(req) if err != nil { - slog.Error("OTel publisher export failed", "error", err, "endpoint", o.cfg.Endpoint, "records", len(records)) - return + return 0, fmt.Errorf("posting batch: %w", err) } defer resp.Body.Close() - respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - if resp.StatusCode < 200 || resp.StatusCode > 299 { - slog.Error("OTel collector rejected the export", - "status", resp.StatusCode, "records", len(records), "response", string(respBody)) + // Read a bounded prefix: enough for the partialSuccess field or an error + // message, and capped so a hostile endpoint cannot grow the heap. + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, otelMaxResponseBytes)) + _, _ = io.Copy(io.Discard, resp.Body) // drain the rest so the connection is reusable + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + o.logPartialSuccess(respBody, records) + return 0, nil + } + + if resp.StatusCode == http.StatusTooManyRequests { + return parseRetryAfter(resp.Header.Get("Retry-After")), + fmt.Errorf("endpoint is rate limiting (429)") + } + if resp.StatusCode >= 500 { + return 0, fmt.Errorf("endpoint returned status %d: %s", resp.StatusCode, otelResponseExcerpt(respBody)) + } + return 0, &otelPermanentExportError{status: resp.StatusCode} +} + +// logPartialSuccess reports records the endpoint accepted the request for but +// rejected. Without this a 200 carrying rejectedLogRecords looks like a clean +// export, and the records are silently gone. +func (o *OTel) logPartialSuccess(respBody []byte, records int) { + if len(respBody) == 0 { + return + } + var parsed otelExportResponse + if err := json.Unmarshal(respBody, &parsed); err != nil { + return // a non-JSON 2xx body is not an error; nothing to report + } + // Proto3 JSON encodes int64 as a string, but some receivers emit a bare + // number, so the field is json.Number to accept either. + rejected, err := parsed.PartialSuccess.RejectedLogRecords.Int64() + if err != nil || rejected <= 0 { return } - slog.Debug("OTel publisher exported analytics events", "records", len(records), "status", resp.StatusCode) + o.countDrops(int(rejected)) + slog.Error("OTel endpoint accepted the export but rejected records", + "rejected", rejected, "records", records, + "endpointMessage", parsed.PartialSuccess.ErrorMessage) +} + +// sleep waits out the backoff before a retry, returning false if shutdown was +// requested first. delay is the endpoint's Retry-After when it supplied one, +// otherwise the jittered exponential backoff for this attempt. +func (o *OTel) sleep(delay time.Duration, attempt int) bool { + if delay <= 0 { + delay = o.backoff(attempt) + } + if delay <= 0 { + return true + } + t := time.NewTimer(delay) + defer t.Stop() + select { + case <-t.C: + return true + case <-o.stop: + return false + } +} + +// backoff returns the delay before the given retry attempt (1-based), growing +// exponentially with full jitter. Jitter matters because every replica retries +// against the same endpoint after a shared outage; without it they reconverge +// into a synchronized herd on the first recovery. +func (o *OTel) backoff(attempt int) time.Duration { + base := o.cfg.RetryBackoff + if base <= 0 { + base = time.Second + } + // Cap the exponent so a large max_retries cannot overflow the shift. + shift := attempt - 1 + if shift > 10 { + shift = 10 + } + delay := base << shift + if half := delay / 2; half > 0 { + delay = half + time.Duration(rand.Int64N(int64(half))) + } + return delay +} + +// countDrops adds n to the dropped total and returns the new total. It does not +// log: every caller has something more specific to say than "a record was lost". +func (o *OTel) countDrops(n int) int { + if n <= 0 { + return 0 + } + o.droppedMu.Lock() + o.dropped += n + total := o.dropped + o.droppedMu.Unlock() + return total +} + +// otelResponseExcerpt renders a bounded, single-line excerpt of an endpoint's +// error body for the log. The body is the endpoint's own text, never ours. +func otelResponseExcerpt(body []byte) string { + const maxExcerpt = 256 + excerpt := strings.TrimSpace(string(body)) + if len(excerpt) > maxExcerpt { + excerpt = excerpt[:maxExcerpt] + "..." + } + return strings.ReplaceAll(excerpt, "\n", " ") +} + +// gzipBytes compresses the payload for Content-Encoding: gzip. +func gzipBytes(body []byte) ([]byte, error) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(body); err != nil { + zw.Close() + return nil, err + } + if err := zw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil } // buildRecord maps the canonical analytics event onto one OTLP log record. @@ -656,6 +848,20 @@ type otelLogRecord struct { Attributes []otelKeyValue `json:"attributes"` } +// otelExportResponse is the OTLP ExportLogsServiceResponse. A 2xx can still +// report records the endpoint refused, which is otherwise indistinguishable from +// a clean export. +type otelExportResponse struct { + PartialSuccess otelPartialSuccess `json:"partialSuccess"` +} + +type otelPartialSuccess struct { + // json.Number because proto3 JSON encodes int64 as a string while some + // receivers emit a bare number; either must parse. + RejectedLogRecords json.Number `json:"rejectedLogRecords"` + ErrorMessage string `json:"errorMessage"` +} + type otelKeyValue struct { Key string `json:"key"` Value otelAnyValue `json:"value"` diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 45dd36c179..8c7a18a937 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -18,6 +18,7 @@ package publishers import ( + "compress/gzip" "context" "crypto/ecdsa" "crypto/elliptic" @@ -485,23 +486,38 @@ func correlatedEvent(id string) *dto.Event { return event } -// newUndrainedOTel builds a publisher with no worker goroutine, so nothing -// consumes the queue and Publish sees it full. NewOTel cannot be used here: it -// starts the worker. +// newTestOTel builds a publisher with no worker goroutine, mirroring the derived +// fields NewOTel computes. NewOTel cannot be used where a test drives export or +// the queue directly, because it starts the worker that would drain them. +func newTestOTel(t *testing.T, cfg config.OTelPublisherConfig) *OTel { + t.Helper() + return &OTel{ + cfg: cfg, + client: &http.Client{Timeout: cfg.Timeout}, + queue: make(chan *otelLogRecord, cfg.QueueCapacity), + stop: make(chan struct{}), + workerDone: make(chan struct{}), + dropOldest: strings.EqualFold(cfg.OnQueueFull, config.QueueDropOldest), + gzip: strings.EqualFold(cfg.Compression, config.OTelCompressionGzip), + retryAbortDepth: cfg.EffectiveRetryAbortDepth(), + } +} + +// newUndrainedOTel builds a publisher whose queue nothing consumes, so Publish +// sees it full. func newUndrainedOTel(t *testing.T, capacity int, onQueueFull string) *OTel { t.Helper() cfg := testOTelConfig("http://127.0.0.1:1/v1/logs") cfg.QueueCapacity = capacity cfg.BatchSize = capacity cfg.OnQueueFull = onQueueFull - return &OTel{ - cfg: cfg, - client: &http.Client{Timeout: cfg.Timeout}, - queue: make(chan *otelLogRecord, cfg.QueueCapacity), - stop: make(chan struct{}), - workerDone: make(chan struct{}), - dropOldest: strings.EqualFold(onQueueFull, config.QueueDropOldest), - } + return newTestOTel(t, cfg) +} + +func (o *OTel) droppedCount() int { + o.droppedMu.Lock() + defer o.droppedMu.Unlock() + return o.dropped } // A full queue must drop rather than block the ALS ingest path — under either @@ -514,10 +530,7 @@ func TestPublishDropsWhenQueueFull(t *testing.T) { o.Publish(restEvent()) } - o.droppedMu.Lock() - dropped := o.dropped - o.droppedMu.Unlock() - if dropped != 9 { + if dropped := o.droppedCount(); dropped != 9 { t.Errorf("dropped = %d, want 9 (queue holds 1)", dropped) } if len(o.queue) != 1 { @@ -753,3 +766,334 @@ func TestNewOTelFailsClosedOnBadTLSMaterial(t *testing.T) { t.Error("expected NewOTel to fail on an unreadable ca_file") } } + +// --- retry and export-failure handling ------------------------------------- + +// scriptedEndpoint serves the given statuses in order, repeating the last one +// once the script is exhausted, and records every request it received. +type scriptedEndpoint struct { + mu sync.Mutex + statuses []int + requests []*http.Request + bodies [][]byte + headers http.Header +} + +func newScriptedEndpoint(t *testing.T, statuses ...int) (*scriptedEndpoint, string) { + t.Helper() + e := &scriptedEndpoint{statuses: statuses, headers: http.Header{}} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + e.mu.Lock() + attempt := len(e.requests) + e.requests = append(e.requests, r) + e.bodies = append(e.bodies, body) + status := e.statuses[len(e.statuses)-1] + if attempt < len(e.statuses) { + status = e.statuses[attempt] + } + e.mu.Unlock() + + for k, vs := range e.headers { + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(status) + })) + t.Cleanup(server.Close) + return e, server.URL + "/v1/logs" +} + +func (e *scriptedEndpoint) attempts() int { + e.mu.Lock() + defer e.mu.Unlock() + return len(e.requests) +} + +// retryConfig enables retries with a short backoff so tests stay fast. +func retryConfig(endpoint string, maxRetries int) config.OTelPublisherConfig { + cfg := testOTelConfig(endpoint) + cfg.MaxRetries = maxRetries + cfg.RetryBackoff = 5 * time.Millisecond + return cfg +} + +func (o *OTel) exportOne(t *testing.T) { + t.Helper() + o.export([]*otelLogRecord{o.buildRecord(restEvent())}) +} + +// A 5xx is transient: retry until it clears, and lose nothing when it does. +func TestExportRetriesOn5xxThenSucceeds(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503, 500, 200) + o := newTestOTel(t, retryConfig(url, 3)) + o.exportOne(t) + + if got := endpoint.attempts(); got != 3 { + t.Errorf("attempts = %d, want 3 (two failures then success)", got) + } + if dropped := o.droppedCount(); dropped != 0 { + t.Errorf("dropped = %d, want 0: the batch was delivered", dropped) + } +} + +// A 4xx other than 429 means the payload's shape was rejected. Retrying cannot +// fix that, and would multiply a permanent failure by the retry budget. +func TestExportDoesNotRetryPermanentRejection(t *testing.T) { + for _, status := range []int{400, 401, 404, 422} { + t.Run(http.StatusText(status), func(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, status) + o := newTestOTel(t, retryConfig(url, 3)) + o.exportOne(t) + + if got := endpoint.attempts(); got != 1 { + t.Errorf("attempts = %d, want 1 (no retry on %d)", got, status) + } + if dropped := o.droppedCount(); dropped != 1 { + t.Errorf("dropped = %d, want 1", dropped) + } + }) + } +} + +// 429 is retryable, and the endpoint's Retry-After replaces our own backoff +// rather than adding to it. +func TestExportHonoursRetryAfterOn429(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 429, 200) + endpoint.headers.Set("Retry-After", "1") + + cfg := retryConfig(url, 3) + cfg.RetryBackoff = time.Millisecond // far shorter than Retry-After + o := newTestOTel(t, cfg) + + start := time.Now() + o.exportOne(t) + elapsed := time.Since(start) + + if got := endpoint.attempts(); got != 2 { + t.Errorf("attempts = %d, want 2", got) + } + // The wait must come from Retry-After, not the 1ms backoff. + if elapsed < 900*time.Millisecond { + t.Errorf("elapsed = %s, want >= ~1s from Retry-After", elapsed) + } + if dropped := o.droppedCount(); dropped != 0 { + t.Errorf("dropped = %d, want 0", dropped) + } +} + +// Exhausting the budget drops the batch exactly once, counting every record. +func TestExportDropsBatchAfterBudgetExhausted(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503) + o := newTestOTel(t, retryConfig(url, 2)) + + batch := []*otelLogRecord{ + o.buildRecord(restEvent()), o.buildRecord(restEvent()), o.buildRecord(restEvent()), + } + o.export(batch) + + if got := endpoint.attempts(); got != 3 { + t.Errorf("attempts = %d, want 3 (initial + 2 retries)", got) + } + if dropped := o.droppedCount(); dropped != 3 { + t.Errorf("dropped = %d, want 3 (every record in the batch)", dropped) + } +} + +// One worker exports, so nothing drains the queue while a batch retries. Past +// the abort depth, retrying to save this batch costs more newer records than it +// rescues — so it must abandon its budget and return to draining. +func TestExportAbandonsRetriesUnderQueuePressure(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503) + cfg := retryConfig(url, 5) + cfg.RetryBackoff = time.Millisecond + cfg.QueueCapacity = 4 + cfg.RetryAbortQueueRatio = 0.5 // abort depth 2 + o := newTestOTel(t, cfg) + + if o.retryAbortDepth != 2 { + t.Fatalf("retryAbortDepth = %d, want 2", o.retryAbortDepth) + } + // Fill past the abort depth; nothing drains it. + for i := 0; i < 3; i++ { + o.queue <- o.buildRecord(restEvent()) + } + + o.exportOne(t) + + // The first attempt happens unconditionally; the depth check runs before the + // second, so exactly one attempt is made instead of the budgeted six. + if got := endpoint.attempts(); got != 1 { + t.Errorf("attempts = %d, want 1 (abandoned before the first retry)", got) + } + if dropped := o.droppedCount(); dropped != 1 { + t.Errorf("dropped = %d, want 1", dropped) + } +} + +// A ratio of 0 disables the check, so every batch gets its full budget. +func TestExportZeroAbortRatioUsesFullBudget(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503) + cfg := retryConfig(url, 2) + cfg.QueueCapacity = 2 + cfg.RetryAbortQueueRatio = 0 + o := newTestOTel(t, cfg) + + if o.retryAbortDepth != 0 { + t.Fatalf("retryAbortDepth = %d, want 0 (check disabled)", o.retryAbortDepth) + } + o.queue <- o.buildRecord(restEvent()) + o.queue <- o.buildRecord(restEvent()) + + o.exportOne(t) + if got := endpoint.attempts(); got != 3 { + t.Errorf("attempts = %d, want 3 despite a full queue", got) + } +} + +// A 2xx carrying partialSuccess is not a clean export: those records are gone, +// and must be counted rather than silently discarded. +func TestExportCountsPartialSuccessRejections(t *testing.T) { + cases := map[string]string{ + // Proto3 JSON encodes int64 as a string; some receivers emit a number. + "int64 as string": `{"partialSuccess":{"rejectedLogRecords":"2","errorMessage":"bad attribute"}}`, + "bare number": `{"partialSuccess":{"rejectedLogRecords":2,"errorMessage":"bad attribute"}}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(body)) + })) + defer server.Close() + + o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 3)) + o.exportOne(t) + + if dropped := o.droppedCount(); dropped != 2 { + t.Errorf("dropped = %d, want 2 from partialSuccess", dropped) + } + }) + } +} + +// The ordinary success shapes must not be read as rejections. +func TestExportCleanSuccessBodiesCountNoDrops(t *testing.T) { + bodies := map[string]string{ + "empty": ``, + "empty object": `{}`, + "empty partial success": `{"partialSuccess":{}}`, + "explicit zero rejected": `{"partialSuccess":{"rejectedLogRecords":"0"}}`, + "not json": `OK`, + } + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(body)) + })) + defer server.Close() + + o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 0)) + o.exportOne(t) + if dropped := o.droppedCount(); dropped != 0 { + t.Errorf("dropped = %d, want 0", dropped) + } + }) + } +} + +// gzip must set Content-Encoding and produce a body the endpoint can inflate +// back into the same OTLP payload. +func TestExportGzipCompression(t *testing.T) { + type received struct { + encoding string + payload otelExportRequest + } + got := make(chan received, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + zr, err := gzip.NewReader(r.Body) + if err != nil { + t.Errorf("body is not gzip: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + defer zr.Close() + raw, err := io.ReadAll(zr) + if err != nil { + t.Errorf("inflate: %v", err) + } + var payload otelExportRequest + if err := json.Unmarshal(raw, &payload); err != nil { + t.Errorf("inflated body is not OTLP JSON: %v", err) + } + got <- received{encoding: r.Header.Get("Content-Encoding"), payload: payload} + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + cfg := retryConfig(server.URL+"/v1/logs", 0) + cfg.Compression = config.OTelCompressionGzip + o := newTestOTel(t, cfg) + o.exportOne(t) + + select { + case r := <-got: + if r.encoding != "gzip" { + t.Errorf("Content-Encoding = %q, want gzip", r.encoding) + } + if len(r.payload.ResourceLogs) != 1 || len(r.payload.ResourceLogs[0].ScopeLogs[0].LogRecords) != 1 { + t.Error("inflated payload did not carry the record") + } + case <-time.After(3 * time.Second): + t.Fatal("no export received") + } +} + +// Uncompressed is the default, and must not claim an encoding it did not apply. +func TestExportUncompressedByDefault(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 200) + o := newTestOTel(t, retryConfig(url, 0)) + o.exportOne(t) + + endpoint.mu.Lock() + defer endpoint.mu.Unlock() + if enc := endpoint.requests[0].Header.Get("Content-Encoding"); enc != "" { + t.Errorf("Content-Encoding = %q, want empty", enc) + } + if !json.Valid(endpoint.bodies[0]) { + t.Error("body is not plain JSON") + } +} + +// Shutdown must not wait out the remaining backoff: a retrying batch has to +// notice the stop signal instead of holding shutdown open. +func TestExportStopsRetryingOnShutdown(t *testing.T) { + endpoint, url := newScriptedEndpoint(t, 503) + cfg := retryConfig(url, 100) + cfg.RetryBackoff = 30 * time.Second // long enough that waiting it out would fail the test + o := newTestOTel(t, cfg) + + done := make(chan struct{}) + go func() { + o.exportOne(t) + close(done) + }() + + // Wait for the first attempt to fail, then signal shutdown mid-backoff. + deadline := time.Now().Add(2 * time.Second) + for endpoint.attempts() == 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + close(o.stop) + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("export kept retrying through shutdown") + } + if got := endpoint.attempts(); got != 1 { + t.Errorf("attempts = %d, want 1 (stopped during the first backoff)", got) + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index bd501dd9a4..5eec64a74e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -158,14 +158,56 @@ type OTelPublisherConfig struct { // QueueCapacity bounds records held in memory when the endpoint is slow. // Once full, OnQueueFull decides which record is dropped. QueueCapacity int `koanf:"queue_capacity"` - // OnQueueFull is QueueDropNew (default) or QueueDropOldest. + // OnQueueFull is QueueDropNew (default) or QueueDropOldest. OnQueueFull string `koanf:"on_queue_full"` // Timeout bounds a single export attempt. Timeout time.Duration `koanf:"timeout"` + // MaxRetries is the number of retry attempts after the initial one. Only + // transport errors, 429 and 5xx are retried; any other 4xx means the endpoint + // rejected the payload's shape, which retrying can only amplify. + MaxRetries int `koanf:"max_retries"` + // RetryBackoff is the base delay for exponential backoff, with full jitter + // applied per attempt so replicas retrying after a shared outage do not + // resynchronize into a thundering herd. + RetryBackoff time.Duration `koanf:"retry_backoff"` + // RetryAbortQueueRatio is the fraction of QueueCapacity at which a retrying + // batch abandons its remaining budget and returns to draining. One worker + // exports, so nothing drains the queue while a batch retries: past this + // depth, retrying to save one batch costs more records than it rescues. 0 + // disables the check and lets every batch use its full budget. + RetryAbortQueueRatio float64 `koanf:"retry_abort_queue_ratio"` + // Compression is "none" (default, the OTLP spec's own default) or "gzip". + // gzip trades CPU on the export worker for a large egress reduction — these + // records are verbose JSON — and every OTLP/HTTP receiver must support it. + Compression string `koanf:"compression"` // TLS configures the client side of an https endpoint. Ignored for http. TLS OTelTLSConfig `koanf:"tls"` } +// Accepted values for analytics.publishers.otel.compression. +const ( + // OTelCompressionNone sends the OTLP-JSON payload uncompressed. + OTelCompressionNone = "none" + // OTelCompressionGzip sends it gzip-encoded with Content-Encoding: gzip. + OTelCompressionGzip = "gzip" +) + +// DefaultOTelRetryAbortQueueRatio is the fraction of the OTel publisher's queue at +// which a retrying batch gives up, matching the traffic-log sink's midpoint: high +// enough that an ordinary blip still gets its full retry budget, low enough that a +// hung endpoint cannot consume the whole queue before exporting resumes. +const DefaultOTelRetryAbortQueueRatio = 0.5 + +// EffectiveRetryAbortDepth returns the queue depth at which a retrying batch stops +// retrying. Zero means the check is disabled. +func (c OTelPublisherConfig) EffectiveRetryAbortDepth() int { + depth := int(float64(c.QueueCapacity) * c.RetryAbortQueueRatio) + if depth < 1 && c.RetryAbortQueueRatio > 0 { + depth = 1 + } + return depth +} + // OTelTLSConfig configures TLS to the OTLP endpoint // ([analytics.publishers.otel.tls]). Deliberately a separate type from // TrafficLogHTTPTLSConfig despite the identical keys: the two config blocks are @@ -1230,6 +1272,11 @@ func defaultConfig() *Config { QueueCapacity: 10000, OnQueueFull: QueueDropNew, Timeout: 10 * time.Second, + MaxRetries: 3, + RetryBackoff: time.Second, + // Half: retry freely while the queue is shallow, stop once it fills. + RetryAbortQueueRatio: DefaultOTelRetryAbortQueueRatio, + Compression: OTelCompressionNone, }, }, GRPCEventServerCfg: map[string]interface{}{ @@ -1554,6 +1601,23 @@ func validateOTelPublisherConfig(cfg OTelPublisherConfig) error { if cfg.Timeout <= 0 { return fmt.Errorf("analytics.publishers.otel.timeout must be > 0, got %s", cfg.Timeout) } + if cfg.MaxRetries < 0 { + return fmt.Errorf("analytics.publishers.otel.max_retries must be >= 0, got %d", cfg.MaxRetries) + } + if cfg.MaxRetries > 0 && cfg.RetryBackoff <= 0 { + return fmt.Errorf("analytics.publishers.otel.retry_backoff must be positive when max_retries > 0, got %s", + cfg.RetryBackoff) + } + if cfg.RetryAbortQueueRatio < 0 || cfg.RetryAbortQueueRatio > 1 { + return fmt.Errorf("analytics.publishers.otel.retry_abort_queue_ratio must be between 0 and 1, got %v", + cfg.RetryAbortQueueRatio) + } + switch strings.ToLower(strings.TrimSpace(cfg.Compression)) { + case "", OTelCompressionNone, OTelCompressionGzip: + default: + return fmt.Errorf("analytics.publishers.otel.compression must be %q or %q, got %q", + OTelCompressionNone, OTelCompressionGzip, cfg.Compression) + } if err := validateOTelTLS(cfg.TLS, u.Host); err != nil { return fmt.Errorf("analytics.publishers.otel.tls: %w", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go index cc0f68dc54..be89344ce1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go @@ -190,3 +190,62 @@ func TestValidate_OTelPublisherTLS(t *testing.T) { assert.NoError(t, cfg.Validate()) }) } + +func TestValidate_OTelPublisherRetry(t *testing.T) { + t.Run("defaults are retry-enabled", func(t *testing.T) { + cfg := defaultConfig().Analytics.Publishers.OTel + assert.Equal(t, 3, cfg.MaxRetries) + assert.Equal(t, time.Second, cfg.RetryBackoff) + assert.Equal(t, DefaultOTelRetryAbortQueueRatio, cfg.RetryAbortQueueRatio) + assert.Equal(t, OTelCompressionNone, cfg.Compression) + }) + + // Retries off is a legitimate choice, so 0 must pass while negative fails. + t.Run("zero retries is allowed", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.MaxRetries = 0; o.RetryBackoff = 0 }) + assert.NoError(t, cfg.Validate()) + }) + + rejected := map[string]func(*OTelPublisherConfig){ + "negative retries": func(o *OTelPublisherConfig) { o.MaxRetries = -1 }, + "retries without backoff": func(o *OTelPublisherConfig) { o.MaxRetries = 3; o.RetryBackoff = 0 }, + "negative backoff": func(o *OTelPublisherConfig) { o.MaxRetries = 3; o.RetryBackoff = -time.Second }, + "abort ratio above one": func(o *OTelPublisherConfig) { o.RetryAbortQueueRatio = 1.5 }, + "negative abort ratio": func(o *OTelPublisherConfig) { o.RetryAbortQueueRatio = -0.1 }, + "unknown compression": func(o *OTelPublisherConfig) { o.Compression = "zstd" }, + } + for name, mutate := range rejected { + t.Run(name, func(t *testing.T) { + err := otelConfig(mutate).Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "analytics.publishers.otel") + }) + } + + t.Run("compression values accepted", func(t *testing.T) { + for _, c := range []string{"", OTelCompressionNone, OTelCompressionGzip, " GZIP "} { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.Compression = c }) + assert.NoError(t, cfg.Validate(), "compression %q", c) + } + }) + + // The abort depth is a fraction of capacity, and must never round down to + // zero for a non-zero ratio: that would silently disable the check. + t.Run("abort depth", func(t *testing.T) { + cases := []struct { + capacity int + ratio float64 + want int + }{ + {10000, 0.5, 5000}, + {4, 0.5, 2}, + {1, 0.5, 1}, // rounds to 0, floored to 1 + {10000, 0, 0}, // explicitly disabled + } + for _, tc := range cases { + cfg := OTelPublisherConfig{QueueCapacity: tc.capacity, RetryAbortQueueRatio: tc.ratio} + assert.Equal(t, tc.want, cfg.EffectiveRetryAbortDepth(), + "capacity %d ratio %v", tc.capacity, tc.ratio) + } + }) +} From 1477d19351c0f9279cc840eb5e70ca95e7ff1e78 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Mon, 7 Sep 2026 17:07:10 +0530 Subject: [PATCH 04/20] Mapping the Publisher self-observability metrics --- .../internal/analytics/publishers/otel.go | 116 ++++++++- .../analytics/publishers/otel_test.go | 243 ++++++++++++++++++ .../analytics/publishers/sink_factory.go | 26 +- .../policy-engine/internal/metrics/metrics.go | 86 +++++++ 4 files changed, 457 insertions(+), 14 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 52af6395ad..25798dbc72 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -41,6 +41,7 @@ import ( "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/constants" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/metrics" ) // Identifiers with more than one reader. Individual attribute names are written @@ -62,6 +63,9 @@ const ( // otelCloseFlushTimeout bounds the shutdown flush when the caller's context // carries no deadline. otelCloseFlushTimeout = 5 * time.Second + // otelPublisherName is this publisher's value for the `publisher` metric + // label. Kept local so metric call sites need no config import. + otelPublisherName = "otel" // otelMaxResponseBytes caps how much of the endpoint's response is read. A // 2xx body carries partialSuccess and a failure body carries an error // message; neither may be allowed to grow the heap. @@ -148,6 +152,7 @@ func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { config.OTelCompressionGzip), retryAbortDepth: cfg.EffectiveRetryAbortDepth(), } + o.initMetrics() go o.run() if u, err := url.Parse(cfg.Endpoint); err == nil && u.Scheme == "http" && !isLoopbackHost(u.Hostname()) { @@ -221,6 +226,7 @@ func (o *OTel) Publish(event *dto.Event) { select { case o.queue <- record: + mAnalyticsQueueDepth(otelPublisherName, len(o.queue)) return default: } @@ -236,6 +242,7 @@ func (o *OTel) Publish(event *dto.Event) { } select { case o.queue <- record: + mAnalyticsQueueDepth(otelPublisherName, len(o.queue)) return default: } @@ -247,6 +254,7 @@ func (o *OTel) Publish(event *dto.Event) { // countQueueDrop records one record dropped for a full queue, warning on the // first and then every hundredth so a sustained outage cannot flood the log. func (o *OTel) countQueueDrop() { + mAnalyticsDropped(otelPublisherName, dropReasonQueueFull, 1) count := o.countDrops(1) if count == 1 || count%100 == 0 { slog.Warn("OTel publisher queue full; dropping analytics event", @@ -257,7 +265,12 @@ func (o *OTel) countQueueDrop() { // run drains the queue, exporting on a full batch or on the flush interval. func (o *OTel) run() { - defer close(o.workerDone) + defer func() { + // The queue is not drained further after this point, so leaving the last + // non-zero depth published would read as a permanently backed-up queue. + mAnalyticsQueueDepth(otelPublisherName, 0) + close(o.workerDone) + }() ticker := time.NewTicker(o.cfg.FlushInterval) defer ticker.Stop() @@ -274,6 +287,7 @@ func (o *OTel) run() { for { select { case record := <-o.queue: + mAnalyticsQueueDepth(otelPublisherName, len(o.queue)) batch = append(batch, record) if len(batch) >= o.cfg.BatchSize { flush() @@ -285,6 +299,7 @@ func (o *OTel) run() { for drained := true; drained; { select { case record := <-o.queue: + mAnalyticsQueueDepth(otelPublisherName, len(o.queue)) batch = append(batch, record) if len(batch) >= o.cfg.BatchSize { flush() @@ -352,19 +367,24 @@ func (o *OTel) export(batch []*otelLogRecord) { }) if err != nil { slog.Error("OTel publisher failed to marshal OTLP payload", "error", err, "records", len(records)) - o.countDrops(len(records)) + o.dropRecords(dropReasonSerializeFailed, len(records)) return } if o.gzip { compressed, err := gzipBytes(body) if err != nil { slog.Error("OTel publisher failed to compress OTLP payload", "error", err, "records", len(records)) - o.countDrops(len(records)) + o.dropRecords(dropReasonSerializeFailed, len(records)) return } body = compressed } + // Covers every attempt and the waits between them: that total is what holds + // the worker, and therefore what lets the queue fill behind it. + start := time.Now() + defer func() { mAnalyticsExportDuration(otelPublisherName, time.Since(start).Seconds()) }() + var lastErr error // Delay before the NEXT attempt. A Retry-After replaces our own backoff // rather than adding to it, so the endpoint's own pacing is what applies. @@ -376,7 +396,7 @@ func (o *OTel) export(batch []*otelLogRecord) { // the abort depth, retrying to save this batch costs more newer records // to queue-full than it rescues — so abandon it and resume draining. if depth := len(o.queue); o.retryAbortDepth > 0 && depth >= o.retryAbortDepth { - o.countDrops(len(records)) + o.dropRecords(dropReasonBackpressure, len(records)) slog.Error("OTel publisher abandoning batch retries to resume draining; the endpoint "+ "is reachable but too slow to keep up", "records", len(records), "attempts", attempt, @@ -390,6 +410,7 @@ func (o *OTel) export(batch []*otelLogRecord) { retryAfter, err := o.post(body, len(records)) if err == nil { + mAnalyticsPublished(otelPublisherName, len(records)) return } lastErr = err @@ -401,7 +422,7 @@ func (o *OTel) export(batch []*otelLogRecord) { nextDelay = retryAfter // 0 unless the endpoint asked for a specific delay } - o.countDrops(len(records)) + o.dropRecords(dropReasonSendFailed, len(records)) slog.Error("OTel publisher failed to export analytics batch; dropping records", "records", len(records), "attempts", o.cfg.MaxRetries+1, "endpoint", o.cfg.Endpoint, "error", lastErr) @@ -435,6 +456,9 @@ func (o *OTel) post(body []byte, records int) (time.Duration, error) { resp, err := o.client.Do(req) if err != nil { + mAnalyticsExportError(otelPublisherName, errCodeTransport, 1) + // The error can embed the endpoint URL but never the payload, so no + // request data can leak into the application log here. return 0, fmt.Errorf("posting batch: %w", err) } defer resp.Body.Close() @@ -448,6 +472,7 @@ func (o *OTel) post(body []byte, records int) (time.Duration, error) { o.logPartialSuccess(respBody, records) return 0, nil } + mAnalyticsExportError(otelPublisherName, strconv.Itoa(resp.StatusCode), 1) if resp.StatusCode == http.StatusTooManyRequests { return parseRetryAfter(resp.Header.Get("Retry-After")), @@ -476,7 +501,7 @@ func (o *OTel) logPartialSuccess(respBody []byte, records int) { if err != nil || rejected <= 0 { return } - o.countDrops(int(rejected)) + o.dropRecords(dropReasonRejected, int(rejected)) slog.Error("OTel endpoint accepted the export but rejected records", "rejected", rejected, "records", records, "endpointMessage", parsed.PartialSuccess.ErrorMessage) @@ -523,6 +548,16 @@ func (o *OTel) backoff(attempt int) time.Duration { return delay } +// dropRecords counts n records lost for the given reason, on both the local +// total and the labelled metric. +func (o *OTel) dropRecords(reason string, n int) { + if n <= 0 { + return + } + mAnalyticsDropped(otelPublisherName, reason, n) + o.countDrops(n) +} + // countDrops adds n to the dropped total and returns the new total. It does not // log: every caller has something more specific to say than "a record was lost". func (o *OTel) countDrops(n int) int { @@ -536,6 +571,75 @@ func (o *OTel) countDrops(n int) int { return total } +// Metric helpers. +// +// Every analytics-publisher metric goes through these rather than touching the +// package vars directly. The vars are nil until metrics.Init() runs — main() +// calls it long before any publisher exists, but a constructor must not depend +// on that ordering, and guarding only in the constructor while the export path +// dereferences freely turns a startup panic into a first-request panic. + +func mAnalyticsPublished(publisher string, n int) { + if metrics.AnalyticsPublishedTotal != nil { + metrics.AnalyticsPublishedTotal.WithLabelValues(publisher).Add(float64(n)) + } +} + +func mAnalyticsDropped(publisher, reason string, n int) { + if metrics.AnalyticsDroppedTotal != nil { + metrics.AnalyticsDroppedTotal.WithLabelValues(publisher, reason).Add(float64(n)) + } +} + +func mAnalyticsQueueDepth(publisher string, depth int) { + if metrics.AnalyticsQueueDepth != nil { + metrics.AnalyticsQueueDepth.WithLabelValues(publisher).Set(float64(depth)) + } +} + +func mAnalyticsQueueCapacity(publisher string, capacity int) { + if metrics.AnalyticsQueueCapacity != nil { + metrics.AnalyticsQueueCapacity.WithLabelValues(publisher).Set(float64(capacity)) + } +} + +func mAnalyticsExportDuration(publisher string, seconds float64) { + if metrics.AnalyticsExportDurationSeconds != nil { + metrics.AnalyticsExportDurationSeconds.WithLabelValues(publisher).Observe(seconds) + } +} + +func mAnalyticsExportError(publisher, code string, n int) { + if metrics.AnalyticsExportErrorsTotal != nil { + metrics.AnalyticsExportErrorsTotal.WithLabelValues(publisher, code).Add(float64(n)) + } +} + +// initOTelMetrics materializes this publisher's counters at zero. +// +// A labelled Prometheus counter does not exist in the scrape until it is first +// incremented, so on a healthy gateway analytics_dropped_total is simply absent. +// That makes a dashboard panel read "No data" rather than 0, and leaves an +// operator unable to tell "nothing was dropped" from "the metrics path is +// broken" — an unacceptable ambiguity for the one series that makes silent +// analytics loss visible. +// +// Export-error codes are deliberately not pre-created: the label carries the +// HTTP status, which is unbounded, and materializing every possible status would +// be worse than the gap it closes. +func (o *OTel) initMetrics() { + mAnalyticsPublished(otelPublisherName, 0) + for _, reason := range []string{ + dropReasonQueueFull, dropReasonSendFailed, dropReasonBackpressure, + dropReasonRejected, dropReasonSerializeFailed, + } { + mAnalyticsDropped(otelPublisherName, reason, 0) + } + mAnalyticsExportError(otelPublisherName, errCodeTransport, 0) + mAnalyticsQueueCapacity(otelPublisherName, o.cfg.QueueCapacity) + mAnalyticsQueueDepth(otelPublisherName, 0) +} + // otelResponseExcerpt renders a bounded, single-line excerpt of an endpoint's // error body for the log. The body is the endpoint's own text, never ours. func otelResponseExcerpt(body []byte) string { diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 8c7a18a937..a27b651e80 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -34,14 +34,18 @@ import ( "net/http/httptest" "os" "path/filepath" + "sort" + "strconv" "strings" "sync" "testing" "time" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/constants" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/metrics" ) // Attribute names are written as literals throughout this file: they are the @@ -1097,3 +1101,242 @@ func TestExportStopsRetryingOnShutdown(t *testing.T) { t.Errorf("attempts = %d, want 1 (stopped during the first backoff)", got) } } + +// --- self-observability ---------------------------------------------------- + +// The publishers package shares one process and one registry across tests, so +// counters accumulate. Every assertion below is therefore on a delta. + +// scrapeMetrics renders the registry exactly as the policy-engine's /metrics +// endpoint does, so these assertions also prove the series actually reach a +// scrape — a metric that was never registered increments happily and is simply +// absent from the endpoint. +func scrapeMetrics(t *testing.T) string { + t.Helper() + rec := httptest.NewRecorder() + promhttp.HandlerFor(metrics.Init(), promhttp.HandlerOpts{}). + ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("/metrics returned %d", rec.Code) + } + return rec.Body.String() +} + +// seriesKey builds the exposition-format identifier for one series. Prometheus +// renders label pairs sorted by label NAME, not in the order the vec declared +// them, so the pairs are sorted here to match: code="..." precedes +// publisher="otel", while reason="..." follows it. +func seriesKey(name string, labels ...string) string { + pairs := append([]string{`publisher="otel"`}, labels...) + sort.Strings(pairs) + return name + "{" + strings.Join(pairs, ",") + "}" +} + +// metricValue reads a series out of a scrape, reporting whether it was present +// at all — "absent" and "zero" are different answers and tests need both. +func metricValue(t *testing.T, scrape, key string) (float64, bool) { + t.Helper() + for _, line := range strings.Split(scrape, "\n") { + rest, ok := strings.CutPrefix(line, key+" ") + if !ok { + continue + } + value, err := strconv.ParseFloat(strings.TrimSpace(rest), 64) + if err != nil { + t.Fatalf("unparseable value for %s: %q", key, rest) + } + return value, true + } + return 0, false +} + +// delta reports how much a series moved, requiring it to exist afterwards. +func delta(t *testing.T, before string, after string, key string) float64 { + t.Helper() + old, _ := metricValue(t, before, key) + current, ok := metricValue(t, after, key) + if !ok { + t.Fatalf("series %s is absent from the scrape", key) + } + return current - old +} + +// A successful export must be counted, and its duration observed. +func TestMetricsSuccessfulExport(t *testing.T) { + _, url := newScriptedEndpoint(t, 200) + o := newTestOTel(t, retryConfig(url, 0)) + + before := scrapeMetrics(t) + o.export([]*otelLogRecord{o.buildRecord(restEvent()), o.buildRecord(restEvent())}) + after := scrapeMetrics(t) + + published := seriesKey("policy_engine_analytics_published_total") + if got := delta(t, before, after, published); got != 2 { + t.Errorf("published delta = %v, want 2", got) + } + duration := seriesKey("policy_engine_analytics_export_duration_seconds_count") + if got := delta(t, before, after, duration); got != 1 { + t.Errorf("duration observation delta = %v, want 1", got) + } +} + +// Each failure mode must land on its own reason, so an operator can tell them +// apart: a slow endpoint (backpressure) is a different problem from a broken one +// (send_failed) or a full queue. +func TestMetricsDropReasons(t *testing.T) { + t.Run("queue_full", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_dropped_total", `reason="`+dropReasonQueueFull+`"`) + before := scrapeMetrics(t) + + o := newUndrainedOTel(t, 1, config.QueueDropNew) + for i := 0; i < 3; i++ { + o.Publish(restEvent()) + } + + if got := delta(t, before, scrapeMetrics(t), key); got != 2 { + t.Errorf("queue_full delta = %v, want 2", got) + } + }) + + t.Run("send_failed", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_dropped_total", `reason="`+dropReasonSendFailed+`"`) + before := scrapeMetrics(t) + + _, url := newScriptedEndpoint(t, 503) + o := newTestOTel(t, retryConfig(url, 1)) + o.exportOne(t) + + if got := delta(t, before, scrapeMetrics(t), key); got != 1 { + t.Errorf("send_failed delta = %v, want 1", got) + } + }) + + t.Run("backpressure", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_dropped_total", `reason="`+dropReasonBackpressure+`"`) + before := scrapeMetrics(t) + + _, url := newScriptedEndpoint(t, 503) + cfg := retryConfig(url, 5) + cfg.QueueCapacity = 2 + cfg.RetryAbortQueueRatio = 0.5 + o := newTestOTel(t, cfg) + o.queue <- o.buildRecord(restEvent()) + o.exportOne(t) + + if got := delta(t, before, scrapeMetrics(t), key); got != 1 { + t.Errorf("backpressure delta = %v, want 1", got) + } + }) + + t.Run("rejected", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_dropped_total", `reason="`+dropReasonRejected+`"`) + before := scrapeMetrics(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"partialSuccess":{"rejectedLogRecords":"3"}}`)) + })) + defer server.Close() + + o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 0)) + o.exportOne(t) + + if got := delta(t, before, scrapeMetrics(t), key); got != 3 { + t.Errorf("rejected delta = %v, want 3", got) + } + }) +} + +// The error code distinguishes an unreachable endpoint from one that answered +// with a status, which are diagnosed differently. +func TestMetricsExportErrorCodes(t *testing.T) { + t.Run("transport", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_export_errors_total", `code="`+errCodeTransport+`"`) + before := scrapeMetrics(t) + + o := newTestOTel(t, retryConfig("http://127.0.0.1:1/v1/logs", 0)) + o.exportOne(t) + + if got := delta(t, before, scrapeMetrics(t), key); got != 1 { + t.Errorf("transport delta = %v, want 1", got) + } + }) + + t.Run("http status", func(t *testing.T) { + key := seriesKey("policy_engine_analytics_export_errors_total", `code="503"`) + before := scrapeMetrics(t) + + _, url := newScriptedEndpoint(t, 503) + o := newTestOTel(t, retryConfig(url, 1)) + o.exportOne(t) + + // Both the initial attempt and the retry answered 503. + if got := delta(t, before, scrapeMetrics(t), key); got != 2 { + t.Errorf("503 delta = %v, want 2", got) + } + }) +} + +// Depth and capacity are published as a pair so an alert can express "the queue +// is 80% full" rather than an absolute depth that means nothing without it. +func TestMetricsQueueDepthAndCapacity(t *testing.T) { + _, url := newScriptedEndpoint(t, 200) + cfg := retryConfig(url, 0) + cfg.QueueCapacity = 8 + cfg.BatchSize = 8 + cfg.FlushInterval = time.Hour // park the worker so the queue holds + + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + defer publisher.Close(context.Background()) + + capacity, ok := metricValue(t, scrapeMetrics(t), seriesKey("policy_engine_analytics_queue_capacity")) + if !ok || capacity != 8 { + t.Errorf("capacity = %v (present=%v), want 8", capacity, ok) + } + + for i := 0; i < 3; i++ { + publisher.Publish(restEvent()) + } + // The worker consumes from the channel as records arrive, so depth is + // whatever is still buffered — assert it never exceeds capacity and was + // published at all rather than pinning an inherently racy exact value. + depth, ok := metricValue(t, scrapeMetrics(t), seriesKey("policy_engine_analytics_queue_depth")) + if !ok || depth < 0 || depth > 8 { + t.Errorf("depth = %v (present=%v), want within 0..8", depth, ok) + } +} + +// A labelled counter is absent from a scrape until first incremented, so a +// healthy gateway would show "No data" instead of 0 for the one series that +// makes silent analytics loss visible. +func TestMetricsPreInitializedAtZero(t *testing.T) { + _, url := newScriptedEndpoint(t, 200) + cfg := retryConfig(url, 0) + publisher, err := NewOTel(&cfg) + if err != nil { + t.Fatalf("NewOTel: %v", err) + } + defer publisher.Close(context.Background()) + + scrape := scrapeMetrics(t) + keys := []string{ + seriesKey("policy_engine_analytics_published_total"), + seriesKey("policy_engine_analytics_queue_capacity"), + seriesKey("policy_engine_analytics_queue_depth"), + seriesKey("policy_engine_analytics_export_errors_total", `code="`+errCodeTransport+`"`), + } + for _, reason := range []string{ + dropReasonQueueFull, dropReasonSendFailed, dropReasonBackpressure, + dropReasonRejected, dropReasonSerializeFailed, + } { + keys = append(keys, seriesKey("policy_engine_analytics_dropped_total", `reason="`+reason+`"`)) + } + for _, key := range keys { + if _, ok := metricValue(t, scrape, key); !ok { + t.Errorf("%s is absent from the scrape; a healthy gateway must report a value, not No data", key) + } + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_factory.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_factory.go index 529270cf97..233030c056 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_factory.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/sink_factory.go @@ -34,25 +34,35 @@ const ( sinkNameHTTP = config.TrafficLogSinkHTTP ) -// Reasons recorded on policy_engine_traffic_log_dropped_total. +// Reasons recorded on policy_engine_traffic_log_dropped_total and +// policy_engine_analytics_dropped_total. Shared so the two subsystems describe +// the same failure with the same label rather than each inventing a spelling. const ( - // dropReasonQueueFull: the HTTP sink's bounded queue had no room. + // dropReasonQueueFull: a bounded queue had no room for the incoming item. dropReasonQueueFull = "queue_full" - // dropReasonSendFailed: the HTTP sink exhausted its retries. + // dropReasonSendFailed: the sender exhausted its retry budget. dropReasonSendFailed = "send_failed" // dropReasonWriteFailed: a local write returned an error. dropReasonWriteFailed = "write_failed" // dropReasonRotateFailed: the file sink could not rotate, so the line that // triggered the rotation was not written. dropReasonRotateFailed = "rotate_failed" - // dropReasonBackpressure: the HTTP sink abandoned a batch's remaining retries - // because the queue was filling behind it. Distinct from send_failed so an - // operator can tell "the receiver is slow" from "the receiver is broken". + // dropReasonBackpressure: a batch's remaining retries were abandoned because + // the queue was filling behind it. Distinct from send_failed so an operator + // can tell "the destination is slow" from "the destination is broken". dropReasonBackpressure = "backpressure" + // dropReasonRejected: the destination accepted the request but refused some + // records (an OTLP partialSuccess). The request succeeded, so this is not an + // export error — but the records are gone just the same. + dropReasonRejected = "rejected" + // dropReasonSerializeFailed: the batch could not be encoded or compressed, so + // it was never sent. Retrying cannot help; the payload itself is the problem. + dropReasonSerializeFailed = "serialize_failed" ) -// Codes recorded on policy_engine_traffic_log_write_errors_total for non-HTTP -// failures. HTTP failures use the numeric status code instead. +// Codes recorded on policy_engine_traffic_log_write_errors_total and +// policy_engine_analytics_export_errors_total for non-HTTP failures. HTTP +// failures use the numeric status code instead. const ( errCodeWrite = "write" errCodeRotate = "rotate" diff --git a/gateway/gateway-runtime/policy-engine/internal/metrics/metrics.go b/gateway/gateway-runtime/policy-engine/internal/metrics/metrics.go index cc10dc58ec..7b36208092 100644 --- a/gateway/gateway-runtime/policy-engine/internal/metrics/metrics.go +++ b/gateway/gateway-runtime/policy-engine/internal/metrics/metrics.go @@ -70,6 +70,15 @@ var ( TrafficLogFlushDurationSecond HistogramVec TrafficLogWriteErrorsTotal CounterVec + // Analytics publisher metrics, labelled by publisher so a second publisher + // (Moesif, or a future one) reports on the same series rather than its own. + AnalyticsPublishedTotal CounterVec + AnalyticsDroppedTotal CounterVec + AnalyticsQueueDepth GaugeVec + AnalyticsQueueCapacity GaugeVec + AnalyticsExportDurationSeconds HistogramVec + AnalyticsExportErrorsTotal CounterVec + // ResolutionFailuresTotal counts requests whose logical operation could not be // resolved to a policy chain, labelled by resolver name and FailureKind. It // sits alongside RouteLookupFailuresTotal rather than replacing it: that one @@ -364,6 +373,75 @@ func initMetrics() { []string{"sink", "code"}, ) + // Analytics publisher metrics. An analytics event that never reaches its + // destination is invisible to the customer's own dashboards and billing + // views, so dropped_total is the series to alert on; the rest exist to + // diagnose it. Labelled by publisher rather than named per publisher, so + // "are we losing analytics?" is one query regardless of how many are enabled. + AnalyticsPublishedTotal = newCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "analytics_published_total", + Help: "Total number of analytics records successfully delivered, by publisher", + }, + []string{"publisher"}, + ) + + AnalyticsDroppedTotal = newCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "analytics_dropped_total", + Help: "Total number of analytics records dropped, by publisher and reason " + + "(queue_full, send_failed, backpressure, rejected, serialize_failed)", + }, + []string{"publisher", "reason"}, + ) + + AnalyticsQueueDepth = newGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "analytics_queue_depth", + Help: "Current number of analytics records queued for export, by publisher", + }, + []string{"publisher"}, + ) + + // Published so an alert can compare depth against capacity as a RATIO. A + // fixed depth threshold is meaningless on its own: 1000 is 10% of the + // default 10000 queue (fires far too early) and unreachable on a queue + // configured smaller than that (never fires at all). + AnalyticsQueueCapacity = newGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "analytics_queue_capacity", + Help: "Configured capacity of the analytics export queue, by publisher", + }, + []string{"publisher"}, + ) + + // Covers the whole delivery of one batch including retries and backoff, not a + // single request: that total is what holds the export worker, and therefore + // what lets the queue fill behind it. + AnalyticsExportDurationSeconds = newHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: "analytics_export_duration_seconds", + Help: "Duration of an analytics batch delivery including retries, by publisher", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0}, + }, + []string{"publisher"}, + ) + + AnalyticsExportErrorsTotal = newCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "analytics_export_errors_total", + Help: "Total number of analytics export errors, by publisher and code " + + "(HTTP status, or a short error class for non-HTTP failures)", + }, + []string{"publisher", "code"}, + ) + ResolutionFailuresTotal = newCounterVec( prometheus.CounterOpts{ Namespace: namespace, @@ -487,6 +565,14 @@ func initRegistry() { registerGaugeVec(TrafficLogQueueCapacity) registerHistogramVec(TrafficLogFlushDurationSecond) registerCounterVec(TrafficLogWriteErrorsTotal) + + registerCounterVec(AnalyticsPublishedTotal) + registerCounterVec(AnalyticsDroppedTotal) + registerGaugeVec(AnalyticsQueueDepth) + registerGaugeVec(AnalyticsQueueCapacity) + registerHistogramVec(AnalyticsExportDurationSeconds) + registerCounterVec(AnalyticsExportErrorsTotal) + registerCounterVec(ResolutionFailuresTotal) registerCounterVec(RouteResolutionIngestFailuresTotal) From b1223388fc1d4c35664cf36efe86ba3d27717c4f Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Tue, 8 Sep 2026 10:29:49 +0530 Subject: [PATCH 05/20] Adding errorCode, mcp_resouce_uri harnessing and mapping logic --- .../internal/analytics/analytics.go | 16 +- .../internal/analytics/dto/event.go | 5 + .../policy-engine/internal/analytics/fault.go | 218 ++++++++++++++ .../internal/analytics/fault_test.go | 281 ++++++++++++++++++ .../internal/analytics/publishers/moesif.go | 20 ++ .../analytics/publishers/moesif_test.go | 58 ++++ .../internal/analytics/publishers/otel.go | 9 +- .../analytics/publishers/otel_test.go | 106 ++++++- .../system-policies/analytics/analytics.go | 36 ++- .../analytics/analytics_test.go | 84 ++++++ 10 files changed, 807 insertions(+), 26 deletions(-) create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/fault.go create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 448c15ad16..d8a9f7e1cd 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -356,7 +356,7 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E // Prepare target target := dto.Target{} - target.ResponseCacheHit = false + target.ResponseCacheHit = isCacheHit(logEntry) if response != nil { target.TargetResponseCode = int(logEntry.GetResponse().GetResponseCode().Value) // target.Destination = keyValuePairsFromMetadata[DestinationKey] @@ -685,6 +685,20 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E event.Properties["mcpAnalytics"] = mcpAnalytics } + // Fault classification, last so it sees the finished event. + fault := classifyFault(logEntry) + event.EventCategory = fault.EventCategory + event.FaultCategory = fault.FaultCategory + event.ErrorType = fault.ErrorType + if fault.SubCategory != "" { + event.Error = &dto.Error{ + // The client-visible status. The in-development fault flow owns the + // real WSO2 numeric codes and should supply them here instead. + ErrorCode: event.ProxyResponseCode, + ErrorMessage: fault.SubCategory, + } + } + return event } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index a475b61041..5cbdf7e64b 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -76,6 +76,11 @@ type Event struct { ErrorType string `json:"errorType,omitempty" bson:"error_type"` Properties map[string]interface{} `json:"properties,omitempty" bson:"properties"` + // EventCategory and FaultCategory are derived from the Envoy access log's + // response flags and detail string by classifyFault + EventCategory EventCategory `json:"eventCategory,omitempty" bson:"event_category"` + FaultCategory FaultCategory `json:"faultCategory,omitempty" bson:"fault_category"` + // TrafficLogLatencies carries microsecond-precision gateway/backend timings // for the stdout traffic-logging publisher. It is computed from the same ALS // CommonProperties timepoints as Latencies but at full precision, and is kept diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go b/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go new file mode 100644 index 0000000000..2e1ecd4680 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package analytics + +import ( + "strconv" + + v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" +) + +// responseCodeDetailsViaUpstream is the value Envoy sets when the upstream +// produced the response. Any other value means Envoy or a filter synthesized it, +// which is the difference between "the backend answered 503" and "the backend +// never saw the request" — a distinction the status code alone cannot make. +const responseCodeDetailsViaUpstream = "via_upstream" + +// faultClassification is the derived error view of one request, written onto the +// canonical event so every publisher reads one classification instead of each +// inventing its own. +type faultClassification struct { + EventCategory dto.EventCategory + FaultCategory dto.FaultCategory + // ErrorType is the value of the stable OpenTelemetry `error.type` attribute: + // the response flag's name for a gateway-originated fault, or the HTTP status + // code for a response the upstream itself produced (which is what the HTTP + // semantic conventions ask for). + ErrorType string + // SubCategory is set only where the response flag determines it. Where the + // flag proves the category but not the specific cause — RateLimited does not + // say which limit was hit — the category's own "OTHER" is used rather than a + // guess. + SubCategory dto.FaultSubCategory +} + +// flagFault maps one Envoy response flag onto a classification. +type flagFault struct { + name string + set func(*v3.ResponseFlags) bool + category dto.FaultCategory + subCategory dto.FaultSubCategory +} + +// flagFaults is deliberately an ordered slice, not a map: several flags are set +// together on the same request (UpstreamRequestTimeout alongside +// UpstreamRetryLimitExceeded, for instance), and Go randomizes map iteration, so +// a map would yield a different error.type for identical requests. Most specific +// first. +// +// Three flags are deliberately absent because they do not describe a failure: +// DelayInjected and FaultInjected mark deliberate fault injection, and +// ResponseFromCacheFilter marks a cache hit. Treating "any flag set" as a fault +// would count every cached response as an error. +var flagFaults = []flagFault{ + // Access control, before anything upstream is attempted. + {"unauthorized_details", func(f *v3.ResponseFlags) bool { return f.GetUnauthorizedDetails() != nil }, + dto.FaultCategoryAuth, dto.AuthenticationOther}, + {"rate_limited", (*v3.ResponseFlags).GetRateLimited, + dto.FaultCategoryThrottled, dto.ThrottlingOther}, + // The rate-limit service itself failed. That is gateway infrastructure, not + // the client being throttled, so it must not inflate throttling counts. + {"rate_limit_service_error", (*v3.ResponseFlags).GetRateLimitServiceError, + dto.FaultCategoryOther, dto.OtherMediationError}, + + // Upstream timeouts. The flag names the cause precisely, so the + // sub-category is derived rather than guessed. + {"upstream_request_timeout", (*v3.ResponseFlags).GetUpstreamRequestTimeout, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionTimeout}, + {"stream_idle_timeout", (*v3.ResponseFlags).GetStreamIdleTimeout, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionTimeout}, + {"duration_timeout", (*v3.ResponseFlags).GetDurationTimeout, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionTimeout}, + {"upstream_max_stream_duration_reached", (*v3.ResponseFlags).GetUpstreamMaxStreamDurationReached, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionTimeout}, + + // Upstream deliberately withheld: no member passed health checking, or a + // circuit breaker shed the request to protect it. + {"no_healthy_upstream", (*v3.ResponseFlags).GetNoHealthyUpstream, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionSuspended}, + {"failed_local_healthcheck", (*v3.ResponseFlags).GetFailedLocalHealthcheck, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionSuspended}, + {"upstream_overflow", (*v3.ResponseFlags).GetUpstreamOverflow, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityConnectionSuspended}, + + // Upstream unreachable or the connection broke mid-flight. + {"dns_resolution_failure", (*v3.ResponseFlags).GetDnsResolutionFailure, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_connection_failure", (*v3.ResponseFlags).GetUpstreamConnectionFailure, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_connection_termination", (*v3.ResponseFlags).GetUpstreamConnectionTermination, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_remote_reset", (*v3.ResponseFlags).GetUpstreamRemoteReset, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_protocol_error", (*v3.ResponseFlags).GetUpstreamProtocolError, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + {"upstream_retry_limit_exceeded", (*v3.ResponseFlags).GetUpstreamRetryLimitExceeded, + dto.FaultCategoryTargetConnectivity, dto.TargetConnectivityOther}, + + // Gateway configuration: the request matched no route, cluster, or filter + // config. Nothing upstream was attempted. + {"no_route_found", (*v3.ResponseFlags).GetNoRouteFound, + dto.FaultCategoryOther, dto.OtherResourceNotFound}, + {"no_cluster_found", (*v3.ResponseFlags).GetNoClusterFound, + dto.FaultCategoryOther, dto.OtherResourceNotFound}, + {"no_filter_config_found", (*v3.ResponseFlags).GetNoFilterConfigFound, + dto.FaultCategoryOther, dto.OtherMediationError}, + // The gateway shed the request to protect itself. + {"overload_manager", (*v3.ResponseFlags).GetOverloadManager, + dto.FaultCategoryOther, dto.OtherMediationError}, + + // Client-side: the caller sent something unusable or went away. Kept as + // distinct error types because downstream_connection_termination is routine + // for streaming and LLM traffic (a cancelled response) and an operator will + // want to exclude it from a fault-rate alert. + {"invalid_envoy_request_headers", (*v3.ResponseFlags).GetInvalidEnvoyRequestHeaders, + dto.FaultCategoryOther, dto.OtherUnclassified}, + {"downstream_protocol_error", (*v3.ResponseFlags).GetDownstreamProtocolError, + dto.FaultCategoryOther, dto.OtherUnclassified}, + {"downstream_connection_termination", (*v3.ResponseFlags).GetDownstreamConnectionTermination, + dto.FaultCategoryOther, dto.OtherUnclassified}, + {"downstream_remote_reset", (*v3.ResponseFlags).GetDownstreamRemoteReset, + dto.FaultCategoryOther, dto.OtherUnclassified}, + // Last: Envoy reset the stream locally, which is often accompanied by a more + // specific flag above. + {"local_reset", (*v3.ResponseFlags).GetLocalReset, + dto.FaultCategoryOther, dto.OtherUnclassified}, +} + +// classifyFault derives the error view of a request from the Envoy access log. +// +// Response flags are the primary signal rather than the status code, because the +// status code cannot separate a gateway fault from a backend one: a 503 with +// NoHealthyUpstream never reached the backend, while a 503 with no flags and +// via_upstream is the backend's own answer. Those belong in different categories +// and would be indistinguishable from a status-range table. +// +// Precedence: a matching response flag, then a response Envoy synthesized +// without setting one, then an upstream-produced 5xx. +func classifyFault(logEntry *v3.HTTPAccessLogEntry) faultClassification { + success := faultClassification{EventCategory: dto.EventCategorySuccess} + + flags := logEntry.GetCommonProperties().GetResponseFlags() + for _, candidate := range flagFaults { + if candidate.set(flags) { + return faultClassification{ + EventCategory: dto.EventCategoryFault, + FaultCategory: candidate.category, + ErrorType: candidate.name, + SubCategory: candidate.subCategory, + } + } + } + + response := logEntry.GetResponse() + if response == nil { + return success + } + status := int(response.GetResponseCode().GetValue()) + details := response.GetResponseCodeDetails() + + // No flag matched, but the response did not come from the upstream: a filter + // or Envoy produced it (a policy denial, a direct response, a redirect). The + // detail string itself is high-cardinality and stays on + // wso2.upstream.response.detail; error.type gets one stable value. + if details != "" && details != responseCodeDetailsViaUpstream { + if status < 400 { + return success // a synthesized redirect or 200 is not a fault + } + return faultClassification{ + EventCategory: dto.EventCategoryFault, + FaultCategory: dto.FaultCategoryOther, + ErrorType: "local_reply", + SubCategory: dto.OtherUnclassified, + } + } + + // The upstream answered. A 5xx is the backend's own failure — the gateway + // worked — so it is a fault in a category that does not blame connectivity. + // A 4xx is the backend's valid answer to an invalid request: error.type is + // set because the HTTP operation did fail, but no fault is recorded against + // the gateway. + switch { + case status >= 500: + return faultClassification{ + EventCategory: dto.EventCategoryFault, + FaultCategory: dto.FaultCategoryOther, + ErrorType: strconv.Itoa(status), + SubCategory: dto.OtherUnclassified, + } + case status >= 400: + success.ErrorType = strconv.Itoa(status) + return success + } + return success +} + +// isCacheHit reports whether Envoy served the response from its cache filter. +// ResponseFromCacheFilter is one of the response flags that does not describe a +// failure, which is why it is handled here and not in flagFaults. +func isCacheHit(logEntry *v3.HTTPAccessLogEntry) bool { + return logEntry.GetCommonProperties().GetResponseFlags().GetResponseFromCacheFilter() +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go new file mode 100644 index 0000000000..a4d71f4128 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package analytics + +import ( + "reflect" + "testing" + + v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" +) + +// faultEntry builds a minimal access-log entry: the flags Envoy set, plus the +// status and detail string the client saw. +func faultEntry(flags *v3.ResponseFlags, status uint32, details string) *v3.HTTPAccessLogEntry { + return &v3.HTTPAccessLogEntry{ + CommonProperties: &v3.AccessLogCommon{ResponseFlags: flags}, + Response: &v3.HTTPResponseProperties{ + ResponseCode: wrapperspb.UInt32(status), + ResponseCodeDetails: details, + }, + } +} + +// Every flag in the table must produce its own error.type and category. Driven +// off flagFaults itself, using reflection to set the one field under test, so a +// flag added to the table without a mapping cannot silently go untested. +func TestClassifyFault_EveryMappedFlag(t *testing.T) { + for _, mapped := range flagFaults { + t.Run(mapped.name, func(t *testing.T) { + flags := &v3.ResponseFlags{} + setFlagByErrorType(t, flags, mapped.name) + + got := classifyFault(faultEntry(flags, 503, "")) + assert.Equal(t, dto.EventCategoryFault, got.EventCategory) + assert.Equal(t, mapped.name, got.ErrorType) + assert.Equal(t, mapped.category, got.FaultCategory) + assert.Equal(t, mapped.subCategory, got.SubCategory) + }) + } +} + +// setFlagByErrorType sets the ResponseFlags field whose snake_case name matches +// the table entry, proving the two agree. +func setFlagByErrorType(t *testing.T, flags *v3.ResponseFlags, errorType string) { + t.Helper() + target := snakeToPascal(errorType) + value := reflect.ValueOf(flags).Elem() + field := value.FieldByName(target) + require.True(t, field.IsValid(), "no ResponseFlags field named %q for error type %q", target, errorType) + switch field.Kind() { + case reflect.Bool: + field.SetBool(true) + case reflect.Ptr: + // UnauthorizedDetails is a message, not a bool. + field.Set(reflect.New(field.Type().Elem())) + default: + t.Fatalf("unexpected kind %s for %s", field.Kind(), target) + } +} + +func snakeToPascal(s string) string { + out := []byte{} + upper := true + for i := 0; i < len(s); i++ { + if s[i] == '_' { + upper = true + continue + } + c := s[i] + if upper && c >= 'a' && c <= 'z' { + c -= 32 + } + upper = false + out = append(out, c) + } + return string(out) +} + +// The three informational flags describe deliberate behaviour, not failure. A +// cache hit counted as a fault would be a silent, permanent error-rate inflation. +func TestClassifyFault_InformationalFlagsAreNotFaults(t *testing.T) { + cases := map[string]*v3.ResponseFlags{ + "cache hit": {ResponseFromCacheFilter: true}, + "delay injected": {DelayInjected: true}, + "fault injected": {FaultInjected: true}, + } + for name, flags := range cases { + t.Run(name, func(t *testing.T) { + got := classifyFault(faultEntry(flags, 200, responseCodeDetailsViaUpstream)) + assert.Equal(t, dto.EventCategorySuccess, got.EventCategory) + assert.Empty(t, got.ErrorType) + assert.Empty(t, got.FaultCategory) + }) + } +} + +// The distinction the status code cannot make: same 503, two different causes. +func TestClassifyFault_SameStatusDifferentCause(t *testing.T) { + gateway := classifyFault(faultEntry(&v3.ResponseFlags{NoHealthyUpstream: true}, 503, "no_healthy_upstream")) + assert.Equal(t, dto.FaultCategoryTargetConnectivity, gateway.FaultCategory, + "a 503 the backend never saw is a connectivity fault") + assert.Equal(t, "no_healthy_upstream", gateway.ErrorType) + + backend := classifyFault(faultEntry(&v3.ResponseFlags{}, 503, responseCodeDetailsViaUpstream)) + assert.Equal(t, dto.FaultCategoryOther, backend.FaultCategory, + "a 503 the backend answered is not a connectivity fault") + assert.Equal(t, "503", backend.ErrorType, + "an upstream-produced error uses the status code, per the HTTP semantic conventions") +} + +func TestClassifyFault_UpstreamResponses(t *testing.T) { + cases := []struct { + name string + status uint32 + wantCat dto.EventCategory + wantType string + wantSub dto.FaultSubCategory + }{ + {"200 is a success", 200, dto.EventCategorySuccess, "", ""}, + {"301 is a success", 301, dto.EventCategorySuccess, "", ""}, + // The gateway did its job; the backend's 404 is a valid answer to an + // invalid request. error.type is still set: the HTTP operation failed. + {"404 sets error.type without a fault", 404, dto.EventCategorySuccess, "404", ""}, + {"429 sets error.type without a fault", 429, dto.EventCategorySuccess, "429", ""}, + {"500 is a backend fault", 500, dto.EventCategoryFault, "500", dto.OtherUnclassified}, + {"502 is a backend fault", 502, dto.EventCategoryFault, "502", dto.OtherUnclassified}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, responseCodeDetailsViaUpstream)) + assert.Equal(t, tc.wantCat, got.EventCategory) + assert.Equal(t, tc.wantType, got.ErrorType) + assert.Equal(t, tc.wantSub, got.SubCategory) + }) + } +} + +// A response Envoy or a filter synthesized without setting a flag — a policy +// denial via ext_proc, a direct response. The detail string is high-cardinality +// so error.type gets one stable value instead. +func TestClassifyFault_SynthesizedResponses(t *testing.T) { + cases := []struct { + details string + status uint32 + wantCat dto.EventCategory + wantType string + }{ + {"ext_authz_denied", 403, dto.EventCategoryFault, "local_reply"}, + {"direct_response", 401, dto.EventCategoryFault, "local_reply"}, + {"ext_proc_error_gRPC_error_13", 500, dto.EventCategoryFault, "local_reply"}, + // A synthesized redirect is not a failure. + {"direct_response", 302, dto.EventCategorySuccess, ""}, + } + for _, tc := range cases { + t.Run(tc.details+"_"+string(rune('0'+tc.status/100)), func(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, tc.details)) + assert.Equal(t, tc.wantCat, got.EventCategory) + assert.Equal(t, tc.wantType, got.ErrorType) + }) + } +} + +// Several flags are routinely set on one request. The table is ordered, so the +// result must be deterministic — a map would return either one at random. +func TestClassifyFault_MultipleFlagsAreDeterministic(t *testing.T) { + flags := &v3.ResponseFlags{ + UpstreamRequestTimeout: true, + UpstreamRetryLimitExceeded: true, + LocalReset: true, + } + for i := 0; i < 50; i++ { + got := classifyFault(faultEntry(flags, 504, "")) + require.Equal(t, "upstream_request_timeout", got.ErrorType, + "the most specific flag must win on every call") + require.Equal(t, dto.TargetConnectivityConnectionTimeout, got.SubCategory) + } +} + +// A rate-limit *service* failure is gateway infrastructure, not the client being +// throttled: counting it as THROTTLED would misattribute an outage to callers. +func TestClassifyFault_RateLimitServiceErrorIsNotThrottling(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{RateLimitServiceError: true}, 500, "")) + assert.Equal(t, dto.FaultCategoryOther, got.FaultCategory) + assert.Equal(t, dto.OtherMediationError, got.SubCategory) +} + +// Nothing may panic on a sparse entry: the ALS message is built by Envoy, and +// every level of it is optional. +func TestClassifyFault_NilSafety(t *testing.T) { + cases := map[string]*v3.HTTPAccessLogEntry{ + "empty entry": {}, + "no common props": {Response: &v3.HTTPResponseProperties{}}, + "no response": {CommonProperties: &v3.AccessLogCommon{}}, + "no flags": {CommonProperties: &v3.AccessLogCommon{}, Response: &v3.HTTPResponseProperties{}}, + "no response code": {CommonProperties: &v3.AccessLogCommon{ResponseFlags: &v3.ResponseFlags{}}, Response: &v3.HTTPResponseProperties{ResponseCodeDetails: responseCodeDetailsViaUpstream}}, + } + for name, entry := range cases { + t.Run(name, func(t *testing.T) { + got := classifyFault(entry) + assert.Equal(t, dto.EventCategorySuccess, got.EventCategory) + }) + } +} + +func TestIsCacheHit(t *testing.T) { + assert.True(t, isCacheHit(faultEntry(&v3.ResponseFlags{ResponseFromCacheFilter: true}, 200, ""))) + assert.False(t, isCacheHit(faultEntry(&v3.ResponseFlags{}, 200, ""))) + assert.False(t, isCacheHit(&v3.HTTPAccessLogEntry{}), "a sparse entry must not panic") +} + +// The classifier is only useful if prepareAnalyticEvent actually writes its +// output onto the event every publisher then reads. +func TestPrepareAnalyticEvent_WritesFaultClassification(t *testing.T) { + analytics := NewAnalytics(&config.Config{}) + + t.Run("gateway fault", func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{APIIDKey: "api-1"}) + logEntry.CommonProperties.ResponseFlags = &v3.ResponseFlags{UpstreamRequestTimeout: true} + logEntry.Response = &v3.HTTPResponseProperties{ResponseCode: wrapperspb.UInt32(504)} + + event := analytics.prepareAnalyticEvent(logEntry) + require.NotNil(t, event) + assert.Equal(t, dto.EventCategoryFault, event.EventCategory) + assert.Equal(t, dto.FaultCategoryTargetConnectivity, event.FaultCategory) + assert.Equal(t, "upstream_request_timeout", event.ErrorType) + require.NotNil(t, event.Error) + assert.Equal(t, dto.TargetConnectivityConnectionTimeout, event.Error.ErrorMessage) + // The client-visible status, populated before the classification runs. + assert.Equal(t, 504, event.Error.ErrorCode) + }) + + t.Run("clean upstream response leaves the fields unset", func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{APIIDKey: "api-1"}) + logEntry.CommonProperties.ResponseFlags = &v3.ResponseFlags{} + logEntry.Response = &v3.HTTPResponseProperties{ + ResponseCode: wrapperspb.UInt32(200), + ResponseCodeDetails: responseCodeDetailsViaUpstream, + } + + event := analytics.prepareAnalyticEvent(logEntry) + require.NotNil(t, event) + assert.Equal(t, dto.EventCategorySuccess, event.EventCategory) + assert.Empty(t, event.ErrorType) + assert.Nil(t, event.Error, "a successful request must not carry an error object") + }) + + // ResponseFromCacheFilter replaces what was a hardcoded false. + t.Run("cache hit reaches Target.ResponseCacheHit", func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{APIIDKey: "api-1"}) + logEntry.CommonProperties.ResponseFlags = &v3.ResponseFlags{ResponseFromCacheFilter: true} + logEntry.Response = &v3.HTTPResponseProperties{ResponseCode: wrapperspb.UInt32(200)} + + event := analytics.prepareAnalyticEvent(logEntry) + require.NotNil(t, event) + require.NotNil(t, event.Target) + assert.True(t, event.Target.ResponseCacheHit) + assert.Equal(t, dto.EventCategorySuccess, event.EventCategory, + "a cache hit is not a fault") + }) +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go index 74f99e102d..bec7fe6686 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go @@ -324,6 +324,26 @@ func (m *Moesif) Publish(event *dto.Event) { metadataMap["responseMediationLatency"] = event.Latencies.ResponseMediationLatency } + // Fault classification, derived in analytics.classifyFault from the Envoy + // response flags. eventCategory is sent unconditionally so a consumer can + // tell "this request succeeded" from "this event predates the field"; the + // rest are omitted when the request was not a fault. + metadataMap["eventCategory"] = string(event.EventCategory) + if event.FaultCategory != "" { + metadataMap["faultCategory"] = string(event.FaultCategory) + } + if event.ErrorType != "" { + metadataMap["errorType"] = event.ErrorType + } + if event.Error != nil { + if event.Error.ErrorCode != 0 { + metadataMap["errorCode"] = event.Error.ErrorCode + } + if event.Error.ErrorMessage != "" { + metadataMap["faultSubCategory"] = string(event.Error.ErrorMessage) + } + } + // commonName if commonName, ok := event.Properties["commonName"]; ok && commonName != nil { metadataMap["commonName"] = commonName diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go index a447fa990b..68e7bc3441 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go @@ -360,6 +360,64 @@ func TestPublish_McpAPIType(t *testing.T) { assert.Equal(t, "search", mcpAnalytics["toolName"]) } +// The fault taxonomy reaches Moesif, not only the OTLP publisher: both read the +// same classification off the canonical event. +func TestPublish_WithFaultClassification(t *testing.T) { + moesif := createTestMoesifWithoutAPI() + + event := createBaseEvent() + event.EventCategory = dto.EventCategoryFault + event.FaultCategory = dto.FaultCategoryTargetConnectivity + event.ErrorType = "upstream_request_timeout" + event.Error = &dto.Error{ErrorCode: 504, ErrorMessage: dto.TargetConnectivityConnectionTimeout} + + moesif.Publish(event) + + assert.Len(t, moesif.events, 1) + metadata := getMetadata(moesif.events[0]) + assert.Equal(t, "FAULT", metadata["eventCategory"]) + assert.Equal(t, "TARGET_CONNECTIVITY", metadata["faultCategory"]) + assert.Equal(t, "upstream_request_timeout", metadata["errorType"]) + assert.Equal(t, 504, metadata["errorCode"]) + assert.Equal(t, "CONNECTION_TIMEOUT", metadata["faultSubCategory"]) +} + +// A successful request carries the category and nothing else, so a fault filter +// in Moesif keys on presence rather than having to exclude empty strings. +func TestPublish_SuccessOmitsFaultFields(t *testing.T) { + moesif := createTestMoesifWithoutAPI() + + event := createBaseEvent() + event.EventCategory = dto.EventCategorySuccess + + moesif.Publish(event) + + assert.Len(t, moesif.events, 1) + metadata := getMetadata(moesif.events[0]) + assert.Equal(t, "SUCCESS", metadata["eventCategory"]) + for _, key := range []string{"faultCategory", "errorType", "errorCode", "faultSubCategory"} { + assert.NotContains(t, metadata, key, "%s must be absent on a successful event", key) + } +} + +// An upstream 4xx sets errorType without recording a gateway fault: the HTTP +// operation failed, but the gateway did its job. +func TestPublish_UpstreamErrorTypeWithoutFault(t *testing.T) { + moesif := createTestMoesifWithoutAPI() + + event := createBaseEvent() + event.EventCategory = dto.EventCategorySuccess + event.ErrorType = "404" + + moesif.Publish(event) + + assert.Len(t, moesif.events, 1) + metadata := getMetadata(moesif.events[0]) + assert.Equal(t, "SUCCESS", metadata["eventCategory"]) + assert.Equal(t, "404", metadata["errorType"]) + assert.NotContains(t, metadata, "faultCategory") +} + func TestPublish_WithPayloads(t *testing.T) { moesif := createTestMoesifWithoutAPI() diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 25798dbc72..6d8c6b87cc 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -757,8 +757,11 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { attrs.b(ns("cache.hit"), event.Target.ResponseCacheHit) } - // Faults. error.type is the one stable error attribute. + // Faults. error.type is the one stable error attribute; the categories are + // ours, derived in analytics.classifyFault from the Envoy response flags. attrs.str("error.type", event.ErrorType) + attrs.str(ns("event.category"), string(event.EventCategory)) + attrs.str(ns("error.category"), string(event.FaultCategory)) if event.Error != nil { attrs.i64(ns("error.code"), int64(event.Error.ErrorCode)) attrs.str(ns("error.sub_category"), string(event.Error.ErrorMessage)) @@ -843,12 +846,14 @@ func (o *OTel) appendMCPAttributes(event *dto.Event, attrs *otelAttrs) { attrs.anyStr("mcp.session.id", mcp["sessionId"]) attrs.anyStr("jsonrpc.request.id", mcp["jsonRpcId"]) + // Tools and prompts are named (params.name); a resource is addressed by URI + // (params.uri), which the analytics policy extracts into its own field. capabilityName, _ := mcp["capabilityName"].(string) switch capability, _ := mcp["capability"].(string); capability { case "TOOL": attrs.str("gen_ai.tool.name", capabilityName) case "RESOURCE": - attrs.str("mcp.resource.uri", capabilityName) + attrs.anyStr("mcp.resource.uri", mcp["resourceUri"]) case "PROMPT": attrs.str("gen_ai.prompt.name", capabilityName) } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index a27b651e80..56c8e0f431 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -249,6 +249,46 @@ func TestBuildRecordFaults(t *testing.T) { } } +// The categories analytics.classifyFault derives must reach the record. +func TestBuildRecordFaultCategories(t *testing.T) { + event := restEvent() + event.EventCategory = dto.EventCategoryFault + event.FaultCategory = dto.FaultCategoryTargetConnectivity + event.ErrorType = "upstream_request_timeout" + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, expected := range map[string]interface{}{ + "wso2.event.category": "FAULT", + "wso2.error.category": "TARGET_CONNECTIVITY", + "error.type": "upstream_request_timeout", + } { + if got[key] != expected { + t.Errorf("%s = %v, want %v", key, got[key], expected) + } + } +} + +// A successful request must carry the SUCCESS category and no error attributes — +// an absent wso2.error.category is what lets a consumer filter faults. +func TestBuildRecordSuccessOmitsErrorAttributes(t *testing.T) { + event := restEvent() + event.EventCategory = dto.EventCategorySuccess + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["wso2.event.category"] != "SUCCESS" { + t.Errorf("wso2.event.category = %v, want SUCCESS", got["wso2.event.category"]) + } + for _, key := range []string{"wso2.error.category", "error.type", "wso2.error.code", "wso2.error.sub_category"} { + if _, present := got[key]; present { + t.Errorf("%s is present on a successful record", key) + } + } +} + func TestBuildRecordGenAI(t *testing.T) { event := restEvent() event.API.APIType = "LlmProxy" @@ -382,21 +422,59 @@ func TestBuildRecordMCPKeepsGatewayErrorType(t *testing.T) { } } +// Each capability's target goes on its own attribute, read from its own source +// field: tools and prompts are named at params.name, a resource is addressed by +// URI at params.uri. func TestMCPCapabilityRouting(t *testing.T) { - for capability, wantKey := range map[string]string{ - "RESOURCE": "mcp.resource.uri", - "PROMPT": "gen_ai.prompt.name", - "TOOL": "gen_ai.tool.name", - } { - event := restEvent() - event.Properties["mcpAnalytics"] = map[string]interface{}{ - "capability": capability, - "capabilityName": "target-1", - } - o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} - got := attrMap(t, o.buildRecord(event)) - if got[wantKey] != "target-1" { - t.Errorf("capability %s: %s = %v, want target-1", capability, wantKey, got[wantKey]) + cases := []struct { + capability string + mcp map[string]interface{} + wantKey string + wantValue string + }{ + {"TOOL", map[string]interface{}{"capabilityName": "search_docs"}, + "gen_ai.tool.name", "search_docs"}, + {"PROMPT", map[string]interface{}{"capabilityName": "summarize"}, + "gen_ai.prompt.name", "summarize"}, + {"RESOURCE", map[string]interface{}{"resourceUri": "file:///docs/readme.md"}, + "mcp.resource.uri", "file:///docs/readme.md"}, + } + for _, tc := range cases { + t.Run(tc.capability, func(t *testing.T) { + event := restEvent() + tc.mcp["capability"] = tc.capability + event.Properties["mcpAnalytics"] = tc.mcp + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if got[tc.wantKey] != tc.wantValue { + t.Errorf("%s = %v, want %v", tc.wantKey, got[tc.wantKey], tc.wantValue) + } + }) + } +} + +// mcp.resource.uri must come from resourceUri only. Reading capabilityName here +// is the bug this replaced: params.name does not exist on a resources/read +// request, so the attribute was always absent. +func TestMCPResourceURIIgnoresCapabilityName(t *testing.T) { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "capability": "RESOURCE", + "capabilityName": "should-not-be-used", + "resourceUri": "file:///docs/readme.md", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["mcp.resource.uri"] != "file:///docs/readme.md" { + t.Errorf("mcp.resource.uri = %v, want the resourceUri value", got["mcp.resource.uri"]) + } + // A resource has no name, so neither name attribute may appear. + for _, key := range []string{"gen_ai.tool.name", "gen_ai.prompt.name"} { + if _, present := got[key]; present { + t.Errorf("%s is present on a resource read", key) } } } diff --git a/gateway/system-policies/analytics/analytics.go b/gateway/system-policies/analytics/analytics.go index 4ce3ab9e7c..2c606f7219 100644 --- a/gateway/system-policies/analytics/analytics.go +++ b/gateway/system-policies/analytics/analytics.go @@ -91,15 +91,27 @@ var ( JsonRpcErrorCodeJsonPath = "$.error.code" ) +// MCP capability kinds, derived from the JSON-RPC method prefix. Emitted on the +// analytics event, so the policy-engine publishers match against these values. +const ( + McpCapabilityTool = "TOOL" + McpCapabilityResource = "RESOURCE" + McpCapabilityPrompt = "PROMPT" +) + // AnalyticsPolicy implements the default analytics data collection process. type AnalyticsPolicy struct{} type McpRequestAnalyticsProperties struct { - JsonRpcMethod string `json:"jsonRpcMethod,omitempty"` - JsonRpcID string `json:"jsonRpcId,omitempty"` - Capability string `json:"capability,omitempty"` - CapabilityName string `json:"capabilityName,omitempty"` - ClientInfo *McpClientInfo `json:"clientInfo,omitempty"` + JsonRpcMethod string `json:"jsonRpcMethod,omitempty"` + JsonRpcID string `json:"jsonRpcId,omitempty"` + Capability string `json:"capability,omitempty"` + // CapabilityName is the target's name, from params.name. Tools and prompts + // are named; resources are not + CapabilityName string `json:"capabilityName,omitempty"` + // ResourceUri is the target of a resources/* method, from params.uri + ResourceUri string `json:"resourceUri,omitempty"` + ClientInfo *McpClientInfo `json:"clientInfo,omitempty"` } type McpClientInfo struct { @@ -412,8 +424,14 @@ func (a *AnalyticsPolicy) OnRequestBody(_ context.Context, ctx *policy.RequestCo props.JsonRpcID = strconv.FormatInt(int64(id), 10) } } - props.CapabilityName = extractString(McpCapabilityNameJsonPath) props.Capability = deriveMCPCapability(props.JsonRpcMethod) + // resources/* addresses its target by URI at params.uri; tools/* and + // prompts/* name theirs at params.name. + if props.Capability == McpCapabilityResource { + props.ResourceUri = extractString(McpResourceUriJsonPath) + } else { + props.CapabilityName = extractString(McpCapabilityNameJsonPath) + } clientInfo := McpClientInfo{ RequestedProtocolVersion: extractStringFromJsonpath(mcpPayload, ProtocolVersionJsonPath), @@ -1217,11 +1235,11 @@ func extractBoolFromJsonpath(payload map[string]interface{}, path string) (bool, func deriveMCPCapability(method string) string { switch { case strings.HasPrefix(method, "tools/"): - return "TOOL" + return McpCapabilityTool case strings.HasPrefix(method, "resources/"): - return "RESOURCE" + return McpCapabilityResource case strings.HasPrefix(method, "prompts/"): - return "PROMPT" + return McpCapabilityPrompt default: return "" } diff --git a/gateway/system-policies/analytics/analytics_test.go b/gateway/system-policies/analytics/analytics_test.go index 1b4eaa0895..2f7e492460 100644 --- a/gateway/system-policies/analytics/analytics_test.go +++ b/gateway/system-policies/analytics/analytics_test.go @@ -416,3 +416,87 @@ func TestExtractMCPResponseAnalyticsProps_IsError(t *testing.T) { }) } } + +// A resources/* method addresses its target by URI at params.uri, while tools/* +// and prompts/* name theirs at params.name. Extracting only params.name left +// mcp.resource.uri permanently empty for every resource read. +func TestOnRequestBody_MCPCapabilityTarget(t *testing.T) { + cases := []struct { + name string + body string + wantName string + wantURI string + }{ + { + name: "tools/call carries a name", + body: `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search_docs"}}`, + wantName: "search_docs", + }, + { + name: "prompts/get carries a name", + body: `{"jsonrpc":"2.0","id":2,"method":"prompts/get","params":{"name":"summarize"}}`, + wantName: "summarize", + }, + { + name: "resources/read carries a uri", + body: `{"jsonrpc":"2.0","id":3,"method":"resources/read","params":{"uri":"file:///docs/readme.md"}}`, + wantURI: "file:///docs/readme.md", + }, + { + // resources/list takes no target at all; neither field may be invented. + name: "resources/list carries neither", + body: `{"jsonrpc":"2.0","id":4,"method":"resources/list","params":{}}`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + props := mcpRequestProps(t, tc.body) + if props.CapabilityName != tc.wantName { + t.Errorf("capabilityName = %q, want %q", props.CapabilityName, tc.wantName) + } + if props.ResourceUri != tc.wantURI { + t.Errorf("resourceUri = %q, want %q", props.ResourceUri, tc.wantURI) + } + }) + } +} + +// A resources/read request that also happens to carry params.name must not have +// it mistaken for the resource's identity. +func TestOnRequestBody_MCPResourceIgnoresName(t *testing.T) { + props := mcpRequestProps(t, + `{"jsonrpc":"2.0","id":5,"method":"resources/read",`+ + `"params":{"uri":"file:///a.md","name":"not-the-target"}}`) + + if props.ResourceUri != "file:///a.md" { + t.Errorf("resourceUri = %q, want file:///a.md", props.ResourceUri) + } + if props.CapabilityName != "" { + t.Errorf("capabilityName = %q, want empty for a resource", props.CapabilityName) + } +} + +// mcpRequestProps runs OnRequestBody over an MCP request body and returns the +// properties it emitted onto analytics metadata. +func mcpRequestProps(t *testing.T, body string) McpRequestAnalyticsProperties { + t.Helper() + action := (&AnalyticsPolicy{}).OnRequestBody(context.Background(), &policy.RequestContext{ + SharedContext: &policy.SharedContext{APIKind: policy.APIKindMCP}, + Body: &policy.Body{Content: []byte(body)}, + }, nil) + + mods, ok := action.(policy.UpstreamRequestModifications) + if !ok { + t.Fatalf("expected UpstreamRequestModifications, got %T", action) + } + raw, ok := mods.AnalyticsMetadata["mcp_request_properties"].(string) + if !ok { + t.Fatalf("mcp_request_properties absent or not a string: %#v", mods.AnalyticsMetadata) + } + var props McpRequestAnalyticsProperties + if err := json.Unmarshal([]byte(raw), &props); err != nil { + t.Fatalf("unmarshal mcp_request_properties: %v", err) + } + return props +} From 177abf323e76d95102b5789a93e3b5a9838afce4 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Tue, 8 Sep 2026 11:30:11 +0530 Subject: [PATCH 06/20] Fix req and res header mapping issue --- gateway/configs/config-template.toml | 7 +- .../internal/analytics/publishers/otel.go | 87 ++++++++- .../analytics/publishers/otel_test.go | 175 ++++++++++++++++++ 3 files changed, 262 insertions(+), 7 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 74d6ca5bdf..43a2b397bc 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -815,9 +815,10 @@ timer_wakeup_seconds = 3 # Full OTLP/HTTP logs URL, including the /v1/logs path. Point it at a collector # you run, or directly at a vendor's OTLP intake. endpoint = "http://otel-collector:4318/v1/logs" -# service.name / service.version on the OTLP resource. -service_name = "policy-engine" -service_version = "" +# Identifies the service +service_name = "gateway-runtime" +# Identifies the deployed build for rollout correlation. +service_version = '{{ env "VERSION" "" }}' # Record count that triggers an export before flush_interval elapses. batch_size = 100 # How long a record waits when traffic is too slow to fill a batch. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 6d8c6b87cc..79bbcad67f 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -33,6 +33,7 @@ import ( "net/http" "net/url" "os" + "sort" "strconv" "strings" "sync" @@ -70,6 +71,10 @@ const ( // 2xx body carries partialSuccess and a failure body carries an error // message; neither may be allowed to grow the heap. otelMaxResponseBytes int64 = 4 << 10 + // otelMaxHeaderAttributes caps how many header attributes one record may + // carry, per direction. Header names become attribute keys, so an over-broad + // allowlist grows the record's schema, not just its size. + otelMaxHeaderAttributes = 32 ) // ns qualifies an attribute name with the WSO2 namespace. @@ -771,6 +776,9 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { attrs.anyStr(ns("request.body"), event.Properties[dto.PropKeyRequestPayload]) attrs.anyStr(ns("response.body"), event.Properties[dto.PropKeyResponsePayload]) + appendHeaderAttributes(attrs, "http.request.header.", event.Properties[dto.PropKeyRequestHeaders]) + appendHeaderAttributes(attrs, "http.response.header.", event.Properties[dto.PropKeyResponseHeaders]) + o.appendAIAttributes(event, attrs, route) o.appendMCPAttributes(event, attrs) @@ -796,6 +804,51 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { } } +// appendHeaderAttributes emits one attribute per header under the given prefix, +// per the HTTP conventions: the header name forms the key, and the value is +// always a string array because a header can repeat. +// +// The headers are already filtered by the operator’s allowlist policy. When +// the policy is absent, nothing is emitted. +// +// otelMaxHeaderAttributes prevents overly broad header allowlists from +// exceeding OTel’s attribute limit and causing silent truncation +func appendHeaderAttributes(attrs *otelAttrs, prefix string, raw interface{}) { + serialized, ok := raw.(string) + if !ok || serialized == "" { + return + } + var headers map[string]string + if err := json.Unmarshal([]byte(serialized), &headers); err != nil { + slog.Debug("OTel publisher could not parse analytics headers", "error", err, "prefix", prefix) + return + } + + // Sorted so a truncated record keeps the same headers across requests; map + // order would drop a different arbitrary subset every time. + names := make([]string, 0, len(headers)) + for name := range headers { + names = append(names, name) + } + sort.Strings(names) + + emitted := 0 + for _, name := range names { + if emitted == otelMaxHeaderAttributes { + slog.Warn("OTel publisher truncated header attributes; narrow the analytics-header-filter allowlist", + "prefix", prefix, "emitted", emitted, "available", len(names)) + return + } + // The analytics policy joins a repeated header into one comma-separated + // string before it reaches the event, so the array carries that single + // value verbatim. + if value := headers[name]; value != "" { + attrs.strs(prefix+strings.ToLower(name), []string{value}) + emitted++ + } + } +} + // appendAIAttributes maps the AI metadata the pipeline stashes in // Event.Properties onto GenAI conventions. Cost has no GenAI equivalent. func (o *OTel) appendAIAttributes(event *dto.Event, attrs *otelAttrs, route string) { @@ -978,10 +1031,18 @@ type otelKeyValue struct { // otelAnyValue is the OTLP AnyValue union; exactly one field is set. type otelAnyValue struct { - StringValue *string `json:"stringValue,omitempty"` - IntValue *string `json:"intValue,omitempty"` - DoubleValue *float64 `json:"doubleValue,omitempty"` - BoolValue *bool `json:"boolValue,omitempty"` + StringValue *string `json:"stringValue,omitempty"` + IntValue *string `json:"intValue,omitempty"` + DoubleValue *float64 `json:"doubleValue,omitempty"` + BoolValue *bool `json:"boolValue,omitempty"` + ArrayValue *otelArrayValue `json:"arrayValue,omitempty"` +} + +// otelArrayValue is the OTLP ArrayValue: a list of AnyValue. Required for the +// header conventions, whose values are always string arrays because a header can +// legitimately repeat. +type otelArrayValue struct { + Values []otelAnyValue `json:"values"` } // otelAttrs accumulates attributes, skipping empty ones so a record carries only @@ -1003,6 +1064,24 @@ func (a *otelAttrs) str(key, value string) *otelAttrs { return a } +// strs sets a string-array attribute, skipping the attribute entirely when there +// is nothing to put in it. +func (a *otelAttrs) strs(key string, values []string) *otelAttrs { + if len(values) == 0 { + return a + } + items := make([]otelAnyValue, 0, len(values)) + for _, value := range values { + v := value + items = append(items, otelAnyValue{StringValue: &v}) + } + a.kvs = append(a.kvs, otelKeyValue{ + Key: key, + Value: otelAnyValue{ArrayValue: &otelArrayValue{Values: items}}, + }) + return a +} + // strIfEmpty sets key only when it is not already present. func (a *otelAttrs) strIfEmpty(key, value string) *otelAttrs { for _, kv := range a.kvs { diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 56c8e0f431..04f7dddb72 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -28,6 +28,7 @@ import ( "crypto/x509/pkix" "encoding/json" "encoding/pem" + "fmt" "io" "math/big" "net/http" @@ -117,6 +118,17 @@ func attrMap(t *testing.T, record *otelLogRecord) map[string]interface{} { out[kv.Key] = *kv.Value.DoubleValue case kv.Value.BoolValue != nil: out[kv.Key] = *kv.Value.BoolValue + case kv.Value.ArrayValue != nil: + // Header attributes are string arrays; flatten to []string so a test + // can assert on them directly. + items := make([]string, 0, len(kv.Value.ArrayValue.Values)) + for _, item := range kv.Value.ArrayValue.Values { + if item.StringValue == nil { + t.Fatalf("attribute %q has a non-string array element", kv.Key) + } + items = append(items, *item.StringValue) + } + out[kv.Key] = items default: t.Fatalf("attribute %q has no value set", kv.Key) } @@ -1418,3 +1430,166 @@ func TestMetricsPreInitializedAtZero(t *testing.T) { } } } + +// --- header attributes ----------------------------------------------------- + +// headerEvent returns the REST fixture with the serialized header properties the +// analytics-header-filter policy produces. +func headerEvent(t *testing.T, request, response map[string]string) *dto.Event { + t.Helper() + event := restEvent() + for property, headers := range map[string]map[string]string{ + dto.PropKeyRequestHeaders: request, + dto.PropKeyResponseHeaders: response, + } { + if headers == nil { + continue + } + serialized, err := json.Marshal(headers) + if err != nil { + t.Fatalf("marshal headers: %v", err) + } + event.Properties[property] = string(serialized) + } + return event +} + +// One attribute per header, name lowercased into the key, value a string array — +// the shape the HTTP conventions require. +func TestBuildRecordHeaderAttributes(t *testing.T) { + event := headerEvent(t, + map[string]string{"Content-Type": "application/json", "X-Tenant": "acme"}, + map[string]string{"Cache-Control": "no-store"}) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, want := range map[string][]string{ + "http.request.header.content-type": {"application/json"}, + "http.request.header.x-tenant": {"acme"}, + "http.response.header.cache-control": {"no-store"}, + } { + values, ok := got[key].([]string) + if !ok { + t.Errorf("%s = %#v, want a string array", key, got[key]) + continue + } + if len(values) != len(want) || values[0] != want[0] { + t.Errorf("%s = %v, want %v", key, values, want) + } + } +} + +// No header-filter policy attached means no header properties on the event, and +// therefore no header attributes — never an empty or partial set. +func TestBuildRecordNoHeaderAttributesWhenAbsent(t *testing.T) { + cases := map[string]*dto.Event{ + "property absent": restEvent(), + "empty string": headerEventRaw(""), + "empty object": headerEventRaw("{}"), + "unparseable json": headerEventRaw("not json"), + "wrong type": headerEventWrongType(), + } + for name, event := range cases { + t.Run(name, func(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + for key := range got { + if strings.HasPrefix(key, "http.request.header.") || + strings.HasPrefix(key, "http.response.header.") { + t.Errorf("unexpected header attribute %q", key) + } + } + }) + } +} + +func headerEventRaw(serialized string) *dto.Event { + event := restEvent() + event.Properties[dto.PropKeyRequestHeaders] = serialized + return event +} + +func headerEventWrongType() *dto.Event { + event := restEvent() + event.Properties[dto.PropKeyRequestHeaders] = map[string]string{"not": "a string"} + return event +} + +// An over-broad allowlist must not grow a record's schema without bound: header +// names become attribute keys, and SDKs cap a record at 128 attributes. +func TestBuildRecordHeaderAttributesAreCapped(t *testing.T) { + headers := map[string]string{} + for i := 0; i < otelMaxHeaderAttributes*2; i++ { + headers[fmt.Sprintf("x-header-%03d", i)] = "value" + } + event := headerEvent(t, headers, nil) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + emitted := []string{} + for key := range got { + if strings.HasPrefix(key, "http.request.header.") { + emitted = append(emitted, key) + } + } + if len(emitted) != otelMaxHeaderAttributes { + t.Errorf("emitted %d header attributes, want the cap of %d", len(emitted), otelMaxHeaderAttributes) + } + // Sorted selection, so a truncated record keeps the same headers every time + // rather than an arbitrary subset that changes per request. + sort.Strings(emitted) + if emitted[0] != "http.request.header.x-header-000" { + t.Errorf("first emitted = %s, want the lowest-sorting name", emitted[0]) + } +} + +// The cap is per direction, so a wide request allowlist cannot starve the +// response headers. +func TestBuildRecordHeaderCapIsPerDirection(t *testing.T) { + request := map[string]string{} + for i := 0; i < otelMaxHeaderAttributes*2; i++ { + request[fmt.Sprintf("x-req-%03d", i)] = "value" + } + event := headerEvent(t, request, map[string]string{"Cache-Control": "no-store"}) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if _, present := got["http.response.header.cache-control"]; !present { + t.Error("response header dropped because request headers hit the cap") + } +} + +// A header with no value is skipped rather than emitted as an empty array, and +// must not consume cap budget. +func TestBuildRecordHeaderAttributesSkipEmptyValues(t *testing.T) { + event := headerEvent(t, map[string]string{"X-Present": "yes", "X-Empty": ""}, nil) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if _, present := got["http.request.header.x-empty"]; present { + t.Error("an empty header value produced an attribute") + } + if _, present := got["http.request.header.x-present"]; !present { + t.Error("x-present is missing") + } +} + +// The array must serialize as OTLP's ArrayValue shape, since that is the wire +// contract a collector parses. +func TestHeaderAttributeWireShape(t *testing.T) { + event := headerEvent(t, map[string]string{"Accept-Encoding": "gzip, br"}, nil) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + encoded, err := json.Marshal(o.buildRecord(event)) + if err != nil { + t.Fatalf("marshal record: %v", err) + } + want := `{"key":"http.request.header.accept-encoding","value":{"arrayValue":{"values":[{"stringValue":"gzip, br"}]}}}` + if !strings.Contains(string(encoded), want) { + t.Errorf("record does not contain the expected ArrayValue attribute.\nwant substring: %s\ngot: %s", want, encoded) + } +} From d4e6e09b9df7c2a8ef7ec4b4305d2427b5f93692 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Tue, 8 Sep 2026 12:26:05 +0530 Subject: [PATCH 07/20] Flatten mcp nested attributes for otel publisher --- .../internal/analytics/publishers/otel.go | 148 +++++++++++++-- .../analytics/publishers/otel_test.go | 171 ++++++++++++++++-- 2 files changed, 292 insertions(+), 27 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 79bbcad67f..e4561520c9 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -808,10 +808,10 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { // per the HTTP conventions: the header name forms the key, and the value is // always a string array because a header can repeat. // -// The headers are already filtered by the operator’s allowlist policy. When -// the policy is absent, nothing is emitted. +// The headers are already filtered by the operator’s allowlist policy. When +// the policy is absent, nothing is emitted. // -// otelMaxHeaderAttributes prevents overly broad header allowlists from +// otelMaxHeaderAttributes prevents overly broad header allowlists from // exceeding OTel’s attribute limit and causing silent truncation func appendHeaderAttributes(attrs *otelAttrs, prefix string, raw interface{}) { serialized, ok := raw.(string) @@ -888,31 +888,47 @@ func (o *OTel) appendAIAttributes(event *dto.Event, attrs *otelAttrs, route stri } // appendMCPAttributes flattens Properties["mcpAnalytics"] onto MCP conventions. -// The capability determines which attribute the capability name belongs on. +// +// Known keys get a curated attribute name, almost always a standard one: +// jsonRpcMethod becomes mcp.method.name, not a mechanical transliteration. +// Anything NOT named below is then swept up under wso2.mcp.* func (o *OTel) appendMCPAttributes(event *dto.Event, attrs *otelAttrs) { mcp, ok := event.Properties["mcpAnalytics"].(map[string]interface{}) if !ok { return } - attrs.anyStr("mcp.method.name", mcp["jsonRpcMethod"]) - attrs.anyStr("mcp.session.id", mcp["sessionId"]) - attrs.anyStr("jsonrpc.request.id", mcp["jsonRpcId"]) + // Each source map has its own claimed-key set and its own take: reading a key + // marks it as having a curated attribute name, which excludes it from the + // sweep at the end — including a key deliberately not emitted. + mapped := map[string]bool{} + take := func(key string) interface{} { + mapped[key] = true + return mcp[key] + } + + attrs.anyStr("mcp.method.name", take("jsonRpcMethod")) + attrs.anyStr("mcp.session.id", take("sessionId")) + attrs.anyStr("jsonrpc.request.id", take("jsonRpcId")) // Tools and prompts are named (params.name); a resource is addressed by URI // (params.uri), which the analytics policy extracts into its own field. - capabilityName, _ := mcp["capabilityName"].(string) - switch capability, _ := mcp["capability"].(string); capability { + capabilityName, _ := take("capabilityName").(string) + resourceURI := take("resourceUri") + // capability itself is not emitted: which attribute below is populated says + // it, and it is derivable from mcp.method.name's prefix. Taking it still + // claims it, so the sweep does not put it back. + switch capability, _ := take("capability").(string); capability { case "TOOL": attrs.str("gen_ai.tool.name", capabilityName) case "RESOURCE": - attrs.anyStr("mcp.resource.uri", mcp["resourceUri"]) + attrs.anyStr("mcp.resource.uri", resourceURI) case "PROMPT": attrs.str("gen_ai.prompt.name", capabilityName) } // A JSON-RPC error code is a string in rpc.response.status_code. - switch code := mcp["errorCode"].(type) { + switch code := take("errorCode").(type) { case int: attrs.str("rpc.response.status_code", strconv.Itoa(code)) case float64: @@ -920,14 +936,91 @@ func (o *OTel) appendMCPAttributes(event *dto.Event, attrs *otelAttrs) { case string: attrs.str("rpc.response.status_code", code) } - if isError, ok := mcp["isError"].(bool); ok && isError { + if isError, ok := take("isError").(bool); ok && isError { attrs.strIfEmpty("error.type", "mcp_error") } - attrs.anyStr("mcp.protocol.version", mcp["protocolVersion"]) - attrs.anyStr(ns("mcp.client.requested_protocol_version"), mcp["requestedProtocolVersion"]) - attrs.anyStr(ns("mcp.client.name"), mcp["name"]) - attrs.anyStr(ns("mcp.client.version"), mcp["version"]) + // clientInfo and serverInfo arrive as sub-objects, and both carry "name" and + // "version" — a flat lookup could not tell a client's name from a server's. + client := otelNestedMap(take("clientInfo")) + clientMapped := map[string]bool{} + clientTake := func(key string) interface{} { + clientMapped[key] = true + return client[key] + } + attrs.anyStr(ns("mcp.client.name"), clientTake("name")) + attrs.anyStr(ns("mcp.client.version"), clientTake("version")) + attrs.anyStr(ns("mcp.client.requested_protocol_version"), clientTake("requestedProtocolVersion")) + + server := otelNestedMap(take("serverInfo")) + serverMapped := map[string]bool{} + serverTake := func(key string) interface{} { + serverMapped[key] = true + return server[key] + } + // The negotiated version, as opposed to the client's requested one above. + attrs.anyStr("mcp.protocol.version", serverTake("protocolVersion")) + attrs.anyStr(ns("mcp.server.name"), serverTake("name")) + attrs.anyStr(ns("mcp.server.version"), serverTake("version")) + + // Sweep every key with no curated name, so a key the analytics policy gains + // later appears under a plainly-custom name rather than being silently + // dropped until someone notices. + otelSweepUnclaimed(attrs, ns("mcp."), mcp, mapped) + otelSweepUnclaimed(attrs, ns("mcp.client."), client, clientMapped) + otelSweepUnclaimed(attrs, ns("mcp.server."), server, serverMapped) +} + +// otelNestedMap reads a sub-object out of an analytics property, returning nil +// when the value is absent or not an object. Reading a key from the nil result is +// safe, so no call site needs an absence check. +func otelNestedMap(value interface{}) map[string]interface{} { + nested, _ := value.(map[string]interface{}) + return nested +} + +// otelSweepUnclaimed emits every unclaimed key under prefix + the key in +// snake_case. Sorted so two identical records produce the same attribute order; +// a sub-object recurses with an extended prefix and nothing claimed. +func otelSweepUnclaimed(attrs *otelAttrs, prefix string, source map[string]interface{}, claimed map[string]bool) { + if len(source) == 0 { + return + } + keys := make([]string, 0, len(source)) + for key := range source { + if !claimed[key] { + keys = append(keys, key) + } + } + sort.Strings(keys) + + for _, key := range keys { + name := prefix + otelSnakeCase(key) + if nested, ok := source[key].(map[string]interface{}); ok { + otelSweepUnclaimed(attrs, name+".", nested, nil) + continue + } + attrs.anyScalar(name, source[key]) + } +} + +// otelSnakeCase converts a camelCase analytics key into the snake_case that +// OpenTelemetry attribute names use. +func otelSnakeCase(s string) string { + var out strings.Builder + out.Grow(len(s) + 4) + for i := 0; i < len(s); i++ { + c := s[i] + if c >= 'A' && c <= 'Z' { + if i > 0 && !(s[i-1] >= 'A' && s[i-1] <= 'Z') { + out.WriteByte('_') + } + out.WriteByte(c - 'A' + 'a') + continue + } + out.WriteByte(c) + } + return out.String() } // otelGenAIProviderName maps a WSO2 LLM provider template name onto the @@ -1158,6 +1251,29 @@ func (a *otelAttrs) anyBool(key string, value interface{}) *otelAttrs { return a } +// anyScalar emits a value of unknown concrete type, choosing the OTLP value kind +// from it. Used by the unmapped-key sweep, where the type is whatever the +// analytics policy put in the map. +func (a *otelAttrs) anyScalar(key string, value interface{}) *otelAttrs { + switch v := value.(type) { + case string: + return a.str(key, v) + case bool: + return a.b(key, v) + case float64: + // JSON has one number type, so an integral value must not be reported as + // a double: a consumer summing "3.0" and 3 does not always get the same + // answer, and backends type the column from the first value they see. + if v == float64(int64(v)) { + return a.i64(key, int64(v)) + } + return a.f64(key, v) + case int, int32, int64, uint32, uint64: + return a.anyInt(key, v) + } + return a +} + func (a *otelAttrs) list() []otelKeyValue { return a.kvs } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 04f7dddb72..e68c7bb811 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -377,18 +377,27 @@ func TestGenAIOperationName(t *testing.T) { func TestBuildRecordMCP(t *testing.T) { event := restEvent() event.API.APIType = "Mcp" + // Nested exactly as the analytics policy serializes it: clientInfo and + // serverInfo are sub-objects, and both carry "name" and "version". A flat + // fixture here is what let the nested-lookup bug pass for so long. event.Properties["mcpAnalytics"] = map[string]interface{}{ - "jsonRpcMethod": "tools/call", - "jsonRpcId": "7", - "sessionId": "sess-1", - "capability": "TOOL", - "capabilityName": "search_docs", - "errorCode": -32602, - "isError": true, - "protocolVersion": "2025-06-18", - "requestedProtocolVersion": "2025-03-26", - "name": "claude-desktop", - "version": "1.2.0", + "jsonRpcMethod": "tools/call", + "jsonRpcId": "7", + "sessionId": "sess-1", + "capability": "TOOL", + "capabilityName": "search_docs", + "errorCode": -32602, + "isError": true, + "clientInfo": map[string]interface{}{ + "name": "claude-desktop", + "version": "1.2.0", + "requestedProtocolVersion": "2025-03-26", + }, + "serverInfo": map[string]interface{}{ + "protocolVersion": "2025-06-18", + "name": "everything-server", + "version": "0.9.1", + }, } o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} @@ -404,6 +413,8 @@ func TestBuildRecordMCP(t *testing.T) { "wso2.mcp.client.requested_protocol_version": "2025-03-26", "wso2.mcp.client.name": "claude-desktop", "wso2.mcp.client.version": "1.2.0", + "wso2.mcp.server.name": "everything-server", + "wso2.mcp.server.version": "0.9.1", } { if got[key] != expected { t.Errorf("%s = %v, want %v", key, got[key], expected) @@ -1593,3 +1604,141 @@ func TestHeaderAttributeWireShape(t *testing.T) { t.Errorf("record does not contain the expected ArrayValue attribute.\nwant substring: %s\ngot: %s", want, encoded) } } + +// OTLP log records have traceId/spanId envelope fields. We deliberately do not +// emit them (design doc §2.10), and this pins that: adding the fields "for +// completeness" would send an all-zero id to every destination on every record. +// +// Note that a collector's own re-serialization may still show traceId:"" — its +// internal representation holds those as fixed-size values that are always +// present. That is the collector's output format, not our payload. +func TestRecordOmitsTraceEnvelopeFields(t *testing.T) { + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + encoded, err := json.Marshal(o.buildRecord(restEvent())) + if err != nil { + t.Fatalf("marshal record: %v", err) + } + for _, field := range []string{"traceId", "spanId"} { + if strings.Contains(string(encoded), field) { + t.Errorf("record carries %q; it must be absent, not empty: %s", field, encoded) + } + } +} + +// A key the analytics policy gains later must appear in the export rather than +// vanish until someone notices. That is the whole point of the sweep: flattening +// to curated names would otherwise mean the publisher and the policy drift +// silently. +func TestMCPUnmappedKeysAreSweptUp(t *testing.T) { + event := restEvent() + event.API.APIType = "Mcp" + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "jsonRpcMethod": "tools/call", + // Hypothetical future additions, at both levels. + "toolInvocationCount": float64(3), + "cacheWasWarm": true, + "upstreamLatencyMs": 12.5, + "clientInfo": map[string]interface{}{ + "name": "claude-desktop", + "platformId": "darwin-arm64", + }, + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, want := range map[string]interface{}{ + // camelCase becomes snake_case under the custom namespace. + "wso2.mcp.tool_invocation_count": "3", // integral float -> IntValue, formatted as a string + "wso2.mcp.cache_was_warm": true, + "wso2.mcp.upstream_latency_ms": 12.5, + "wso2.mcp.client.platform_id": "darwin-arm64", + // The curated name still wins for a key that has one. + "mcp.method.name": "tools/call", + "wso2.mcp.client.name": "claude-desktop", + } { + if got[key] != want { + t.Errorf("%s = %#v, want %#v", key, got[key], want) + } + } + + // A key with a curated name must not also appear under the sweep prefix. + for _, absent := range []string{ + "wso2.mcp.json_rpc_method", "wso2.mcp.client_info", "wso2.mcp.client.name_", + } { + if _, present := got[absent]; present { + t.Errorf("%s was emitted twice / under the wrong name", absent) + } + } +} + +// `capability` is read but deliberately not emitted — which attribute is +// populated already says it. It must not reappear via the sweep. +func TestMCPCapabilityIsNotSwept(t *testing.T) { + event := restEvent() + event.API.APIType = "Mcp" + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "jsonRpcMethod": "tools/call", + "capability": "TOOL", + "capabilityName": "search_docs", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for _, absent := range []string{"wso2.mcp.capability", "wso2.mcp.capability_name"} { + if _, present := got[absent]; present { + t.Errorf("%s must not be swept up; it has a curated mapping", absent) + } + } + if got["gen_ai.tool.name"] != "search_docs" { + t.Errorf("gen_ai.tool.name = %v, want search_docs", got["gen_ai.tool.name"]) + } +} + +func TestOTelSnakeCase(t *testing.T) { + for input, want := range map[string]string{ + "jsonRpcMethod": "json_rpc_method", + "resourceUri": "resource_uri", + "isError": "is_error", + "requestedProtocolVersion": "requested_protocol_version", + "sessionId": "session_id", + "already_snake": "already_snake", + "name": "name", + "": "", + } { + if got := otelSnakeCase(input); got != want { + t.Errorf("otelSnakeCase(%q) = %q, want %q", input, got, want) + } + } +} + +// An integral JSON number must not be reported as a double: a backend types the +// column from the first value it sees, and a consumer summing 3.0 and 3 does not +// reliably get the same answer. +func TestAnyScalarNumberKinds(t *testing.T) { + attrs := newOTelAttrs() + attrs.anyScalar("whole", float64(3)) + attrs.anyScalar("fractional", 12.5) + attrs.anyScalar("text", "hello") + attrs.anyScalar("flag", true) + attrs.anyScalar("unsupported", []string{"nope"}) + + encoded, err := json.Marshal(attrs.list()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{ + `{"key":"whole","value":{"intValue":"3"}}`, + `{"key":"fractional","value":{"doubleValue":12.5}}`, + `{"key":"text","value":{"stringValue":"hello"}}`, + `{"key":"flag","value":{"boolValue":true}}`, + } { + if !strings.Contains(string(encoded), want) { + t.Errorf("missing %s in %s", want, encoded) + } + } + if strings.Contains(string(encoded), "unsupported") { + t.Error("an unsupported type produced an attribute") + } +} From d44a15c62748c3749004d0e7eaed02e22fa0217e Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Tue, 8 Sep 2026 16:54:02 +0530 Subject: [PATCH 08/20] Minor bug fix for parsing pesudo headers --- .../internal/analytics/publishers/otel.go | 9 ++++++ .../analytics/publishers/otel_test.go | 28 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index e4561520c9..30c2235ea6 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -834,6 +834,15 @@ func appendHeaderAttributes(attrs *otelAttrs, prefix string, raw interface{}) { emitted := 0 for _, name := range names { + // HTTP/2 pseudo-headers (:method, :path, :scheme, :authority, :status) + // are not headers: Envoy surfaces them alongside the real ones, they + // duplicate attributes already mapped from their own event fields + // (http.request.method, url.path, server.address, + // http.response.status_code), and ":path" can carry a query string. + // They also make an attribute key that starts with a colon. + if strings.HasPrefix(name, ":") { + continue + } if emitted == otelMaxHeaderAttributes { slog.Warn("OTel publisher truncated header attributes; narrow the analytics-header-filter allowlist", "prefix", prefix, "emitted", emitted, "available", len(names)) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index e68c7bb811..be885227e1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -1527,6 +1527,34 @@ func headerEventWrongType() *dto.Event { return event } +// HTTP/2 pseudo-headers are not headers. Envoy surfaces them next to the real +// ones, and each duplicates an attribute already mapped from its own event +// field — while ":path" can carry a query string into an attribute key's value. +func TestBuildRecordHeaderAttributesSkipPseudoHeaders(t *testing.T) { + event := headerEvent(t, + map[string]string{ + ":method": "GET", ":path": "/otele2e/anything?token=secret", + ":scheme": "http", ":authority": "localhost:8080", + "x-tenant": "acme", + }, + map[string]string{":status": "200", "content-type": "application/json"}) + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key := range got { + if strings.Contains(key, ".header.:") { + t.Errorf("pseudo-header emitted as %q", key) + } + } + // The real headers alongside them must still come through. + for _, want := range []string{"http.request.header.x-tenant", "http.response.header.content-type"} { + if _, present := got[want]; !present { + t.Errorf("%s is missing", want) + } + } +} + // An over-broad allowlist must not grow a record's schema without bound: header // names become attribute keys, and SDKs cap a record at 128 attributes. func TestBuildRecordHeaderAttributesAreCapped(t *testing.T) { From d4cb996ea199d89303c4f5309aabeffb3372ede4 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Thu, 10 Sep 2026 10:08:07 +0530 Subject: [PATCH 09/20] Minor modifications to the error related analytics attribute mapping --- .../gateway-controller/pkg/xds/translator.go | 4 + .../internal/analytics/analytics.go | 6 +- .../internal/analytics/dto/event.go | 5 - .../policy-engine/internal/analytics/fault.go | 106 ++++++------- .../internal/analytics/fault_test.go | 147 +++++++++++++----- .../internal/analytics/publishers/moesif.go | 13 +- .../analytics/publishers/moesif_test.go | 62 ++++---- .../internal/analytics/publishers/otel.go | 9 +- .../analytics/publishers/otel_test.go | 69 +++----- 9 files changed, 225 insertions(+), 196 deletions(-) diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 8d0c4b33e2..4957596d56 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -2730,6 +2730,10 @@ func (t *Translator) createGRPCAccessLog() (*accesslog.AccessLog, error) { Timeout: durationpb.New(time.Duration(grpcConfig.GRPCRequestTimeout)), }, }, + // populates HTTPResponseProperties.ResponseHeaders + // so responseContentType can resolve for a request that matched no route and + // therefore never reached the policy chain. + AdditionalResponseHeadersToLog: []string{"content-type"}, } grpcAccessLogAny, err := anypb.New(httpGrpcAccessLog) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index d8a9f7e1cd..b6aae2b6b3 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -685,11 +685,9 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E event.Properties["mcpAnalytics"] = mcpAnalytics } - // Fault classification, last so it sees the finished event. + // Fault classification, last so it sees the finished event. fault := classifyFault(logEntry) - event.EventCategory = fault.EventCategory - event.FaultCategory = fault.FaultCategory - event.ErrorType = fault.ErrorType + event.ErrorType = string(fault.ErrorType) if fault.SubCategory != "" { event.Error = &dto.Error{ // The client-visible status. The in-development fault flow owns the diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 5cbdf7e64b..a475b61041 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -76,11 +76,6 @@ type Event struct { ErrorType string `json:"errorType,omitempty" bson:"error_type"` Properties map[string]interface{} `json:"properties,omitempty" bson:"properties"` - // EventCategory and FaultCategory are derived from the Envoy access log's - // response flags and detail string by classifyFault - EventCategory EventCategory `json:"eventCategory,omitempty" bson:"event_category"` - FaultCategory FaultCategory `json:"faultCategory,omitempty" bson:"fault_category"` - // TrafficLogLatencies carries microsecond-precision gateway/backend timings // for the stdout traffic-logging publisher. It is computed from the same ALS // CommonProperties timepoints as Latencies but at full precision, and is kept diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go b/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go index 2e1ecd4680..2add99bb42 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/fault.go @@ -18,7 +18,7 @@ package analytics import ( - "strconv" + "net/http" v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" @@ -35,17 +35,11 @@ const responseCodeDetailsViaUpstream = "via_upstream" // canonical event so every publisher reads one classification instead of each // inventing its own. type faultClassification struct { - EventCategory dto.EventCategory - FaultCategory dto.FaultCategory - // ErrorType is the value of the stable OpenTelemetry `error.type` attribute: - // the response flag's name for a gateway-originated fault, or the HTTP status - // code for a response the upstream itself produced (which is what the HTTP - // semantic conventions ask for). - ErrorType string + // ErrorType is the fault category: one of dto.FaultCategory's four values, + // or empty when the request was not a gateway fault. + ErrorType dto.FaultCategory // SubCategory is set only where the response flag determines it. Where the - // flag proves the category but not the specific cause — RateLimited does not - // say which limit was hit — the category's own "OTHER" is used rather than a - // guess. + // flag proves the category but not the specific cause SubCategory dto.FaultSubCategory } @@ -142,6 +136,32 @@ var flagFaults = []flagFault{ dto.FaultCategoryOther, dto.OtherUnclassified}, } +// statusFault is the classification a gateway-synthesized status code implies. +type statusFault struct { + category dto.FaultCategory + subCategory dto.FaultSubCategory +} + +// statusFaults names the cause for the status codes the gateway itself produces +// for a known reason. A map is safe here where flagFaults needed a slice: these +// are exact lookups on one status code, so there is no overlap for iteration +// order to disturb. +// +// This is an interim table. It exists because the flag path cannot see these +// faults at all: the policy engine's own denials (api-key-auth's 401, +// basic-ratelimit's 429) reach the access log with no response flag set and an +// empty response_code_details, so classifyFault has nothing else to key on. When +// the fault flow lands and reports the specific cause, it supersedes this table +// and the sub-categories become exact (which limit was exceeded, which claim +// failed) rather than the generic value used here. +var statusFaults = map[int]statusFault{ + http.StatusUnauthorized: {dto.FaultCategoryAuth, dto.AuthenticationFailure}, + http.StatusForbidden: {dto.FaultCategoryAuth, dto.AuthenticationAuthorizationFailure}, + http.StatusNotFound: {dto.FaultCategoryOther, dto.OtherResourceNotFound}, + http.StatusMethodNotAllowed: {dto.FaultCategoryOther, dto.OtherMethodNotAllowed}, + http.StatusTooManyRequests: {dto.FaultCategoryThrottled, dto.ThrottlingOther}, +} + // classifyFault derives the error view of a request from the Envoy access log. // // Response flags are the primary signal rather than the status code, because the @@ -150,64 +170,46 @@ var flagFaults = []flagFault{ // via_upstream is the backend's own answer. Those belong in different categories // and would be indistinguishable from a status-range table. // -// Precedence: a matching response flag, then a response Envoy synthesized -// without setting one, then an upstream-produced 5xx. +// Precedence: a matching response flag, then — for a response the gateway +// synthesized — the status code, then a generic error for any other 4xx/5xx. func classifyFault(logEntry *v3.HTTPAccessLogEntry) faultClassification { - success := faultClassification{EventCategory: dto.EventCategorySuccess} - flags := logEntry.GetCommonProperties().GetResponseFlags() for _, candidate := range flagFaults { if candidate.set(flags) { - return faultClassification{ - EventCategory: dto.EventCategoryFault, - FaultCategory: candidate.category, - ErrorType: candidate.name, - SubCategory: candidate.subCategory, - } + return faultClassification{ErrorType: candidate.category, SubCategory: candidate.subCategory} } } response := logEntry.GetResponse() if response == nil { - return success + return faultClassification{} } status := int(response.GetResponseCode().GetValue()) details := response.GetResponseCodeDetails() - // No flag matched, but the response did not come from the upstream: a filter - // or Envoy produced it (a policy denial, a direct response, a redirect). The - // detail string itself is high-cardinality and stays on - // wso2.upstream.response.detail; error.type gets one stable value. - if details != "" && details != responseCodeDetailsViaUpstream { - if status < 400 { - return success // a synthesized redirect or 200 is not a fault - } - return faultClassification{ - EventCategory: dto.EventCategoryFault, - FaultCategory: dto.FaultCategoryOther, - ErrorType: "local_reply", - SubCategory: dto.OtherUnclassified, - } + // Anything below 400 is not an error + if status < 400 { + return faultClassification{} } - // The upstream answered. A 5xx is the backend's own failure — the gateway - // worked — so it is a fault in a category that does not blame connectivity. - // A 4xx is the backend's valid answer to an invalid request: error.type is - // set because the HTTP operation did fail, but no fault is recorded against - // the gateway. - switch { - case status >= 500: - return faultClassification{ - EventCategory: dto.EventCategoryFault, - FaultCategory: dto.FaultCategoryOther, - ErrorType: strconv.Itoa(status), - SubCategory: dto.OtherUnclassified, + // Only a response the gateway synthesized lets the status code name the + // cause. via_upstream means the backend chose this status, and then the + // specific sub-categories would assert something false: a backend enforcing + // its own auth returns 401 after the gateway's authentication already + // succeeded, and a backend with its own rate limit returns 429 without the + // gateway throttling anything. Attributing either to the gateway sends + // triage to the wrong component and inflates the gateway's own throttling + // counts. The origin stays legible to consumers regardless: the detail + // string is published as wso2.upstream.response.detail. + if details != responseCodeDetailsViaUpstream { + if fault, ok := statusFaults[status]; ok { + return faultClassification{ErrorType: fault.category, SubCategory: fault.subCategory} } - case status >= 400: - success.ErrorType = strconv.Itoa(status) - return success } - return success + + // An upstream-chosen status, or one the table does not name: still an error, + // in the category that claims nothing about the cause. + return faultClassification{ErrorType: dto.FaultCategoryOther, SubCategory: dto.OtherUnclassified} } // isCacheHit reports whether Envoy served the response from its cache filter. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go index a4d71f4128..75fef91640 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/fault_test.go @@ -18,6 +18,8 @@ package analytics import ( + "fmt" + "net/http" "reflect" "testing" @@ -52,9 +54,9 @@ func TestClassifyFault_EveryMappedFlag(t *testing.T) { setFlagByErrorType(t, flags, mapped.name) got := classifyFault(faultEntry(flags, 503, "")) - assert.Equal(t, dto.EventCategoryFault, got.EventCategory) - assert.Equal(t, mapped.name, got.ErrorType) - assert.Equal(t, mapped.category, got.FaultCategory) + // The category is what lands in ErrorType — the field existing Moesif + // consumers read. The flag name only identifies the table row. + assert.Equal(t, mapped.category, got.ErrorType) assert.Equal(t, mapped.subCategory, got.SubCategory) }) } @@ -108,9 +110,8 @@ func TestClassifyFault_InformationalFlagsAreNotFaults(t *testing.T) { for name, flags := range cases { t.Run(name, func(t *testing.T) { got := classifyFault(faultEntry(flags, 200, responseCodeDetailsViaUpstream)) - assert.Equal(t, dto.EventCategorySuccess, got.EventCategory) assert.Empty(t, got.ErrorType) - assert.Empty(t, got.FaultCategory) + assert.Empty(t, got.SubCategory) }) } } @@ -118,38 +119,37 @@ func TestClassifyFault_InformationalFlagsAreNotFaults(t *testing.T) { // The distinction the status code cannot make: same 503, two different causes. func TestClassifyFault_SameStatusDifferentCause(t *testing.T) { gateway := classifyFault(faultEntry(&v3.ResponseFlags{NoHealthyUpstream: true}, 503, "no_healthy_upstream")) - assert.Equal(t, dto.FaultCategoryTargetConnectivity, gateway.FaultCategory, + assert.Equal(t, dto.FaultCategoryTargetConnectivity, gateway.ErrorType, "a 503 the backend never saw is a connectivity fault") - assert.Equal(t, "no_healthy_upstream", gateway.ErrorType) backend := classifyFault(faultEntry(&v3.ResponseFlags{}, 503, responseCodeDetailsViaUpstream)) - assert.Equal(t, dto.FaultCategoryOther, backend.FaultCategory, + assert.Equal(t, dto.FaultCategoryOther, backend.ErrorType, "a 503 the backend answered is not a connectivity fault") - assert.Equal(t, "503", backend.ErrorType, - "an upstream-produced error uses the status code, per the HTTP semantic conventions") } func TestClassifyFault_UpstreamResponses(t *testing.T) { cases := []struct { name string status uint32 - wantCat dto.EventCategory - wantType string + wantType dto.FaultCategory wantSub dto.FaultSubCategory }{ - {"200 is a success", 200, dto.EventCategorySuccess, "", ""}, - {"301 is a success", 301, dto.EventCategorySuccess, "", ""}, - // The gateway did its job; the backend's 404 is a valid answer to an - // invalid request. error.type is still set: the HTTP operation failed. - {"404 sets error.type without a fault", 404, dto.EventCategorySuccess, "404", ""}, - {"429 sets error.type without a fault", 429, dto.EventCategorySuccess, "429", ""}, - {"500 is a backend fault", 500, dto.EventCategoryFault, "500", dto.OtherUnclassified}, - {"502 is a backend fault", 502, dto.EventCategoryFault, "502", dto.OtherUnclassified}, + {"200 is not a fault", 200, "", ""}, + {"301 is not a fault", 301, "", ""}, + // The backend chose these, so they are recorded as errors but the + // gateway-specific sub-categories are withheld: the gateway neither + // authenticated, throttled, nor failed to route. + {"400 is a generic error", 400, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"401 is not attributed to gateway auth", 401, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"403 is not attributed to gateway authz", 403, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"404 is a generic error", 404, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"429 is not attributed to gateway throttling", 429, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"500 is a backend fault", 500, dto.FaultCategoryOther, dto.OtherUnclassified}, + {"502 is a backend fault", 502, dto.FaultCategoryOther, dto.OtherUnclassified}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, responseCodeDetailsViaUpstream)) - assert.Equal(t, tc.wantCat, got.EventCategory) assert.Equal(t, tc.wantType, got.ErrorType) assert.Equal(t, tc.wantSub, got.SubCategory) }) @@ -163,24 +163,95 @@ func TestClassifyFault_SynthesizedResponses(t *testing.T) { cases := []struct { details string status uint32 - wantCat dto.EventCategory - wantType string + wantType dto.FaultCategory }{ - {"ext_authz_denied", 403, dto.EventCategoryFault, "local_reply"}, - {"direct_response", 401, dto.EventCategoryFault, "local_reply"}, - {"ext_proc_error_gRPC_error_13", 500, dto.EventCategoryFault, "local_reply"}, + // A status in statusFaults names its own cause when the gateway + // synthesized the response. + {"ext_authz_denied", 403, dto.FaultCategoryAuth}, + {"direct_response", 401, dto.FaultCategoryAuth}, + {"direct_response", 404, dto.FaultCategoryOther}, + // Not in the table: an error, with nothing claimed about the cause. + {"ext_proc_error_gRPC_error_13", 500, dto.FaultCategoryOther}, // A synthesized redirect is not a failure. - {"direct_response", 302, dto.EventCategorySuccess, ""}, + {"direct_response", 302, ""}, } for _, tc := range cases { - t.Run(tc.details+"_"+string(rune('0'+tc.status/100)), func(t *testing.T) { + t.Run(fmt.Sprintf("%s_%d", tc.details, tc.status), func(t *testing.T) { got := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, tc.details)) - assert.Equal(t, tc.wantCat, got.EventCategory) assert.Equal(t, tc.wantType, got.ErrorType) }) } } +// Every entry in statusFaults must be reachable for a gateway-synthesized +// response. Driven off the table itself, so an entry added without a test +// cannot slip through. Empty details are used deliberately: that is exactly +// what the policy engine's own denials produce. +func TestClassifyFault_EveryMappedStatus(t *testing.T) { + require.NotEmpty(t, statusFaults) + for status, want := range statusFaults { + t.Run(fmt.Sprintf("status_%d", status), func(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, uint32(status), "")) + assert.Equal(t, want.category, got.ErrorType) + assert.Equal(t, want.subCategory, got.SubCategory) + }) + } +} + +// The gating: an identical status code classifies differently depending on who +// produced the response. Without this, a backend's own 401/429 would be +// attributed to the gateway's authentication or rate limiting. +func TestClassifyFault_StatusMappingIsGatedOnOrigin(t *testing.T) { + cases := []struct { + status uint32 + wantGateway dto.FaultCategory + wantSubGw dto.FaultSubCategory + }{ + {http.StatusUnauthorized, dto.FaultCategoryAuth, dto.AuthenticationFailure}, + {http.StatusForbidden, dto.FaultCategoryAuth, dto.AuthenticationAuthorizationFailure}, + {http.StatusNotFound, dto.FaultCategoryOther, dto.OtherResourceNotFound}, + {http.StatusMethodNotAllowed, dto.FaultCategoryOther, dto.OtherMethodNotAllowed}, + {http.StatusTooManyRequests, dto.FaultCategoryThrottled, dto.ThrottlingOther}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("status_%d", tc.status), func(t *testing.T) { + gw := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, "direct_response")) + assert.Equal(t, tc.wantGateway, gw.ErrorType, "gateway-synthesized names the cause") + assert.Equal(t, tc.wantSubGw, gw.SubCategory) + + up := classifyFault(faultEntry(&v3.ResponseFlags{}, tc.status, responseCodeDetailsViaUpstream)) + assert.Equal(t, dto.FaultCategoryOther, up.ErrorType, + "an upstream-chosen status must not be attributed to the gateway") + assert.Equal(t, dto.OtherUnclassified, up.SubCategory) + }) + } +} + +// A response flag outranks the status table: a 429 carrying RateLimited is +// classified by the flag, and a 404 from NoRouteFound keeps its flag-derived +// sub-category rather than being re-derived from the status. +func TestClassifyFault_FlagOutranksStatus(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{RateLimited: true}, 429, "direct_response")) + assert.Equal(t, dto.FaultCategoryThrottled, got.ErrorType) + assert.Equal(t, dto.ThrottlingOther, got.SubCategory) + + got = classifyFault(faultEntry(&v3.ResponseFlags{NoRouteFound: true}, 404, "route_not_found")) + assert.Equal(t, dto.FaultCategoryOther, got.ErrorType) + assert.Equal(t, dto.OtherResourceNotFound, got.SubCategory) +} + +// The boundary is >= 400, not > 400: a 400 is an error, a 399 is not. +func TestClassifyFault_ErrorBoundaryIncludes400(t *testing.T) { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, 400, "direct_response")) + assert.Equal(t, dto.FaultCategoryOther, got.ErrorType, "400 is an error") + assert.Equal(t, dto.OtherUnclassified, got.SubCategory) + + for _, status := range []uint32{200, 204, 301, 304, 399} { + got := classifyFault(faultEntry(&v3.ResponseFlags{}, status, "direct_response")) + assert.Empty(t, got.ErrorType, "status %d must not be a fault", status) + } +} + // Several flags are routinely set on one request. The table is ordered, so the // result must be deterministic — a map would return either one at random. func TestClassifyFault_MultipleFlagsAreDeterministic(t *testing.T) { @@ -191,9 +262,10 @@ func TestClassifyFault_MultipleFlagsAreDeterministic(t *testing.T) { } for i := 0; i < 50; i++ { got := classifyFault(faultEntry(flags, 504, "")) - require.Equal(t, "upstream_request_timeout", got.ErrorType, + require.Equal(t, dto.FaultCategoryTargetConnectivity, got.ErrorType, "the most specific flag must win on every call") - require.Equal(t, dto.TargetConnectivityConnectionTimeout, got.SubCategory) + require.Equal(t, dto.TargetConnectivityConnectionTimeout, got.SubCategory, + "upstream_request_timeout maps to CONNECTION_TIMEOUT, not the generic OTHER") } } @@ -201,7 +273,7 @@ func TestClassifyFault_MultipleFlagsAreDeterministic(t *testing.T) { // throttled: counting it as THROTTLED would misattribute an outage to callers. func TestClassifyFault_RateLimitServiceErrorIsNotThrottling(t *testing.T) { got := classifyFault(faultEntry(&v3.ResponseFlags{RateLimitServiceError: true}, 500, "")) - assert.Equal(t, dto.FaultCategoryOther, got.FaultCategory) + assert.Equal(t, dto.FaultCategoryOther, got.ErrorType) assert.Equal(t, dto.OtherMediationError, got.SubCategory) } @@ -218,7 +290,7 @@ func TestClassifyFault_NilSafety(t *testing.T) { for name, entry := range cases { t.Run(name, func(t *testing.T) { got := classifyFault(entry) - assert.Equal(t, dto.EventCategorySuccess, got.EventCategory) + assert.Empty(t, got.ErrorType) }) } } @@ -241,9 +313,8 @@ func TestPrepareAnalyticEvent_WritesFaultClassification(t *testing.T) { event := analytics.prepareAnalyticEvent(logEntry) require.NotNil(t, event) - assert.Equal(t, dto.EventCategoryFault, event.EventCategory) - assert.Equal(t, dto.FaultCategoryTargetConnectivity, event.FaultCategory) - assert.Equal(t, "upstream_request_timeout", event.ErrorType) + // The category lands in ErrorType — the field existing consumers read. + assert.Equal(t, string(dto.FaultCategoryTargetConnectivity), event.ErrorType) require.NotNil(t, event.Error) assert.Equal(t, dto.TargetConnectivityConnectionTimeout, event.Error.ErrorMessage) // The client-visible status, populated before the classification runs. @@ -260,7 +331,6 @@ func TestPrepareAnalyticEvent_WritesFaultClassification(t *testing.T) { event := analytics.prepareAnalyticEvent(logEntry) require.NotNil(t, event) - assert.Equal(t, dto.EventCategorySuccess, event.EventCategory) assert.Empty(t, event.ErrorType) assert.Nil(t, event.Error, "a successful request must not carry an error object") }) @@ -275,7 +345,6 @@ func TestPrepareAnalyticEvent_WritesFaultClassification(t *testing.T) { require.NotNil(t, event) require.NotNil(t, event.Target) assert.True(t, event.Target.ResponseCacheHit) - assert.Equal(t, dto.EventCategorySuccess, event.EventCategory, - "a cache hit is not a fault") + assert.Empty(t, event.ErrorType, "a cache hit is not a fault") }) } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go index bec7fe6686..7302be2cfa 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif.go @@ -325,13 +325,10 @@ func (m *Moesif) Publish(event *dto.Event) { } // Fault classification, derived in analytics.classifyFault from the Envoy - // response flags. eventCategory is sent unconditionally so a consumer can - // tell "this request succeeded" from "this event predates the field"; the - // rest are omitted when the request was not a fault. - metadataMap["eventCategory"] = string(event.EventCategory) - if event.FaultCategory != "" { - metadataMap["faultCategory"] = string(event.FaultCategory) - } + // response flags. Three keys only, matching what API Manager's Moesif + // integration already reads: errorType carries the fault category (AUTH / + // TARGET_CONNECTIVITY / THROTTLED / OTHER), errorMessage the sub-category. + // All three are omitted when the request was not a gateway fault. if event.ErrorType != "" { metadataMap["errorType"] = event.ErrorType } @@ -340,7 +337,7 @@ func (m *Moesif) Publish(event *dto.Event) { metadataMap["errorCode"] = event.Error.ErrorCode } if event.Error.ErrorMessage != "" { - metadataMap["faultSubCategory"] = string(event.Error.ErrorMessage) + metadataMap["errorMessage"] = string(event.Error.ErrorMessage) } } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go index 68e7bc3441..90589bf042 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go @@ -361,61 +361,57 @@ func TestPublish_McpAPIType(t *testing.T) { } // The fault taxonomy reaches Moesif, not only the OTLP publisher: both read the -// same classification off the canonical event. +// same classification off the canonical event. Exactly three keys, matching what +// API Manager's Moesif integration already reads. func TestPublish_WithFaultClassification(t *testing.T) { moesif := createTestMoesifWithoutAPI() event := createBaseEvent() - event.EventCategory = dto.EventCategoryFault - event.FaultCategory = dto.FaultCategoryTargetConnectivity - event.ErrorType = "upstream_request_timeout" + event.ErrorType = string(dto.FaultCategoryTargetConnectivity) event.Error = &dto.Error{ErrorCode: 504, ErrorMessage: dto.TargetConnectivityConnectionTimeout} moesif.Publish(event) assert.Len(t, moesif.events, 1) metadata := getMetadata(moesif.events[0]) - assert.Equal(t, "FAULT", metadata["eventCategory"]) - assert.Equal(t, "TARGET_CONNECTIVITY", metadata["faultCategory"]) - assert.Equal(t, "upstream_request_timeout", metadata["errorType"]) + assert.Equal(t, "TARGET_CONNECTIVITY", metadata["errorType"]) assert.Equal(t, 504, metadata["errorCode"]) - assert.Equal(t, "CONNECTION_TIMEOUT", metadata["faultSubCategory"]) + assert.Equal(t, "CONNECTION_TIMEOUT", metadata["errorMessage"]) + // The category/event-category enums belong to the in-development fault flow + // and must not be published from here. + for _, absent := range []string{"eventCategory", "faultCategory", "faultSubCategory"} { + assert.NotContains(t, metadata, absent) + } } -// A successful request carries the category and nothing else, so a fault filter -// in Moesif keys on presence rather than having to exclude empty strings. +// A request that was not a gateway fault carries none of the three keys, so a +// consumer can filter on presence. func TestPublish_SuccessOmitsFaultFields(t *testing.T) { moesif := createTestMoesifWithoutAPI() - - event := createBaseEvent() - event.EventCategory = dto.EventCategorySuccess - - moesif.Publish(event) + moesif.Publish(createBaseEvent()) assert.Len(t, moesif.events, 1) metadata := getMetadata(moesif.events[0]) - assert.Equal(t, "SUCCESS", metadata["eventCategory"]) - for _, key := range []string{"faultCategory", "errorType", "errorCode", "faultSubCategory"} { + for _, key := range []string{"errorType", "errorCode", "errorMessage"} { assert.NotContains(t, metadata, key, "%s must be absent on a successful event", key) } } -// An upstream 4xx sets errorType without recording a gateway fault: the HTTP -// operation failed, but the gateway did its job. -func TestPublish_UpstreamErrorTypeWithoutFault(t *testing.T) { - moesif := createTestMoesifWithoutAPI() - - event := createBaseEvent() - event.EventCategory = dto.EventCategorySuccess - event.ErrorType = "404" - - moesif.Publish(event) - - assert.Len(t, moesif.events, 1) - metadata := getMetadata(moesif.events[0]) - assert.Equal(t, "SUCCESS", metadata["eventCategory"]) - assert.Equal(t, "404", metadata["errorType"]) - assert.NotContains(t, metadata, "faultCategory") +// Every value errorType can take is one of the four categories API Manager +// expects — never an Envoy flag name or a status code. +func TestPublish_ErrorTypeIsAlwaysAFaultCategory(t *testing.T) { + for _, category := range []dto.FaultCategory{ + dto.FaultCategoryAuth, dto.FaultCategoryThrottled, + dto.FaultCategoryTargetConnectivity, dto.FaultCategoryOther, + } { + moesif := createTestMoesifWithoutAPI() + event := createBaseEvent() + event.ErrorType = string(category) + moesif.Publish(event) + + metadata := getMetadata(moesif.events[0]) + assert.Equal(t, string(category), metadata["errorType"]) + } } func TestPublish_WithPayloads(t *testing.T) { diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 30c2235ea6..7454bc0b5b 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -762,14 +762,13 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { attrs.b(ns("cache.hit"), event.Target.ResponseCacheHit) } - // Faults. error.type is the one stable error attribute; the categories are - // ours, derived in analytics.classifyFault from the Envoy response flags. + // Faults. error.type is the one stable error attribute, and it carries the + // fault category derived in analytics.classifyFault — the same value existing + // Moesif consumers read from errorType. attrs.str("error.type", event.ErrorType) - attrs.str(ns("event.category"), string(event.EventCategory)) - attrs.str(ns("error.category"), string(event.FaultCategory)) if event.Error != nil { attrs.i64(ns("error.code"), int64(event.Error.ErrorCode)) - attrs.str(ns("error.sub_category"), string(event.Error.ErrorMessage)) + attrs.str(ns("error.message"), string(event.Error.ErrorMessage)) } // Payloads, only present when body capture is enabled on the collector. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index be885227e1..9d804b7937 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -252,7 +252,7 @@ func TestBuildRecordFaults(t *testing.T) { for key, expected := range map[string]interface{}{ "error.type": "AUTH", "wso2.error.code": "900901", - "wso2.error.sub_category": "AUTHENTICATION_FAILURE", + "wso2.error.message": "AUTHENTICATION_FAILURE", "http.response.status_code": "401", } { if got[key] != expected { @@ -261,40 +261,40 @@ func TestBuildRecordFaults(t *testing.T) { } } -// The categories analytics.classifyFault derives must reach the record. -func TestBuildRecordFaultCategories(t *testing.T) { +// error.type carries the fault category derived by analytics.classifyFault — the +// same value existing Moesif consumers read from errorType. +func TestBuildRecordFaultCategoryInErrorType(t *testing.T) { event := restEvent() - event.EventCategory = dto.EventCategoryFault - event.FaultCategory = dto.FaultCategoryTargetConnectivity - event.ErrorType = "upstream_request_timeout" + event.ErrorType = string(dto.FaultCategoryTargetConnectivity) + event.Error = &dto.Error{ErrorCode: 504, ErrorMessage: dto.TargetConnectivityConnectionTimeout} o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} got := attrMap(t, o.buildRecord(event)) for key, expected := range map[string]interface{}{ - "wso2.event.category": "FAULT", - "wso2.error.category": "TARGET_CONNECTIVITY", - "error.type": "upstream_request_timeout", + "error.type": "TARGET_CONNECTIVITY", + "wso2.error.code": "504", + "wso2.error.message": "CONNECTION_TIMEOUT", } { if got[key] != expected { t.Errorf("%s = %v, want %v", key, got[key], expected) } } + // The category/event-category enums belong to the in-development fault flow. + for _, absent := range []string{"wso2.event.category", "wso2.error.category", "wso2.error.sub_category"} { + if _, present := got[absent]; present { + t.Errorf("%s must not be emitted", absent) + } + } } -// A successful request must carry the SUCCESS category and no error attributes — -// an absent wso2.error.category is what lets a consumer filter faults. +// A request that was not a gateway fault carries no error attributes at all — +// absence is what lets a consumer filter faults. func TestBuildRecordSuccessOmitsErrorAttributes(t *testing.T) { - event := restEvent() - event.EventCategory = dto.EventCategorySuccess - o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} - got := attrMap(t, o.buildRecord(event)) + got := attrMap(t, o.buildRecord(restEvent())) - if got["wso2.event.category"] != "SUCCESS" { - t.Errorf("wso2.event.category = %v, want SUCCESS", got["wso2.event.category"]) - } - for _, key := range []string{"wso2.error.category", "error.type", "wso2.error.code", "wso2.error.sub_category"} { + for _, key := range []string{"error.type", "wso2.error.code", "wso2.error.message"} { if _, present := got[key]; present { t.Errorf("%s is present on a successful record", key) } @@ -343,37 +343,6 @@ func TestBuildRecordGenAI(t *testing.T) { } } -// An unrecognised provider leaves the enum attribute unset rather than carrying -// a non-member value, and keeps its identity in the wso2.* attribute. -func TestBuildRecordUnknownGenAIProvider(t *testing.T) { - event := restEvent() - event.Properties["aiMetadata"] = dto.AIMetadata{VendorName: "some-private-llm", Model: "m1"} - - o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} - got := attrMap(t, o.buildRecord(event)) - - if _, present := got["gen_ai.provider.name"]; present { - t.Errorf("gen_ai.provider.name should be unset for an unknown provider, got %v", got["gen_ai.provider.name"]) - } - if got["wso2.gen_ai.provider.template_name"] != "some-private-llm" { - t.Errorf("template_name = %v", got["wso2.gen_ai.provider.template_name"]) - } -} - -func TestGenAIOperationName(t *testing.T) { - for route, want := range map[string]string{ - "/ai/chat/completions": "chat", - "/anthropic/messages": "chat", - "/ai/embeddings": "embeddings", - "/ai/completions": "text_completion", - "/petstore/pet/{id}": "", - } { - if got := otelGenAIOperationName(route); got != want { - t.Errorf("otelGenAIOperationName(%q) = %q, want %q", route, got, want) - } - } -} - func TestBuildRecordMCP(t *testing.T) { event := restEvent() event.API.APIType = "Mcp" From 9169cb1a09a1cc58779c73f3a233f78187d32860 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Thu, 10 Sep 2026 13:37:30 +0530 Subject: [PATCH 10/20] fix(otel): preserve zero-valued attributes --- .../internal/analytics/publishers/otel.go | 43 ++-- .../analytics/publishers/otel_test.go | 127 +++++++++++ .../config/otel_publisher_toml_test.go | 201 ++++++++++++++++++ 3 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 7454bc0b5b..2d8a178542 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -676,7 +676,7 @@ func gzipBytes(body []byte) ([]byte, error) { // MCP conventions (both Development-stability, and now maintained in // open-telemetry/semantic-conventions-genai) for AI fields, and the wso2.* // namespace where OpenTelemetry defines nothing — API-product concepts and cost, -// chiefly. See gateway/spec/analytics-otel-attribute-mapping.md. +// chiefly. func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { attrs := newOTelAttrs() attrs.str("event.name", otelEventName) @@ -693,7 +693,7 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { route = event.API.APIContext } attrs.str("url.path", route) - attrs.i64("http.response.status_code", int64(event.ProxyResponseCode)) + attrs.i64NonZero("http.response.status_code", int64(event.ProxyResponseCode)) attrs.str("client.address", event.UserIP) attrs.str("user_agent.original", event.UserAgentHeader) attrs.anyInt("http.request.body.size", event.Properties["requestSize"]) @@ -751,13 +751,13 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { if host, port, err := net.SplitHostPort(authority); err == nil { attrs.str("server.address", host) if p, convErr := strconv.Atoi(port); convErr == nil { - attrs.i64("server.port", int64(p)) + attrs.i64NonZero("server.port", int64(p)) } } else { attrs.str("server.address", authority) } } - attrs.i64(ns("upstream.response.status_code"), int64(event.Target.TargetResponseCode)) + attrs.i64NonZero(ns("upstream.response.status_code"), int64(event.Target.TargetResponseCode)) attrs.str(ns("upstream.response.detail"), event.Target.ResponseCodeDetail) attrs.b(ns("cache.hit"), event.Target.ResponseCacheHit) } @@ -767,7 +767,7 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { // Moesif consumers read from errorType. attrs.str("error.type", event.ErrorType) if event.Error != nil { - attrs.i64(ns("error.code"), int64(event.Error.ErrorCode)) + attrs.i64NonZero(ns("error.code"), int64(event.Error.ErrorCode)) attrs.str(ns("error.message"), string(event.Error.ErrorMessage)) } @@ -1146,8 +1146,13 @@ type otelArrayValue struct { Values []otelAnyValue `json:"values"` } -// otelAttrs accumulates attributes, skipping empty ones so a record carries only -// what the event populated. +// otelAttrs accumulates the attributes of one record. +// +// A string attribute is omitted when empty: OpenTelemetry discourages +// empty-string attributes and nothing in this mapping means anything by "". +// +// Zero and false values are preserved as valid measurements; only values +// explicitly marked with `i64NonZero` are omitted when zero means “no value.” type otelAttrs struct { kvs []otelKeyValue } @@ -1156,6 +1161,8 @@ func newOTelAttrs() *otelAttrs { return &otelAttrs{kvs: make([]otelKeyValue, 0, 48)} } +// str sets a string attribute, omitting it when empty. Deliberately unlike +// i64/f64/b below — see the type comment. func (a *otelAttrs) str(key, value string) *otelAttrs { if value == "" { return a @@ -1193,28 +1200,36 @@ func (a *otelAttrs) strIfEmpty(key, value string) *otelAttrs { return a.str(key, value) } +// i64 sets an integer attribute, including when the value is 0. func (a *otelAttrs) i64(key string, value int64) *otelAttrs { - if value == 0 { - return a - } v := strconv.FormatInt(value, 10) a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{IntValue: &v}}) return a } -func (a *otelAttrs) f64(key string, value float64) *otelAttrs { +// i64NonZero sets an integer attribute only when it is non-zero, for the fields +// where 0 is a sentinel for "no value" rather than a measurement: there is no +// HTTP status 0, no TCP port 0, and no error code 0. Every other integer +// attribute uses i64 so a real zero survives. +func (a *otelAttrs) i64NonZero(key string, value int64) *otelAttrs { if value == 0 { return a } + return a.i64(key, value) +} + +// f64 sets a floating-point attribute, including when the value is 0. A zero +// cost — a free model, a cached completion, a blocked request — is a fact. +func (a *otelAttrs) f64(key string, value float64) *otelAttrs { v := value a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{DoubleValue: &v}}) return a } +// b sets a boolean attribute, including when false. A false cache.hit is a cache +// miss; omitting it would make a miss indistinguishable from an API with no cache +// filter at all, which is exactly what makes a hit-ratio query unanswerable. func (a *otelAttrs) b(key string, value bool) *otelAttrs { - if !value { - return a - } v := value a.kvs = append(a.kvs, otelKeyValue{Key: key, Value: otelAnyValue{BoolValue: &v}}) return a diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 9d804b7937..9a29ad3f59 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -1739,3 +1739,130 @@ func TestAnyScalarNumberKinds(t *testing.T) { t.Error("an unsupported type produced an attribute") } } + +// --- Zero is a value, not an absence ----------------------------------------- +// +// A record has one way to say "this attribute does not apply to this request": +// leave it out. So a measured zero must be emitted, or "the guardrail blocked +// this request so it produced no output tokens" and "this is a REST call with no +// tokens at all" become the same record to a consumer — and every avg() over the +// field drops the zeros from its denominator instead of counting them. + +// The wire shape is what actually carries the distinction. omitempty on a +// pointer field tests only for nil, which is why a *string holding "0" and a +// *bool holding false still marshal. Asserted here because a later refactor to +// non-pointer fields would silently restore the bug this test exists to prevent. +func TestZeroValuedAttributesReachTheWire(t *testing.T) { + attrs := newOTelAttrs() + attrs.i64("zero.int", 0) + attrs.f64("zero.double", 0) + attrs.b("zero.bool", false) + + encoded, err := json.Marshal(attrs.list()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{ + `{"key":"zero.int","value":{"intValue":"0"}}`, + `{"key":"zero.double","value":{"doubleValue":0}}`, + `{"key":"zero.bool","value":{"boolValue":false}}`, + } { + if !strings.Contains(string(encoded), want) { + t.Errorf("missing %s in %s", want, encoded) + } + } +} + +// A guardrail-blocked completion: the prompt was tokenized, nothing was +// generated, nothing was billed. Reporting no output tokens is the whole point +// of the record. +func TestBuildRecordGenAIZeroUsageIsReported(t *testing.T) { + event := restEvent() + event.API.APIType = "LlmProxy" + event.Operation.APIResourceTemplate = "/ai/chat/completions" + event.Properties["aiMetadata"] = dto.AIMetadata{ + Model: "claude-opus-4", + VendorName: "anthropic", + LLMCost: float64(0), + } + event.Properties["aiTokenUsage"] = dto.AITokenUsage{PromptToken: 1841, CompletionToken: 0, TotalToken: 1841} + event.Properties[constants.GuardrailHitMetadataKey] = true + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, want := range map[string]interface{}{ + "gen_ai.usage.input_tokens": "1841", + "gen_ai.usage.output_tokens": "0", + "wso2.gen_ai.usage.total_tokens": "1841", + "wso2.gen_ai.cost.total": float64(0), + "wso2.guardrail.hit": true, + } { + actual, present := got[key] + if !present { + t.Errorf("%s is absent; a measured zero must be emitted, not omitted", key) + continue + } + if actual != want { + t.Errorf("%s = %v (%T), want %v (%T)", key, actual, actual, want, want) + } + } +} + +// A GET has no request body and a cache miss is not a cache hit. Both are +// measurements the record must carry: without them "empty body" is +// indistinguishable from "body size not measured", and a cache miss from an API +// with no cache filter at all — which is what makes a hit ratio uncomputable. +func TestBuildRecordZeroSizesAndFalseFlagsAreReported(t *testing.T) { + event := restEvent() + event.Properties["requestSize"] = uint64(0) + event.Properties["responseSize"] = uint64(0) + event.Target.ResponseCacheHit = false + event.Latencies = &dto.Latencies{ResponseLatency: 4, BackendLatency: 0, Duration: 4} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for key, want := range map[string]interface{}{ + "http.request.body.size": "0", + "http.response.body.size": "0", + "wso2.cache.hit": false, + // Served from cache or sub-millisecond: zero backend time, measured. + "wso2.latency.backend_ms": "0", + "wso2.latency.request_mediation_ms": "0", + "wso2.latency.response_mediation_ms": "0", + } { + actual, present := got[key] + if !present { + t.Errorf("%s is absent; a measured zero must be emitted, not omitted", key) + continue + } + if actual != want { + t.Errorf("%s = %v (%T), want %v (%T)", key, actual, actual, want, want) + } + } +} + +// The exceptions. For these four, 0 is a sentinel rather than a measurement — +// there is no HTTP status 0, no TCP port 0, no error code 0 — so they keep +// suppressing it via i64NonZero. +func TestBuildRecordSentinelZerosStayOmitted(t *testing.T) { + event := restEvent() + event.ProxyResponseCode = 0 + event.Target = &dto.Target{TargetResponseCode: 0, Destination: "backend:0/pet/1"} + event.Error = &dto.Error{ErrorCode: 0, ErrorMessage: dto.OtherUnclassified} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + for _, key := range []string{ + "http.response.status_code", + "server.port", + "wso2.upstream.response.status_code", + "wso2.error.code", + } { + if actual, present := got[key]; present { + t.Errorf("%s = %v; 0 is a sentinel for this attribute and must be omitted", key, actual) + } + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go new file mode 100644 index 0000000000..60359f3e1e --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The struct-level validation tests build OTelPublisherConfig values in Go, which +// proves the rules but never exercises the koanf tags. A mistyped tag fails +// silently — the key is ignored and the default is used — so an operator sets +// on_queue_full or queue_capacity, sees no error, and gets default behaviour. +// These tests load real TOML through Load() and assert every field. + +func writeOTelTOML(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(p, []byte(body), 0o600)) + return p +} + +// Every OTel publisher key set to a non-default value, so a field that failed to +// bind shows up as its default rather than what the TOML asked for. +func TestLoad_OTelPublisher_EveryKeyBindsFromTOML(t *testing.T) { + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +endpoint = "https://collector.example.com:4318/v1/logs" +service_name = "plumbing-test" +service_version = "9.9.9" +batch_size = 7 +flush_interval = "3s" +queue_capacity = 42 +on_queue_full = "drop_oldest" +timeout = "11s" +max_retries = 5 +retry_backoff = "250ms" +retry_abort_queue_ratio = 0.25 +compression = "gzip" + +[analytics.publishers.otel.headers] +"x-api-key" = "secret-value" + +[analytics.publishers.otel.resource_attributes] +"deployment.environment" = "staging" +"service.namespace" = "gw" +`) + cfg, err := Load(path) + require.NoError(t, err) + + o := cfg.Analytics.Publishers.OTel + assert.Equal(t, "https://collector.example.com:4318/v1/logs", o.Endpoint, "endpoint") + assert.Equal(t, "plumbing-test", o.ServiceName, "service_name") + assert.Equal(t, "9.9.9", o.ServiceVersion, "service_version") + assert.Equal(t, 7, o.BatchSize, "batch_size") + assert.Equal(t, 3*time.Second, o.FlushInterval, "flush_interval") + assert.Equal(t, 42, o.QueueCapacity, "queue_capacity") + assert.Equal(t, QueueDropOldest, o.OnQueueFull, "on_queue_full") + assert.Equal(t, 11*time.Second, o.Timeout, "timeout") + assert.Equal(t, 5, o.MaxRetries, "max_retries") + assert.Equal(t, 250*time.Millisecond, o.RetryBackoff, "retry_backoff") + assert.InDelta(t, 0.25, o.RetryAbortQueueRatio, 1e-9, "retry_abort_queue_ratio") + assert.Equal(t, OTelCompressionGzip, o.Compression, "compression") + assert.Equal(t, map[string]string{"x-api-key": "secret-value"}, o.Headers, "headers") + assert.Equal(t, map[string]string{ + "deployment.environment": "staging", + "service.namespace": "gw", + }, o.ResourceAttributes, "resource_attributes") + + // The abort depth an operator actually gets from these two keys together. + assert.Equal(t, 10, o.EffectiveRetryAbortDepth(), "42 * 0.25 truncates to 10") +} + +// The TLS sub-table is its own koanf level, so it binds separately from the +// publisher keys above. +func TestLoad_OTelPublisher_TLSBindsFromTOML(t *testing.T) { + // writeSelfSignedPair (otel_publisher_test.go) emits a self-signed CA cert and + // its key; the same cert serves as ca_file and as the mTLS client cert here, + // which is all the validation needs — it checks the material parses, not that + // it chains to anything. + certPath, keyPath := writeSelfSignedPair(t) + caPath := certPath + + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +endpoint = "https://collector.example.com:4318/v1/logs" + +[analytics.publishers.otel.tls] +ca_file = "`+caPath+`" +cert_file = "`+certPath+`" +key_file = "`+keyPath+`" +insecure_skip_verify = true +`) + cfg, err := Load(path) + require.NoError(t, err) + + tlsCfg := cfg.Analytics.Publishers.OTel.TLS + assert.Equal(t, caPath, tlsCfg.CAFile, "ca_file") + assert.Equal(t, certPath, tlsCfg.CertFile, "cert_file") + assert.Equal(t, keyPath, tlsCfg.KeyFile, "key_file") + assert.True(t, tlsCfg.InsecureSkipVerify, "insecure_skip_verify") +} + +// Keys left out of the TOML must fall back to the shipped defaults rather than +// zero values, which would fail validation or silently disable batching. +func TestLoad_OTelPublisher_OmittedKeysKeepDefaults(t *testing.T) { + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +endpoint = "http://otel-collector:4318/v1/logs" +`) + cfg, err := Load(path) + require.NoError(t, err) + + o := cfg.Analytics.Publishers.OTel + assert.Equal(t, "policy-engine", o.ServiceName, "default service_name") + assert.Equal(t, 100, o.BatchSize, "default batch_size") + assert.Equal(t, 5*time.Second, o.FlushInterval, "default flush_interval") + assert.Equal(t, 10000, o.QueueCapacity, "default queue_capacity") + assert.Equal(t, QueueDropNew, o.OnQueueFull, "default on_queue_full") + assert.Equal(t, 10*time.Second, o.Timeout, "default timeout") + assert.Equal(t, 3, o.MaxRetries, "default max_retries") + assert.Equal(t, time.Second, o.RetryBackoff, "default retry_backoff") + assert.InDelta(t, DefaultOTelRetryAbortQueueRatio, o.RetryAbortQueueRatio, 1e-9, "default abort ratio") + assert.Equal(t, OTelCompressionNone, o.Compression, "default compression") +} + +// A bad value in the TOML must fail the load, not be silently coerced. This is +// what makes the process refuse to start rather than run with defaults. +func TestLoad_OTelPublisher_InvalidTOMLValuesFailClosed(t *testing.T) { + cases := []struct { + name string + body string + // wantErr is a substring the rejection must contain, so a case cannot + // pass on an unrelated error (a duplicate TOML key, say). + wantErr string + }{ + {"unknown drop policy", `on_queue_full = "sometimes"`, "on_queue_full must be"}, + {"queue smaller than batch", "queue_capacity = 5\nbatch_size = 50", "must be >= batch_size"}, + {"zero queue capacity", `queue_capacity = 0`, "queue_capacity must be > 0"}, + {"unknown compression", `compression = "snappy"`, "compression must be"}, + {"abort ratio above one", `retry_abort_queue_ratio = 1.5`, "retry_abort_queue_ratio must be between"}, + {"negative retries", `max_retries = -1`, "max_retries must be >= 0"}, + {"zero flush interval", `flush_interval = "0s"`, "flush_interval must be > 0"}, + {"zero timeout", `timeout = "0s"`, "timeout must be > 0"}, + {"retries without backoff", "max_retries = 2\nretry_backoff = \"0s\"", "retry_backoff must be positive"}, + {"non-http scheme", `endpoint = "ftp://collector:4318/v1/logs"`, "scheme must be http or https"}, + {"missing ca file", `[analytics.publishers.otel.tls]` + "\n" + `ca_file = "/nonexistent/ca.pem"`, "cannot read ca_file"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + base := "endpoint = \"http://otel-collector:4318/v1/logs\"\n" + if strings.Contains(tc.body, "endpoint =") { + base = "" // the case sets its own; a second one is a TOML parse error + } + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +`+base+tc.body+"\n") + _, err := Load(path) + require.Error(t, err, "invalid TOML value must fail the load") + assert.Contains(t, err.Error(), tc.wantErr, "must be rejected for the stated reason") + t.Logf("rejected with: %v", err) + }) + } +} From 5b4ab1f49fcccba2e2aa0aa2f1068df7c8b40fd2 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Thu, 10 Sep 2026 14:23:21 +0530 Subject: [PATCH 11/20] Add proper URL path extraction logic and mapping --- .../internal/analytics/analytics.go | 6 +++ .../internal/analytics/analytics_test.go | 43 ++++++++++++++++++ .../internal/analytics/publishers/otel.go | 8 ++-- .../analytics/publishers/otel_test.go | 44 ++++++++++++++----- .../internal/constants/constants.go | 3 +- 5 files changed, 87 insertions(+), 17 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index b6aae2b6b3..3a011cef00 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -26,6 +26,7 @@ import ( "maps" "net" "strconv" + "strings" "sync" "time" @@ -618,6 +619,11 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E // requestSize is common to all API kinds; mirror responseSize using the Envoy access-log byte count. if request != nil { event.Properties["requestSize"] = request.GetRequestBodyBytes() + + // Store the concrete request path (without query parameters), separate from the route template. + if requestPath, _, _ := strings.Cut(request.GetPath(), "?"); requestPath != "" { + event.Properties[constants.RequestPathPropertyKey] = requestPath + } } //Adding request and response headers for the analytics event diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 2969693f9c..7f69edf346 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -1312,3 +1312,46 @@ func createLogEntryWithStreamID(streamID string) *v3.HTTPAccessLogEntry { }, } } + +// The concrete request path is the only record of what a client actually asked +// for: the route template groups requests for an operation, and for a request +// that matched no route there is no template at all. The query string is cut off +// here rather than at each publisher, because an API key or token in a query +// parameter is an ordinary pattern in this product and publishers forward +// analytics to third parties. +func TestPrepareAnalyticEvent_RequestPathDropsQueryString(t *testing.T) { + for name, tc := range map[string]struct { + path string + want interface{} // nil = the property must be absent + }{ + "no query": {"/petstore/pet/12345", "/petstore/pet/12345"}, + "single param": {"/petstore/pet/12345?apikey=secret", "/petstore/pet/12345"}, + "multiple params": {"/search?q=cat&token=abc123&page=2", "/search"}, + "empty query": {"/petstore/pet/12345?", "/petstore/pet/12345"}, + "query only": {"?apikey=secret", nil}, + "root": {"/", "/"}, + "absent path": {"", nil}, + "encoded question mark": {"/pet/a%3Fb", "/pet/a%3Fb"}, + } { + t.Run(name, func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{}) + logEntry.Request.Path = tc.path + + event := NewAnalytics(&config.Config{}).prepareAnalyticEvent(logEntry) + actual, present := event.Properties[constants.RequestPathPropertyKey] + + if tc.want == nil { + if present { + t.Errorf("%s = %v; want absent", constants.RequestPathPropertyKey, actual) + } + return + } + if !present { + t.Fatalf("%s is missing for path %q", constants.RequestPathPropertyKey, tc.path) + } + if actual != tc.want { + t.Errorf("%s = %v, want %v", constants.RequestPathPropertyKey, actual, tc.want) + } + }) + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 2d8a178542..102bd16824 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -681,8 +681,7 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { attrs := newOTelAttrs() attrs.str("event.name", otelEventName) - // HTTP. APIResourceTemplate already carries the full path including the API - // context; the context is only a fallback when there is no template. + // Keep http.route as the route template and url.path as the literal client-requested path. route := "" if event.Operation != nil { route = event.Operation.APIResourceTemplate @@ -692,7 +691,8 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { if route == "" && event.API != nil { route = event.API.APIContext } - attrs.str("url.path", route) + // Query strings are omitted to avoid exposing tokens or API keys to third-party analytics. + attrs.anyStr("url.path", event.Properties[constants.RequestPathPropertyKey]) attrs.i64NonZero("http.response.status_code", int64(event.ProxyResponseCode)) attrs.str("client.address", event.UserIP) attrs.str("user_agent.original", event.UserAgentHeader) @@ -1151,7 +1151,7 @@ type otelArrayValue struct { // A string attribute is omitted when empty: OpenTelemetry discourages // empty-string attributes and nothing in this mapping means anything by "". // -// Zero and false values are preserved as valid measurements; only values +// Zero and false values are preserved as valid measurements; only values // explicitly marked with `i64NonZero` are omitted when zero means “no value.” type otelAttrs struct { kvs []otelKeyValue diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 9a29ad3f59..d51bee121f 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -94,8 +94,11 @@ func restEvent() *dto.Event { Properties: map[string]interface{}{ dto.PropKeyAuthUserID: "user-1", "requestSize": uint64(12), - "responseSize": uint64(463), - "responseContentType": "application/json", + // The concrete path, as analytics.go now supplies it: the route + // template is "/petstore/pet/{petId}", this is one request against it. + constants.RequestPathPropertyKey: "/petstore/pet/12345", + "responseSize": uint64(463), + "responseContentType": "application/json", }, } } @@ -153,10 +156,11 @@ func TestBuildRecordRestAPI(t *testing.T) { got := attrMap(t, record) want := map[string]interface{}{ - "event.name": "wso2.api.transaction", - "http.request.method": "GET", - "http.route": "/petstore/pet/{petId}", - "url.path": "/petstore/pet/{petId}", + "event.name": "wso2.api.transaction", + "http.request.method": "GET", + "http.route": "/petstore/pet/{petId}", + // The template groups requests; the path identifies one. Never equal. + "url.path": "/petstore/pet/12345", "http.response.status_code": "200", "client.address": "10.0.0.5", "user_agent.original": "curl/8.7.1", @@ -221,25 +225,41 @@ func TestBuildRecordRestAPI(t *testing.T) { func TestBuildRecordDoesNotDuplicateContext(t *testing.T) { o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} got := attrMap(t, o.buildRecord(restEvent())) - if got["url.path"] != "/petstore/pet/{petId}" { - t.Errorf("url.path = %v, want /petstore/pet/{petId}", got["url.path"]) + if got["http.route"] != "/petstore/pet/{petId}" { + t.Errorf("http.route = %v, want /petstore/pet/{petId}", got["http.route"]) } } -// With no resource template the API context is the fallback path. -func TestBuildRecordFallsBackToContext(t *testing.T) { +// No resource template means no route to report — but the client still asked for +// something, and url.path is the only record of what. This is the request that +// most needs a path: nothing matched, so triage has the template nowhere else. +// url.path deliberately does NOT fall back to the API context, which would put a +// value in the attribute that was never the requested path. +func TestBuildRecordWithNoRouteKeepsConcretePath(t *testing.T) { event := restEvent() event.Operation.APIResourceTemplate = "" o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} got := attrMap(t, o.buildRecord(event)) - if got["url.path"] != "/petstore" { - t.Errorf("url.path = %v, want /petstore", got["url.path"]) + if got["url.path"] != "/petstore/pet/12345" { + t.Errorf("url.path = %v, want /petstore/pet/12345", got["url.path"]) } if _, present := got["http.route"]; present { t.Error("http.route should be absent when there is no resource template") } } +// Absent rather than guessed: with no path on the event there is nothing +// truthful to put in url.path, and the API context is not the requested path. +func TestBuildRecordOmitsURLPathWhenUnavailable(t *testing.T) { + event := restEvent() + delete(event.Properties, constants.RequestPathPropertyKey) + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + if actual, present := got["url.path"]; present { + t.Errorf("url.path = %v; it must be absent, not fall back to the context", actual) + } +} + func TestBuildRecordFaults(t *testing.T) { event := restEvent() event.ProxyResponseCode = 401 diff --git a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go index cb395bb2f5..236814e860 100644 --- a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go +++ b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go @@ -128,5 +128,6 @@ const ( GuardrailNameMetadataKey = "guardrailName" LLMCostMetadataKey = "x-llm-cost" LLMCostPropertyKey = "llmCost" - RequestModelPropertyKey = "requestModel" // Holds the model named in the request + RequestModelPropertyKey = "requestModel" // Holds the model named in the request + RequestPathPropertyKey = "requestPath" // Holds the concrete request path, query string removed. ) From da16abdde767fcd1f061283731b81461690d3a93 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Thu, 10 Sep 2026 14:50:35 +0530 Subject: [PATCH 12/20] fix(analytics): strip query strings from request paths --- .../internal/analytics/analytics.go | 9 +++-- .../internal/analytics/analytics_test.go | 37 +++++++++++++++++++ .../internal/analytics/publishers/otel.go | 2 +- .../internal/constants/constants.go | 2 +- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 3a011cef00..041fbe9184 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -347,6 +347,9 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E request := logEntry.GetRequest() response := logEntry.GetResponse() + // Strip the query once at the source since it is shared across publishers and may contain credentials. + requestPath, _, _ := strings.Cut(request.GetPath(), "?") + // Prepare operation operation := dto.Operation{} // operation.APIResourceTemplate = keyValuePairsFromMetadata[APIResourceTemplateKey] @@ -361,7 +364,7 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E if response != nil { target.TargetResponseCode = int(logEntry.GetResponse().GetResponseCode().Value) // target.Destination = keyValuePairsFromMetadata[DestinationKey] - target.Destination = logEntry.GetRequest().GetAuthority() + logEntry.GetRequest().GetPath() + target.Destination = logEntry.GetRequest().GetAuthority() + requestPath target.ResponseCodeDetail = logEntry.GetResponse().GetResponseCodeDetails() } @@ -619,9 +622,9 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E // requestSize is common to all API kinds; mirror responseSize using the Envoy access-log byte count. if request != nil { event.Properties["requestSize"] = request.GetRequestBodyBytes() - + // Store the concrete request path (without query parameters), separate from the route template. - if requestPath, _, _ := strings.Cut(request.GetPath(), "?"); requestPath != "" { + if requestPath != "" { event.Properties[constants.RequestPathPropertyKey] = requestPath } } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 7f69edf346..5157278fb1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -20,6 +20,7 @@ package analytics import ( "bytes" "log/slog" + "strings" "testing" "time" @@ -1355,3 +1356,39 @@ func TestPrepareAnalyticEvent_RequestPathDropsQueryString(t *testing.T) { }) } } + +// Target.Destination reaches three consumers — the OTel publisher's +// wso2.upstream.destination, the traffic log's destination field, and the +// target.destination policy expression — so the query string is stripped at this +// single point rather than in each of them. An API key or token in a query +// parameter is an ordinary pattern here, and all three carry records off-box. +func TestPrepareAnalyticEvent_DestinationDropsQueryString(t *testing.T) { + for name, tc := range map[string]struct { + authority string + path string + want string + }{ + "no query": {"api.example.com", "/orders/v1.0/listings", "api.example.com/orders/v1.0/listings"}, + "credential": {"api.example.com", "/orders/v1.0/listings?apikey=secret", "api.example.com/orders/v1.0/listings"}, + "multiple params": {"localhost:8080", "/nofilter/anything?q=1&token=abc", "localhost:8080/nofilter/anything"}, + "bare question": {"localhost:8080", "/everything?", "localhost:8080/everything"}, + "no path": {"api.example.com", "", "api.example.com"}, + } { + t.Run(name, func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{}) + logEntry.Request.Authority = tc.authority + logEntry.Request.Path = tc.path + + event := NewAnalytics(&config.Config{}).prepareAnalyticEvent(logEntry) + if event.Target == nil { + t.Fatal("event.Target is nil") + } + if event.Target.Destination != tc.want { + t.Errorf("Destination = %q, want %q", event.Target.Destination, tc.want) + } + if strings.Contains(event.Target.Destination, "?") { + t.Errorf("Destination %q still carries a query string", event.Target.Destination) + } + }) + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 102bd16824..adedb060c1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -681,7 +681,7 @@ func (o *OTel) buildRecord(event *dto.Event) *otelLogRecord { attrs := newOTelAttrs() attrs.str("event.name", otelEventName) - // Keep http.route as the route template and url.path as the literal client-requested path. + // Keep http.route as the route template and url.path as the literal client-requested path. route := "" if event.Operation != nil { route = event.Operation.APIResourceTemplate diff --git a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go index 236814e860..0c61aa0b5d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go +++ b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go @@ -129,5 +129,5 @@ const ( LLMCostMetadataKey = "x-llm-cost" LLMCostPropertyKey = "llmCost" RequestModelPropertyKey = "requestModel" // Holds the model named in the request - RequestPathPropertyKey = "requestPath" // Holds the concrete request path, query string removed. + RequestPathPropertyKey = "requestPath" // Holds the concrete request path, query string removed. ) From f92b6f4ef6a192b232ba3c0b1e7aa666fe67574f Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Thu, 10 Sep 2026 15:06:51 +0530 Subject: [PATCH 13/20] Fix otel publisher metrics to count records actually accepted by the collector rather than records merely sent --- .../internal/analytics/publishers/otel.go | 58 ++++++++++++------- .../analytics/publishers/otel_test.go | 50 +++++++++++++++- 2 files changed, 83 insertions(+), 25 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index adedb060c1..51f1df0fc4 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -413,9 +413,12 @@ func (o *OTel) export(batch []*otelLogRecord) { } } - retryAfter, err := o.post(body, len(records)) + retryAfter, rejected, err := o.post(body, len(records)) if err == nil { - mAnalyticsPublished(otelPublisherName, len(records)) + // Records refused inside a 2xx are already counted as dropped, so + // publishing the whole batch would count them twice and let + // published+dropped exceed the number of events that ever existed. + mAnalyticsPublished(otelPublisherName, len(records)-rejected) return } lastErr = err @@ -440,16 +443,17 @@ func (e *otelPermanentExportError) Error() string { return fmt.Sprintf("endpoint rejected the batch with status %d", e.status) } -// post performs one export attempt, returning the endpoint's requested +// post performs one export attempt. It returns the endpoint's requested // Retry-After when it supplies one so the caller can honor it over its own -// backoff. -func (o *OTel) post(body []byte, records int) (time.Duration, error) { +// backoff, and how many records the endpoint refused inside a 2xx so the caller +// does not count those as published. +func (o *OTel) post(body []byte, records int) (time.Duration, int, error) { ctx, cancel := context.WithTimeout(context.Background(), o.cfg.Timeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPost, o.cfg.Endpoint, bytes.NewReader(body)) if err != nil { - return 0, fmt.Errorf("building request: %w", err) + return 0, 0, fmt.Errorf("building request: %w", err) } req.Header.Set("Content-Type", "application/json") if o.gzip { @@ -464,7 +468,7 @@ func (o *OTel) post(body []byte, records int) (time.Duration, error) { mAnalyticsExportError(otelPublisherName, errCodeTransport, 1) // The error can embed the endpoint URL but never the payload, so no // request data can leak into the application log here. - return 0, fmt.Errorf("posting batch: %w", err) + return 0, 0, fmt.Errorf("posting batch: %w", err) } defer resp.Body.Close() @@ -474,42 +478,52 @@ func (o *OTel) post(body []byte, records int) (time.Duration, error) { _, _ = io.Copy(io.Discard, resp.Body) // drain the rest so the connection is reusable if resp.StatusCode >= 200 && resp.StatusCode < 300 { - o.logPartialSuccess(respBody, records) - return 0, nil + return 0, o.recordPartialSuccess(respBody, records), nil } mAnalyticsExportError(otelPublisherName, strconv.Itoa(resp.StatusCode), 1) if resp.StatusCode == http.StatusTooManyRequests { - return parseRetryAfter(resp.Header.Get("Retry-After")), + return parseRetryAfter(resp.Header.Get("Retry-After")), 0, fmt.Errorf("endpoint is rate limiting (429)") } if resp.StatusCode >= 500 { - return 0, fmt.Errorf("endpoint returned status %d: %s", resp.StatusCode, otelResponseExcerpt(respBody)) + return 0, 0, fmt.Errorf("endpoint returned status %d: %s", resp.StatusCode, otelResponseExcerpt(respBody)) } - return 0, &otelPermanentExportError{status: resp.StatusCode} + return 0, 0, &otelPermanentExportError{status: resp.StatusCode} } -// logPartialSuccess reports records the endpoint accepted the request for but -// rejected. Without this a 200 carrying rejectedLogRecords looks like a clean -// export, and the records are silently gone. -func (o *OTel) logPartialSuccess(respBody []byte, records int) { +// recordPartialSuccess counts the records the endpoint accepted the request for +// but rejected, and returns that count so the caller can exclude them from the +// published tally. Without this a 200 carrying rejectedLogRecords looks like a +// clean export, and the records are silently gone. +func (o *OTel) recordPartialSuccess(respBody []byte, records int) int { if len(respBody) == 0 { - return + return 0 } var parsed otelExportResponse if err := json.Unmarshal(respBody, &parsed); err != nil { - return // a non-JSON 2xx body is not an error; nothing to report + return 0 // a non-JSON 2xx body is not an error; nothing to report } // Proto3 JSON encodes int64 as a string, but some receivers emit a bare // number, so the field is json.Number to accept either. - rejected, err := parsed.PartialSuccess.RejectedLogRecords.Int64() - if err != nil || rejected <= 0 { - return + reported, err := parsed.PartialSuccess.RejectedLogRecords.Int64() + if err != nil || reported <= 0 { + return 0 + } + // The count is the endpoint's claim, and it cannot exceed what was sent. An + // over-report has to be clamped rather than trusted: the caller subtracts + // this from the batch size, and a Prometheus counter panics on a negative + // Add — which in the export worker would take the process down. + rejected := int(min(reported, int64(records))) + if reported > int64(records) { + slog.Warn("OTel endpoint reported more rejected records than were sent; clamping", + "reported", reported, "records", records) } - o.dropRecords(dropReasonRejected, int(rejected)) + o.dropRecords(dropReasonRejected, rejected) slog.Error("OTel endpoint accepted the export but rejected records", "rejected", rejected, "records", records, "endpointMessage", parsed.PartialSuccess.ErrorMessage) + return rejected } // sleep waits out the backoff before a retry, returning false if shutdown was diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index d51bee121f..5f33dc68d2 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -915,7 +915,19 @@ func retryConfig(endpoint string, maxRetries int) config.OTelPublisherConfig { func (o *OTel) exportOne(t *testing.T) { t.Helper() - o.export([]*otelLogRecord{o.buildRecord(restEvent())}) + o.exportN(t, 1) +} + +// exportN exports a batch of n identical records. Any test asserting a rejected +// count needs this: the endpoint cannot reject more records than were sent, so a +// single-record batch can only ever exercise a rejected count of 1. +func (o *OTel) exportN(t *testing.T, n int) { + t.Helper() + batch := make([]*otelLogRecord, 0, n) + for i := 0; i < n; i++ { + batch = append(batch, o.buildRecord(restEvent())) + } + o.export(batch) } // A 5xx is transient: retry until it clears, and lose nothing when it does. @@ -1063,15 +1075,47 @@ func TestExportCountsPartialSuccessRejections(t *testing.T) { defer server.Close() o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 3)) - o.exportOne(t) + before := scrapeMetrics(t) + o.exportN(t, 5) if dropped := o.droppedCount(); dropped != 2 { t.Errorf("dropped = %d, want 2 from partialSuccess", dropped) } + // The request succeeded, so 3 of the 5 were published — not all 5. + // Counting the whole batch would report 5 published and 2 dropped for + // 5 events that existed. + published := seriesKey("policy_engine_analytics_published_total") + if got := delta(t, before, scrapeMetrics(t), published); got != 3 { + t.Errorf("published delta = %v, want 3 (5 sent, 2 rejected)", got) + } }) } } +// An endpoint claiming more rejections than were sent is claiming something +// impossible. It has to be clamped rather than trusted: the published tally is +// the batch size minus this count, and a Prometheus counter panics on a negative +// Add — which on the export worker would take the process down. +func TestExportClampsOverReportedRejections(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"partialSuccess":{"rejectedLogRecords":"99999"}}`)) + })) + defer server.Close() + + o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 0)) + before := scrapeMetrics(t) + o.exportN(t, 2) // must not panic + + if dropped := o.droppedCount(); dropped != 2 { + t.Errorf("dropped = %d, want 2 clamped to the batch size", dropped) + } + published := seriesKey("policy_engine_analytics_published_total") + if got := delta(t, before, scrapeMetrics(t), published); got != 0 { + t.Errorf("published delta = %v, want 0 — the whole batch was rejected", got) + } +} + // The ordinary success shapes must not be read as rejections. func TestExportCleanSuccessBodiesCountNoDrops(t *testing.T) { bodies := map[string]string{ @@ -1329,7 +1373,7 @@ func TestMetricsDropReasons(t *testing.T) { defer server.Close() o := newTestOTel(t, retryConfig(server.URL+"/v1/logs", 0)) - o.exportOne(t) + o.exportN(t, 5) if got := delta(t, before, scrapeMetrics(t), key); got != 3 { t.Errorf("rejected delta = %v, want 3", got) From 4ba06198cab05562db3dde325c04ef79ad35850a Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Thu, 10 Sep 2026 15:10:19 +0530 Subject: [PATCH 14/20] Fix default otel service name misalignment --- .../policy-engine/internal/analytics/publishers/otel_test.go | 4 ++-- .../gateway-runtime/policy-engine/internal/config/config.go | 2 +- .../policy-engine/internal/config/otel_publisher_toml_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 5f33dc68d2..2bc7861b73 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -56,7 +56,7 @@ import ( func testOTelConfig(endpoint string) config.OTelPublisherConfig { return config.OTelPublisherConfig{ Endpoint: endpoint, - ServiceName: "policy-engine", + ServiceName: "gateway-runtime", BatchSize: 100, FlushInterval: 50 * time.Millisecond, QueueCapacity: 100, @@ -562,7 +562,7 @@ func TestExportPayloadAndHeaders(t *testing.T) { } } for key, want := range map[string]string{ - "service.name": "policy-engine", + "service.name": "gateway-runtime", "service.version": "1.2.0", "deployment.environment.name": "test", } { diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 5eec64a74e..90263031f4 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -1265,7 +1265,7 @@ func defaultConfig() *Config { }, OTel: OTelPublisherConfig{ Endpoint: "http://otel-collector:4318/v1/logs", - ServiceName: "policy-engine", + ServiceName: "gateway-runtime", ServiceVersion: "", BatchSize: 100, FlushInterval: 5 * time.Second, diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go index 60359f3e1e..d66220995d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go @@ -145,7 +145,7 @@ endpoint = "http://otel-collector:4318/v1/logs" require.NoError(t, err) o := cfg.Analytics.Publishers.OTel - assert.Equal(t, "policy-engine", o.ServiceName, "default service_name") + assert.Equal(t, "gateway-runtime", o.ServiceName, "default service_name") assert.Equal(t, 100, o.BatchSize, "default batch_size") assert.Equal(t, 5*time.Second, o.FlushInterval, "default flush_interval") assert.Equal(t, 10000, o.QueueCapacity, "default queue_capacity") From 628d91700c5637ddf9f4561761837a95c1e4b84c Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Thu, 10 Sep 2026 15:23:03 +0530 Subject: [PATCH 15/20] Add a ceil value for backoff --- .../internal/analytics/publishers/otel.go | 13 ++++ .../analytics/publishers/otel_test.go | 65 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 51f1df0fc4..f8fb031d7b 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -64,6 +64,9 @@ const ( // otelCloseFlushTimeout bounds the shutdown flush when the caller's context // carries no deadline. otelCloseFlushTimeout = 5 * time.Second + // Bound retry waits to 30s to prevent an export worker from stalling and to + // match the endpoint Retry-After limit. + otelMaxRetryBackoff = 30 * time.Second // otelPublisherName is this publisher's value for the `publisher` metric // label. Kept local so metric call sites need no config import. otelPublisherName = "otel" @@ -560,7 +563,17 @@ func (o *OTel) backoff(attempt int) time.Duration { if shift > 10 { shift = 10 } + // Cap BEFORE jittering, so the jitter still spans the full range below the + // ceiling. Capping afterwards would land every attempt at the cap exactly, + // re-synchronising the replicas the jitter exists to spread out. + // + // The <= 0 arm also covers the shift overflowing int64 into a negative + // duration, which needs an absurd retry_backoff (~104 days) but would + // otherwise skip the wait altogether rather than lengthen it. delay := base << shift + if delay <= 0 || delay > otelMaxRetryBackoff { + delay = otelMaxRetryBackoff + } if half := delay / 2; half > 0 { delay = half + time.Duration(rand.Int64N(int64(half))) } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 2bc7861b73..cbe082d70e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -1930,3 +1930,68 @@ func TestBuildRecordSentinelZerosStayOmitted(t *testing.T) { } } } + +// --- Retry backoff ceiling --------------------------------------------------- +// +// One worker exports, so nothing drains the queue while a batch sleeps between +// attempts. An uncapped exponential backoff therefore parks that worker for as +// long as retry_backoff and max_retries multiply out to, and the queue behind it +// fills meanwhile — the same failure retryAfterCap already prevents an endpoint +// from causing with its own Retry-After. + +func TestBackoffGrowsExponentiallyWithinJitterRange(t *testing.T) { + o := &OTel{cfg: config.OTelPublisherConfig{RetryBackoff: time.Second}} + for attempt, want := range map[int]time.Duration{1: time.Second, 2: 2 * time.Second, 3: 4 * time.Second} { + got := o.backoff(attempt) + // Full jitter: [want/2, want). + if got < want/2 || got >= want { + t.Errorf("backoff(%d) = %s, want within [%s, %s)", attempt, got, want/2, want) + } + } +} + +func TestBackoffIsCappedAndStillJittered(t *testing.T) { + // 10m base: attempt 3 alone would be 40m uncapped. + o := &OTel{cfg: config.OTelPublisherConfig{RetryBackoff: 10 * time.Minute}} + + distinct := map[time.Duration]bool{} + for i := 0; i < 200; i++ { + got := o.backoff(3) + if got >= otelMaxRetryBackoff { + t.Fatalf("backoff = %s, want under the %s ceiling", got, otelMaxRetryBackoff) + } + if got < otelMaxRetryBackoff/2 { + t.Fatalf("backoff = %s, want at least half the ceiling", got) + } + distinct[got] = true + } + // Capped before jittering, so the delays still spread below the ceiling. Were + // the cap applied after, every attempt would land on it exactly and the + // replicas the jitter exists to spread out would resynchronise. + if len(distinct) < 2 { + t.Errorf("capped backoff produced %d distinct value(s); jitter was lost to the cap", len(distinct)) + } +} + +// A shift large enough to overflow int64 yields a negative duration, which would +// skip the wait entirely rather than lengthen it. It needs an absurd +// retry_backoff to reach, but the result must still be a real wait. +func TestBackoffOverflowFallsBackToCeiling(t *testing.T) { + o := &OTel{cfg: config.OTelPublisherConfig{RetryBackoff: 200 * 24 * time.Hour}} + got := o.backoff(11) // shift clamps to 10; 200d << 10 overflows + if got <= 0 { + t.Fatalf("backoff = %s, want a positive wait", got) + } + if got >= otelMaxRetryBackoff { + t.Errorf("backoff = %s, want under the %s ceiling", got, otelMaxRetryBackoff) + } +} + +// A zero/unset retry_backoff must still produce a real wait rather than a +// busy retry loop. +func TestBackoffUnsetBaseUsesOneSecond(t *testing.T) { + o := &OTel{cfg: config.OTelPublisherConfig{RetryBackoff: 0}} + if got := o.backoff(1); got < 500*time.Millisecond || got >= time.Second { + t.Errorf("backoff(1) = %s, want within [500ms, 1s)", got) + } +} From 14c9739f1757aa055b52084f3b96264a02b32fb6 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Fri, 11 Sep 2026 08:01:47 +0530 Subject: [PATCH 16/20] Add config to allow insecure transport --- gateway/configs/config-template.toml | 3 + .../internal/analytics/publishers/otel.go | 15 +--- .../policy-engine/internal/config/config.go | 28 ++++++- .../internal/config/otel_publisher_test.go | 74 +++++++++++++++++++ .../config/otel_publisher_toml_test.go | 26 ++++++- 5 files changed, 126 insertions(+), 20 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 43a2b397bc..b5e0466bbf 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -815,6 +815,9 @@ timer_wakeup_seconds = 3 # Full OTLP/HTTP logs URL, including the /v1/logs path. Point it at a collector # you run, or directly at a vendor's OTLP intake. endpoint = "http://otel-collector:4318/v1/logs" +# Permits a plaintext http:// endpoint, and is required by the example endpoint +# above. Off by default in code +allow_insecure_transport = true # Identifies the service service_name = "gateway-runtime" # Identifies the deployed build for rollout correlation. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index f8fb031d7b..caf78e687a 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -31,7 +31,6 @@ import ( "math/rand/v2" "net" "net/http" - "net/url" "os" "sort" "strconv" @@ -64,7 +63,7 @@ const ( // otelCloseFlushTimeout bounds the shutdown flush when the caller's context // carries no deadline. otelCloseFlushTimeout = 5 * time.Second - // Bound retry waits to 30s to prevent an export worker from stalling and to + // Bound retry waits to 30s to prevent an export worker from stalling and to // match the endpoint Retry-After limit. otelMaxRetryBackoff = 30 * time.Second // otelPublisherName is this publisher's value for the `publisher` metric @@ -163,10 +162,6 @@ func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { o.initMetrics() go o.run() - if u, err := url.Parse(cfg.Endpoint); err == nil && u.Scheme == "http" && !isLoopbackHost(u.Hostname()) { - slog.Warn("OTel publisher is exporting analytics over plaintext HTTP to a non-loopback endpoint", - "endpoint", cfg.Endpoint) - } // Headers are deliberately omitted: they carry credentials. slog.Info("OTel analytics publisher started", "endpoint", cfg.Endpoint, "batchSize", cfg.BatchSize, @@ -214,14 +209,6 @@ func buildOTelTLSConfig(cfg config.OTelTLSConfig) (*tls.Config, error) { return out, nil } -func isLoopbackHost(host string) bool { - if host == "localhost" { - return true - } - ip := net.ParseIP(host) - return ip != nil && ip.IsLoopback() -} - // Publish converts the event to an OTLP log record and enqueues it. // // Never blocks: analytics is strictly downstream of request handling, so a full diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 90263031f4..3e0bb6d757 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -177,9 +177,9 @@ type OTelPublisherConfig struct { // disables the check and lets every batch use its full budget. RetryAbortQueueRatio float64 `koanf:"retry_abort_queue_ratio"` // Compression is "none" (default, the OTLP spec's own default) or "gzip". - // gzip trades CPU on the export worker for a large egress reduction — these - // records are verbose JSON — and every OTLP/HTTP receiver must support it. Compression string `koanf:"compression"` + // AllowInsecureTransport permits a plaintext http:// endpoint. Off by default + AllowInsecureTransport bool `koanf:"allow_insecure_transport"` // TLS configures the client side of an https endpoint. Ignored for http. TLS OTelTLSConfig `koanf:"tls"` } @@ -1571,8 +1571,28 @@ func validateOTelPublisherConfig(cfg OTelPublisherConfig) error { if err != nil || u.Host == "" { return fmt.Errorf("analytics.publishers.otel.endpoint must be a valid URL (e.g. http://otel-collector:4318/v1/logs), got %q", cfg.Endpoint) } - if u.Scheme != "http" && u.Scheme != "https" { - return fmt.Errorf("analytics.publishers.otel.endpoint scheme must be http or https, got %q", u.Scheme) + switch u.Scheme { + case "https": + case "http": + if !cfg.AllowInsecureTransport { + return fmt.Errorf("analytics.publishers.otel.endpoint uses plaintext http:// but "+ + "analytics.publishers.otel.allow_insecure_transport is false; analytics records carry "+ + "API keys and consumer identity, so set allow_insecure_transport = true only for a "+ + "trusted local collector, got %q", cfg.Endpoint) + } + slog.Warn("analytics.publishers.otel endpoint is plaintext http://; analytics records are "+ + "transmitted unencrypted", "host", u.Host) + // Headers are the endpoint's intake credential. Over plaintext they are on + // the wire in clear text on every export, which the scheme warning alone + // does not convey. + if len(cfg.Headers) > 0 { + slog.Warn("analytics.publishers.otel.headers are sent over a plaintext http:// endpoint; "+ + "the credential they carry is exposed to anyone able to intercept this connection", + "host", u.Host, "headerCount", len(cfg.Headers)) + } + default: + return fmt.Errorf("analytics.publishers.otel.endpoint scheme must be https (or http with "+ + "allow_insecure_transport), got %q", u.Scheme) } if cfg.ServiceName == "" { return fmt.Errorf("analytics.publishers.otel.service_name is required") diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go index be89344ce1..3f90ecc278 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go @@ -40,10 +40,84 @@ func otelConfig(mutate func(*OTelPublisherConfig)) *Config { cfg := defaultConfig() cfg.Analytics.Enabled = true cfg.Analytics.EnabledPublishers = []string{"otel"} + // The default endpoint is plaintext http://, which now requires an explicit + // opt-in. Granting it here keeps the transport question out of every case + // that is really about batching, retries or TLS material; the gate itself is + // covered by TestValidate_OTelPublisherPlaintextTransport, and a case can + // still switch it back off via mutate. + cfg.Analytics.Publishers.OTel.AllowInsecureTransport = true mutate(&cfg.Analytics.Publishers.OTel) return cfg } +// The plaintext gate, matching traffic_logging.http.allow_insecure_transport: +// analytics records carry API keys and consumer identity, and every Headers +// value is an intake credential put on the wire on each export. +func TestValidate_OTelPublisherPlaintextTransport(t *testing.T) { + const plaintext = "http://collector.example.com:4318/v1/logs" + + t.Run("http without the flag is refused", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = plaintext + o.AllowInsecureTransport = false + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "allow_insecure_transport is false") + }) + + t.Run("http with the flag is allowed", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = plaintext + o.AllowInsecureTransport = true + }) + assert.NoError(t, cfg.Validate()) + }) + + t.Run("loopback is not exempt", func(t *testing.T) { + // The sibling sink grants no loopback exemption, so neither does this: + // "localhost" inside a container is not the operator's machine. + for _, host := range []string{"http://127.0.0.1:4318/v1/logs", "http://localhost:4318/v1/logs"} { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = host + o.AllowInsecureTransport = false + }) + err := cfg.Validate() + require.Error(t, err, "%s must still require the opt-in", host) + assert.Contains(t, err.Error(), "allow_insecure_transport is false") + } + }) + + t.Run("https needs no flag", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.AllowInsecureTransport = false + }) + assert.NoError(t, cfg.Validate()) + }) + + t.Run("credentialed plaintext is still gated", func(t *testing.T) { + // CWE-319: Headers authenticate to the intake, so plaintext exposes the + // credential itself. Covered by the same gate rather than a second rule. + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = plaintext + o.AllowInsecureTransport = false + o.Headers = map[string]string{"x-api-key": "s3cr3t"} + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "allow_insecure_transport is false") + assert.NotContains(t, err.Error(), "s3cr3t", "the error must not echo the credential") + }) + + t.Run("a non-http scheme names the opt-in", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { o.Endpoint = "ftp://collector:4318/v1/logs" }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "must be https (or http with allow_insecure_transport)") + }) +} + // writeSelfSignedPair writes a throwaway self-signed certificate and its key. func writeSelfSignedPair(t *testing.T) (certPath, keyPath string) { t.Helper() diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go index d66220995d..42235cc32d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go @@ -62,6 +62,7 @@ max_retries = 5 retry_backoff = "250ms" retry_abort_queue_ratio = 0.25 compression = "gzip" +allow_insecure_transport = true [analytics.publishers.otel.headers] "x-api-key" = "secret-value" @@ -86,6 +87,7 @@ compression = "gzip" assert.Equal(t, 250*time.Millisecond, o.RetryBackoff, "retry_backoff") assert.InDelta(t, 0.25, o.RetryAbortQueueRatio, 1e-9, "retry_abort_queue_ratio") assert.Equal(t, OTelCompressionGzip, o.Compression, "compression") + assert.True(t, o.AllowInsecureTransport, "allow_insecure_transport") assert.Equal(t, map[string]string{"x-api-key": "secret-value"}, o.Headers, "headers") assert.Equal(t, map[string]string{ "deployment.environment": "staging", @@ -140,6 +142,7 @@ enabled_publishers = ["otel"] [analytics.publishers.otel] endpoint = "http://otel-collector:4318/v1/logs" +allow_insecure_transport = true `) cfg, err := Load(path) require.NoError(t, err) @@ -157,6 +160,23 @@ endpoint = "http://otel-collector:4318/v1/logs" assert.Equal(t, OTelCompressionNone, o.Compression, "default compression") } +// allow_insecure_transport must default to off. Asserted against an https +// endpoint, so the load succeeds for a reason other than the flag. +func TestLoad_OTelPublisher_InsecureTransportDefaultsOff(t *testing.T) { + path := writeOTelTOML(t, ` +[analytics] +enabled = true +enabled_publishers = ["otel"] + +[analytics.publishers.otel] +endpoint = "https://collector.example.com:4318/v1/logs" +`) + cfg, err := Load(path) + require.NoError(t, err) + assert.False(t, cfg.Analytics.Publishers.OTel.AllowInsecureTransport, + "plaintext must be opt-in, never the default") +} + // A bad value in the TOML must fail the load, not be silently coerced. This is // what makes the process refuse to start rather than run with defaults. func TestLoad_OTelPublisher_InvalidTOMLValuesFailClosed(t *testing.T) { @@ -176,12 +196,14 @@ func TestLoad_OTelPublisher_InvalidTOMLValuesFailClosed(t *testing.T) { {"zero flush interval", `flush_interval = "0s"`, "flush_interval must be > 0"}, {"zero timeout", `timeout = "0s"`, "timeout must be > 0"}, {"retries without backoff", "max_retries = 2\nretry_backoff = \"0s\"", "retry_backoff must be positive"}, - {"non-http scheme", `endpoint = "ftp://collector:4318/v1/logs"`, "scheme must be http or https"}, + {"non-http scheme", `endpoint = "ftp://collector:4318/v1/logs"`, "must be https (or http with allow_insecure_transport)"}, + {"plaintext without opt-in", `endpoint = "http://collector.example.com:4318/v1/logs"`, "allow_insecure_transport is false"}, {"missing ca file", `[analytics.publishers.otel.tls]` + "\n" + `ca_file = "/nonexistent/ca.pem"`, "cannot read ca_file"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - base := "endpoint = \"http://otel-collector:4318/v1/logs\"\n" + base := "endpoint = \"http://otel-collector:4318/v1/logs\"\n" + + "allow_insecure_transport = true\n" if strings.Contains(tc.body, "endpoint =") { base = "" // the case sets its own; a second one is a TOML parse error } From 67282c3384d09f81e6ecec0f6d4d35d29108eba9 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Fri, 11 Sep 2026 11:31:03 +0530 Subject: [PATCH 17/20] Fix for Reject credentials in endpoint URL userinfo --- .../internal/analytics/publishers/otel.go | 31 +++++-- .../analytics/publishers/otel_test.go | 92 +++++++++++++++++++ .../policy-engine/internal/config/config.go | 7 ++ .../internal/config/otel_publisher_test.go | 49 ++++++++++ .../config/otel_publisher_toml_test.go | 1 + 5 files changed, 172 insertions(+), 8 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index caf78e687a..1a95ce7589 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -31,6 +31,7 @@ import ( "math/rand/v2" "net" "net/http" + "net/url" "os" "sort" "strconv" @@ -102,6 +103,8 @@ func ns(name string) string { return otelAttrNamespace + "." + name } type OTel struct { cfg config.OTelPublisherConfig client *http.Client + // endpoint with credential-bearing parts stripped, for logs + logEndpoint string queue chan *otelLogRecord @@ -136,7 +139,8 @@ func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { } o := &OTel{ - cfg: *cfg, + cfg: *cfg, + logEndpoint: endpointForLog(cfg.Endpoint), client: &http.Client{ Timeout: cfg.Timeout, Transport: &http.Transport{ @@ -164,12 +168,22 @@ func NewOTel(cfg *config.OTelPublisherConfig) (*OTel, error) { // Headers are deliberately omitted: they carry credentials. slog.Info("OTel analytics publisher started", - "endpoint", cfg.Endpoint, "batchSize", cfg.BatchSize, + "endpoint", o.logEndpoint, "batchSize", cfg.BatchSize, "flushInterval", cfg.FlushInterval, "queueCapacity", cfg.QueueCapacity, "onQueueFull", cfg.OnQueueFull) return o, nil } +// endpointForLog strips userinfo and the query string, either of which can carry +// an intake credential that would then be logged. +func endpointForLog(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "(unparseable endpoint)" + } + return u.Scheme + "://" + u.Host + u.Path +} + // buildOTelTLSConfig assembles the client TLS configuration. // // X25519MLKEM768 leads CurvePreferences per the repository's post-quantum @@ -423,7 +437,7 @@ func (o *OTel) export(batch []*otelLogRecord) { o.dropRecords(dropReasonSendFailed, len(records)) slog.Error("OTel publisher failed to export analytics batch; dropping records", "records", len(records), "attempts", o.cfg.MaxRetries+1, - "endpoint", o.cfg.Endpoint, "error", lastErr) + "endpoint", o.logEndpoint, "error", lastErr) } // otelPermanentExportError marks a response that must not be retried. @@ -878,7 +892,8 @@ func (o *OTel) appendAIAttributes(event *dto.Event, attrs *otelAttrs, route stri return } - if md, ok := event.Properties["aiMetadata"].(dto.AIMetadata); ok { + md, hasAIMetadata := event.Properties["aiMetadata"].(dto.AIMetadata) + if hasAIMetadata { attrs.str("gen_ai.provider.name", otelGenAIProviderName(md.VendorName)) attrs.str(ns("gen_ai.provider.template_name"), md.VendorName) // aitoken:modelid is the response model when the provider returns one, @@ -891,6 +906,10 @@ func (o *OTel) appendAIAttributes(event *dto.Event, attrs *otelAttrs, route stri case string: attrs.str(ns("gen_ai.cost.total"), cost) } + + if op := otelGenAIOperationName(route); op != "" { + attrs.str("gen_ai.operation.name", op) + } } attrs.anyStr("gen_ai.request.model", event.Properties[constants.RequestModelPropertyKey]) @@ -900,10 +919,6 @@ func (o *OTel) appendAIAttributes(event *dto.Event, attrs *otelAttrs, route stri attrs.i64(ns("gen_ai.usage.total_tokens"), int64(usage.TotalToken)) } - if op := otelGenAIOperationName(route); op != "" { - attrs.str("gen_ai.operation.name", op) - } - attrs.anyBool(ns("gen_ai.egress"), event.Properties["isEgress"]) attrs.anyBool(ns("guardrail.hit"), event.Properties[constants.GuardrailHitMetadataKey]) attrs.anyStr(ns("guardrail.name"), event.Properties[constants.GuardrailNameMetadataKey]) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index cbe082d70e..6dbffbdadd 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -321,6 +321,65 @@ func TestBuildRecordSuccessOmitsErrorAttributes(t *testing.T) { } } +// gen_ai.operation.name is the only AI attribute derived from the route rather +// than from a property the AI pipeline wrote, and otelGenAIOperationName +// substring-matches "/messages" and "/completions". Without a gate, ordinary REST +// routes containing those words emit a GenAI attribute and pollute GenAI +// dashboards with traffic that never reached a model. +func TestBuildRecordNonAIRouteOmitsGenAIOperation(t *testing.T) { + for _, route := range []string{ + "/notify/messages", // contains "/messages" + "/billing/completions", // contains "/completions" + "/v1/chat/completions", // the real LLM shape, on a plain REST API + "/inbox/messages/{id}", + } { + t.Run(route, func(t *testing.T) { + event := restEvent() + event.API.APIType = "RestApi" + event.Operation.APIResourceTemplate = route + // No aiMetadata: nothing in this request went near a model. + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if _, present := got["gen_ai.operation.name"]; present { + t.Fatalf("gen_ai.operation.name emitted for non-AI route %q (value %v)", + route, got["gen_ai.operation.name"]) + } + for _, k := range []string{"gen_ai.provider.name", "gen_ai.request.model", + "gen_ai.response.model", "gen_ai.usage.input_tokens"} { + if _, present := got[k]; present { + t.Errorf("%s emitted for a non-AI request", k) + } + } + }) + } +} + +// The gate keys off aiMetadata, so an AI request still gets the attribute — and +// still gets it for each recognised operation shape. +func TestBuildRecordAIRouteKeepsGenAIOperation(t *testing.T) { + for route, want := range map[string]string{ + "/v1/chat/completions": "chat", + "/v1/messages": "chat", + "/v1/embeddings": "embeddings", + } { + t.Run(route, func(t *testing.T) { + event := restEvent() + event.API.APIType = "LlmProxy" + event.Operation.APIResourceTemplate = route + event.Properties["aiMetadata"] = dto.AIMetadata{VendorName: "openai", Model: "gpt-4o"} + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["gen_ai.operation.name"] != want { + t.Fatalf("gen_ai.operation.name = %v, want %q", got["gen_ai.operation.name"], want) + } + }) + } +} + func TestBuildRecordGenAI(t *testing.T) { event := restEvent() event.API.APIType = "LlmProxy" @@ -1995,3 +2054,36 @@ func TestBackoffUnsetBaseUsesOneSecond(t *testing.T) { t.Errorf("backoff(1) = %s, want within [500ms, 1s)", got) } } + +// endpointForLog must drop anything that can carry a credential. +func TestEndpointForLog(t *testing.T) { + for raw, want := range map[string]string{ + "https://collector:4318/v1/logs": "https://collector:4318/v1/logs", + "https://collector:4318/v1/logs?api-key=s3cr3t": "https://collector:4318/v1/logs", + "https://svc:pw@collector:4318/v1/logs": "https://collector:4318/v1/logs", + "https://svc:pw@collector:4318/v1/logs?token=s3cr3t": "https://collector:4318/v1/logs", + "http://otel-collector:4318/v1/logs": "http://otel-collector:4318/v1/logs", + "://bad url": "(unparseable endpoint)", + } { + if got := endpointForLog(raw); got != want { + t.Errorf("endpointForLog(%q) = %q, want %q", raw, got, want) + } + } +} + +// The logged endpoint must never carry the query string, whatever the config holds. +func TestNewOTelStoresRedactedEndpoint(t *testing.T) { + cfg := testOTelConfig("https://collector:4318/v1/logs?api-key=s3cr3t") + o, err := NewOTel(&cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = o.Close(context.Background()) }() + + if strings.Contains(o.logEndpoint, "s3cr3t") || strings.Contains(o.logEndpoint, "api-key") { + t.Fatalf("logEndpoint leaks the credential: %q", o.logEndpoint) + } + if o.cfg.Endpoint != "https://collector:4318/v1/logs?api-key=s3cr3t" { + t.Errorf("cfg.Endpoint must keep the full URL for the request, got %q", o.cfg.Endpoint) + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 3e0bb6d757..98e50ed4c5 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -1571,6 +1571,13 @@ func validateOTelPublisherConfig(cfg OTelPublisherConfig) error { if err != nil || u.Host == "" { return fmt.Errorf("analytics.publishers.otel.endpoint must be a valid URL (e.g. http://otel-collector:4318/v1/logs), got %q", cfg.Endpoint) } + // Reject URL credentials to prevent endpoint leakage through logs and HTTP errors; + // use headers for authentication. + if u.User != nil { + return fmt.Errorf("analytics.publishers.otel.endpoint must not contain credentials in the "+ + "URL (user:password@%s); the endpoint is written to logs, so use "+ + "analytics.publishers.otel.headers to authenticate instead", u.Host) + } switch u.Scheme { case "https": case "http": diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go index 3f90ecc278..cfca53878b 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go @@ -50,6 +50,55 @@ func otelConfig(mutate func(*OTelPublisherConfig)) *Config { return cfg } +// Credentials in the endpoint's userinfo (CWE-532). The endpoint reaches the log +// three ways — the startup line, the export-failure line, and the HTTP client's +// own error, which embeds the URL and is wrapped and logged in turn — so the +// password would be on disk three times over. +func TestValidate_OTelPublisherRejectsCredentialsInEndpoint(t *testing.T) { + const password = "sup3rs3cr3t" + + t.Run("user and password are refused", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://svc:" + password + "@collector.example.com:4318/v1/logs" + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain credentials") + // The rejection must not reproduce the leak it exists to prevent. + assert.NotContains(t, err.Error(), password, "the error echoed the password back") + }) + + t.Run("a bare username is refused too", func(t *testing.T) { + // url.Parse sets User for "user@host" with no password, and a username is + // still a credential half worth keeping out of logs. + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://svc@collector.example.com:4318/v1/logs" + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain credentials") + }) + + t.Run("plaintext endpoints are covered as well", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "http://svc:" + password + "@collector.example.com:4318/v1/logs" + o.AllowInsecureTransport = true + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain credentials") + assert.NotContains(t, err.Error(), password) + }) + + t.Run("headers remain the supported way to authenticate", func(t *testing.T) { + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.Headers = map[string]string{"authorization": "Bearer " + password} + }) + assert.NoError(t, cfg.Validate()) + }) +} + // The plaintext gate, matching traffic_logging.http.allow_insecure_transport: // analytics records carry API keys and consumer identity, and every Headers // value is an intake credential put on the wire on each export. diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go index 42235cc32d..9f57afeef9 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_toml_test.go @@ -198,6 +198,7 @@ func TestLoad_OTelPublisher_InvalidTOMLValuesFailClosed(t *testing.T) { {"retries without backoff", "max_retries = 2\nretry_backoff = \"0s\"", "retry_backoff must be positive"}, {"non-http scheme", `endpoint = "ftp://collector:4318/v1/logs"`, "must be https (or http with allow_insecure_transport)"}, {"plaintext without opt-in", `endpoint = "http://collector.example.com:4318/v1/logs"`, "allow_insecure_transport is false"}, + {"credentials in the endpoint URL", `endpoint = "https://svc:pw@collector.example.com:4318/v1/logs"`, "must not contain credentials"}, {"missing ca file", `[analytics.publishers.otel.tls]` + "\n" + `ca_file = "/nonexistent/ca.pem"`, "cannot read ca_file"}, } for _, tc := range cases { From 150073ce7780c201bf3c2b330f59b6aaf2e1b45d Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Fri, 11 Sep 2026 11:42:31 +0530 Subject: [PATCH 18/20] Adding validation to reject permissive client private-key permissions --- .../policy-engine/internal/config/config.go | 27 +++++++- .../internal/config/otel_publisher_test.go | 61 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 98e50ed4c5..a4acf959a8 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -1571,7 +1571,7 @@ func validateOTelPublisherConfig(cfg OTelPublisherConfig) error { if err != nil || u.Host == "" { return fmt.Errorf("analytics.publishers.otel.endpoint must be a valid URL (e.g. http://otel-collector:4318/v1/logs), got %q", cfg.Endpoint) } - // Reject URL credentials to prevent endpoint leakage through logs and HTTP errors; + // Reject URL credentials to prevent endpoint leakage through logs and HTTP errors; // use headers for authentication. if u.User != nil { return fmt.Errorf("analytics.publishers.otel.endpoint must not contain credentials in the "+ @@ -1651,6 +1651,27 @@ func validateOTelPublisherConfig(cfg OTelPublisherConfig) error { return nil } +// tlsKeyPermMask is the set of permission bits that must be clear on a TLS +// private key: anything readable by group or other. +const tlsKeyPermMask os.FileMode = 0o077 + +// verifyTLSKeyPerms fails when a TLS private key is readable by group or other +// (GO-AUTH-018). Shared so traffic_logging.http.tls can adopt the same check. +func verifyTLSKeyPerms(field, path string) error { + fi, err := os.Stat(path) + if err != nil { + return fmt.Errorf("cannot stat %s %q: %w", field, path, err) + } + if fi.IsDir() { + return fmt.Errorf("%s %q is a directory, not a private key file", field, path) + } + if perm := fi.Mode().Perm(); perm&tlsKeyPermMask != 0 { + return fmt.Errorf("%s %q has permissions %#o, which allow group/other access to a private "+ + "key; fix it with `chmod 600 %s` and restart", field, path, perm, path) + } + return nil +} + // validateOTelTLS checks that any referenced TLS material exists and parses, so a // bad path fails at startup rather than on the first export. func validateOTelTLS(cfg OTelTLSConfig, host string) error { @@ -1672,6 +1693,10 @@ func validateOTelTLS(cfg OTelTLSConfig, host string) error { return fmt.Errorf("cert_file and key_file must be set together for mTLS (one is set, the other is not)") } if cfg.CertFile != "" { + // Permissions first: a key anyone can read is a finding whether or not it parses. + if err := verifyTLSKeyPerms("key_file", cfg.KeyFile); err != nil { + return err + } if _, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile); err != nil { return fmt.Errorf("cannot load client certificate/key pair: %w", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go index cfca53878b..0a248c11b9 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/otel_publisher_test.go @@ -24,6 +24,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" "math/big" "os" "path/filepath" @@ -372,3 +373,63 @@ func TestValidate_OTelPublisherRetry(t *testing.T) { } }) } + +// GO-AUTH-018: a group/other-readable client key must abort startup, not warn. +func TestValidate_OTelPublisherRejectsPermissiveKey(t *testing.T) { + for _, mode := range []os.FileMode{0o644, 0o640, 0o604, 0o660, 0o666, 0o777} { + t.Run(fmt.Sprintf("mode_%#o", mode), func(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + require.NoError(t, os.Chmod(keyPath, mode)) + + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.TLS.CertFile = certPath + o.TLS.KeyFile = keyPath + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "allow group/other access to a private key") + }) + } + + for _, mode := range []os.FileMode{0o600, 0o400} { + t.Run(fmt.Sprintf("owner_only_%#o_passes", mode), func(t *testing.T) { + certPath, keyPath := writeSelfSignedPair(t) + require.NoError(t, os.Chmod(keyPath, mode)) + + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.TLS.CertFile = certPath + o.TLS.KeyFile = keyPath + }) + assert.NoError(t, cfg.Validate()) + }) + } +} + +// Permissions are checked before the key is parsed, so a readable key is reported +// as a permissions problem rather than a parse failure. +func TestValidate_OTelPublisherKeyPermsCheckedBeforeParse(t *testing.T) { + certPath, _ := writeSelfSignedPair(t) + badKey := filepath.Join(t.TempDir(), "garbage.key") + require.NoError(t, os.WriteFile(badKey, []byte("not a key"), 0o644)) + + cfg := otelConfig(func(o *OTelPublisherConfig) { + o.Endpoint = "https://collector.example.com:4318/v1/logs" + o.TLS.CertFile = certPath + o.TLS.KeyFile = badKey + }) + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "allow group/other access to a private key") + assert.NotContains(t, err.Error(), "cannot load client certificate/key pair") +} + +func TestVerifyTLSKeyPerms(t *testing.T) { + _, keyPath := writeSelfSignedPair(t) + assert.NoError(t, verifyTLSKeyPerms("key_file", keyPath)) + + assert.ErrorContains(t, verifyTLSKeyPerms("key_file", filepath.Join(t.TempDir(), "absent.key")), + "cannot stat") + assert.ErrorContains(t, verifyTLSKeyPerms("key_file", t.TempDir()), "is a directory") +} From a5a7d4d2ed2875abb696995a5f16eb4f35e91f24 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Fri, 11 Sep 2026 16:18:41 +0530 Subject: [PATCH 19/20] Add helm config mapping and minor mapping issues --- .../internal/analytics/publishers/otel.go | 17 ++-- .../analytics/publishers/otel_test.go | 77 ++++++++++++++++ .../templates/gateway/gateway-config.yaml | 88 +++++++++++++++++++ 3 files changed, 171 insertions(+), 11 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go index 1a95ce7589..c3b029a379 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel.go @@ -948,20 +948,15 @@ func (o *OTel) appendMCPAttributes(event *dto.Event, attrs *otelAttrs) { attrs.anyStr("mcp.session.id", take("sessionId")) attrs.anyStr("jsonrpc.request.id", take("jsonRpcId")) - // Tools and prompts are named (params.name); a resource is addressed by URI - // (params.uri), which the analytics policy extracts into its own field. - capabilityName, _ := take("capabilityName").(string) - resourceURI := take("resourceUri") - // capability itself is not emitted: which attribute below is populated says - // it, and it is derivable from mcp.method.name's prefix. Taking it still - // claims it, so the sweep does not put it back. + // Emit tool/prompt names or resource URIs only for recognized capabilities; + // leave unrecognized fields for the sweep. switch capability, _ := take("capability").(string); capability { case "TOOL": - attrs.str("gen_ai.tool.name", capabilityName) - case "RESOURCE": - attrs.anyStr("mcp.resource.uri", resourceURI) + attrs.anyStr("gen_ai.tool.name", take("capabilityName")) case "PROMPT": - attrs.str("gen_ai.prompt.name", capabilityName) + attrs.anyStr("gen_ai.prompt.name", take("capabilityName")) + case "RESOURCE": + attrs.anyStr("mcp.resource.uri", take("resourceUri")) } // A JSON-RPC error code is a string in rpc.response.status_code. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go index 6dbffbdadd..8ad5757b87 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/otel_test.go @@ -2087,3 +2087,80 @@ func TestNewOTelStoresRedactedEndpoint(t *testing.T) { t.Errorf("cfg.Endpoint must keep the full URL for the request, got %q", o.cfg.Endpoint) } } + +// An unrecognized capability emits no capability-target attribute, so +// capabilityName/resourceUri must stay unclaimed and reach the sweep rather than +// being dropped. Moesif already receives capabilityName for these methods. +func TestMCPUnrecognizedCapabilityFallsToSweep(t *testing.T) { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "jsonRpcMethod": "completion/complete", + "capability": "", // deriveMCPCapability returns "" for a non tools/prompts/resources prefix + "capabilityName": "greet", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["wso2.mcp.capability_name"] != "greet" { + t.Errorf("wso2.mcp.capability_name = %v, want greet", got["wso2.mcp.capability_name"]) + } + for _, k := range []string{"gen_ai.tool.name", "gen_ai.prompt.name", "mcp.resource.uri"} { + if _, present := got[k]; present { + t.Errorf("%s must not be emitted for an unrecognized capability", k) + } + } +} + +// The sweep must not duplicate a value a branch already emitted under its +// curated name. +func TestMCPRecognizedCapabilityIsNotAlsoSwept(t *testing.T) { + cases := []struct { + capability, key, curated string + }{ + {"TOOL", "capabilityName", "gen_ai.tool.name"}, + {"PROMPT", "capabilityName", "gen_ai.prompt.name"}, + {"RESOURCE", "resourceUri", "mcp.resource.uri"}, + } + for _, tc := range cases { + t.Run(tc.capability, func(t *testing.T) { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "capability": tc.capability, + tc.key: "value-x", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got[tc.curated] != "value-x" { + t.Errorf("%s = %v, want value-x", tc.curated, got[tc.curated]) + } + swept := ns("mcp." + otelSnakeCase(tc.key)) + if _, present := got[swept]; present { + t.Errorf("%s was swept as well as emitted under %s", swept, tc.curated) + } + }) + } +} + +// The unused sibling key stays unclaimed, so a request carrying both still +// reports the one its capability does not use. +func TestMCPUnusedSiblingKeyIsSwept(t *testing.T) { + event := restEvent() + event.Properties["mcpAnalytics"] = map[string]interface{}{ + "capability": "TOOL", + "capabilityName": "search_docs", + "resourceUri": "file:///unexpected.md", + } + + o := &OTel{cfg: testOTelConfig("http://collector/v1/logs")} + got := attrMap(t, o.buildRecord(event)) + + if got["gen_ai.tool.name"] != "search_docs" { + t.Errorf("gen_ai.tool.name = %v, want search_docs", got["gen_ai.tool.name"]) + } + if got["wso2.mcp.resource_uri"] != "file:///unexpected.md" { + t.Errorf("wso2.mcp.resource_uri = %v, want the unused sibling to be swept", got["wso2.mcp.resource_uri"]) + } +} diff --git a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml index 26bb9fcb58..ca125e2dd7 100644 --- a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml +++ b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml @@ -370,12 +370,100 @@ data: {{- end }} {{- if .Values.gateway.config.analytics.publishers }} {{- range $name, $publisher := .Values.gateway.config.analytics.publishers }} + {{- if eq $name "otel" }} + {{- with $publisher }} + {{- /* headers carry the intake credential and config.toml renders into a ConfigMap, not a Secret — require an {{ env }}/{{ file }} token, as traffic_logging.http.auth does. */ -}} + {{- $allowPlain := false -}} + {{- if hasKey . "allow_plaintext_credentials" -}} + {{- $allowPlain = eq (toString .allow_plaintext_credentials) "true" -}} + {{- end -}} + {{- if not $allowPlain -}} + {{- $suffixed := "^[^{}]*\\{\\{-? *(env|file) [^{}]*\\}\\}$" -}} + {{- range $header, $value := .headers -}} + {{- if not (regexMatch $suffixed (trim (toString $value))) -}} + {{- fail (printf "gateway.config.analytics.publishers.otel.headers.%s holds a literal value, which would be written in plaintext into the gateway ConfigMap. Supply it as an interpolation token resolved at runtime instead, e.g. '{{ env \"APIP_GW_OTEL_INTAKE_TOKEN\" }}' or '{{ file \"/secrets/gateway-runtime/otel-token\" }}' (allowed source dirs: /etc/gateway-runtime, /secrets/gateway-runtime). Set gateway.config.analytics.publishers.otel.allow_plaintext_credentials=true only for local development." $header) -}} + {{- end -}} + {{- end -}} + {{- end }} + [analytics.publishers.otel] + {{- if .endpoint }} + endpoint = {{ .endpoint | quote }} + {{- end }} + {{- if kindIs "bool" .allow_insecure_transport }} + allow_insecure_transport = {{ .allow_insecure_transport }} + {{- end }} + {{- if .service_name }} + service_name = {{ .service_name | quote }} + {{- end }} + {{- if .service_version }} + service_version = {{ .service_version | quote }} + {{- end }} + {{- if not (kindIs "invalid" .batch_size) }} + batch_size = {{ .batch_size | int64 }} + {{- end }} + {{- if not (kindIs "invalid" .flush_interval) }} + flush_interval = {{ .flush_interval | quote }} + {{- end }} + {{- if not (kindIs "invalid" .queue_capacity) }} + queue_capacity = {{ .queue_capacity | int64 }} + {{- end }} + {{- if .on_queue_full }} + on_queue_full = {{ .on_queue_full | quote }} + {{- end }} + {{- if not (kindIs "invalid" .timeout) }} + timeout = {{ .timeout | quote }} + {{- end }} + {{- if not (kindIs "invalid" .max_retries) }} + max_retries = {{ .max_retries | int64 }} + {{- end }} + {{- if not (kindIs "invalid" .retry_backoff) }} + retry_backoff = {{ .retry_backoff | quote }} + {{- end }} + {{- if not (kindIs "invalid" .retry_abort_queue_ratio) }} + retry_abort_queue_ratio = {{ .retry_abort_queue_ratio }} + {{- end }} + {{- if .compression }} + compression = {{ .compression | quote }} + {{- end }} + {{- with .headers }} + + [analytics.publishers.otel.headers] + {{- range $key, $value := . }} + {{ $key | quote }} = {{ $value | quote }} + {{- end }} + {{- end }} + {{- with .resource_attributes }} + + [analytics.publishers.otel.resource_attributes] + {{- range $key, $value := . }} + {{ $key | quote }} = {{ $value | quote }} + {{- end }} + {{- end }} + {{- with .tls }} + + [analytics.publishers.otel.tls] + {{- if .ca_file }} + ca_file = {{ .ca_file | quote }} + {{- end }} + {{- if .cert_file }} + cert_file = {{ .cert_file | quote }} + {{- end }} + {{- if .key_file }} + key_file = {{ .key_file | quote }} + {{- end }} + {{- if kindIs "bool" .insecure_skip_verify }} + insecure_skip_verify = {{ .insecure_skip_verify }} + {{- end }} + {{- end }} + {{- end }} + {{- else }} [analytics.publishers.{{ $name }}] {{- range $key, $value := $publisher }} {{ $key }} = {{ kindIs "string" $value | ternary ($value | quote) $value }} {{- end }} {{- end }} {{- end }} + {{- end }} {{- if .Values.gateway.config.analytics.grpc_event_server }} [analytics.grpc_event_server] host = "127.0.0.1" From b2ad860a83cf4a763e2a9f36b5aa020d31df4df6 Mon Sep 17 00:00:00 2001 From: Osura Viduranga Date: Fri, 11 Sep 2026 18:01:20 +0530 Subject: [PATCH 20/20] Sanitize OriginalPath before exporting --- .../internal/analytics/analytics.go | 4 +- .../internal/analytics/analytics_test.go | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 041fbe9184..2a5cbe0629 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -349,12 +349,14 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E // Strip the query once at the source since it is shared across publishers and may contain credentials. requestPath, _, _ := strings.Cut(request.GetPath(), "?") + // OriginalPath is the pre-rewrite :path, so it carries the client's query too. + originalPath, _, _ := strings.Cut(request.GetOriginalPath(), "?") // Prepare operation operation := dto.Operation{} // operation.APIResourceTemplate = keyValuePairsFromMetadata[APIResourceTemplateKey] if request != nil { - operation.APIResourceTemplate = logEntry.GetRequest().GetOriginalPath() + operation.APIResourceTemplate = originalPath operation.APIMethod = logEntry.Request.GetRequestMethod().String() } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 5157278fb1..4f4ec64eb5 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -1357,6 +1357,46 @@ func TestPrepareAnalyticEvent_RequestPathDropsQueryString(t *testing.T) { } } +// APIResourceTemplate comes from Envoy's original_path (the pre-rewrite :path, +// which every proxied route here produces via context-path stripping), so it +// carries the client's query string just as Path does. It reaches http.route on +// the OTel publisher, the Moesif uri and the traffic-log path, so it is cut at +// this same single point rather than at each of them. +func TestPrepareAnalyticEvent_APIResourceTemplateDropsQueryString(t *testing.T) { + for name, tc := range map[string]struct { + originalPath string + path string + want string + }{ + "no query": {"/petstore/pet/12345", "/pet/12345", "/petstore/pet/12345"}, + "single param": {"/petstore/pet/12345?apikey=secret", "/pet/12345", "/petstore/pet/12345"}, + "multiple params": {"/search?q=cat&token=abc123", "/search", "/search"}, + "empty query": {"/petstore/pet?", "/pet", "/petstore/pet"}, + "query only": {"?apikey=secret", "/pet", ""}, + "absent original path": {"", "/pet/12345", ""}, + "encoded question mark": {"/pet/a%3Fb", "/a%3Fb", "/pet/a%3Fb"}, + // The reviewer's case: the rewritten Path is clean while the pre-rewrite + // OriginalPath still carries the credential. + "query only on original path": {"/petstore/pet?apikey=secret", "/pet", "/petstore/pet"}, + } { + t.Run(name, func(t *testing.T) { + logEntry := createLogEntryWithMetadata(map[string]string{}) + logEntry.Request.OriginalPath = tc.originalPath + logEntry.Request.Path = tc.path + + event := NewAnalytics(&config.Config{}).prepareAnalyticEvent(logEntry) + + if got := event.Operation.APIResourceTemplate; got != tc.want { + t.Errorf("APIResourceTemplate = %q, want %q", got, tc.want) + } + if strings.Contains(event.Operation.APIResourceTemplate, "?") { + t.Errorf("APIResourceTemplate %q still carries a query string", + event.Operation.APIResourceTemplate) + } + }) + } +} + // Target.Destination reaches three consumers — the OTel publisher's // wso2.upstream.destination, the traffic log's destination field, and the // target.destination policy expression — so the query string is stripped at this