From aa35a8badf7d9e58e3d6d55fa51feae6f5e0d37e Mon Sep 17 00:00:00 2001 From: Yehor Nesterov Date: Sun, 8 Feb 2026 10:56:43 +0100 Subject: [PATCH 1/6] test: add reproduction for Prometheus exemplar 128-rune limit breach --- exporters/prometheus/exporter_test.go | 28 +++++++++++++++++++ .../testdata/exemplar_dropped_attrs_limit.txt | 10 +++++++ 2 files changed, 38 insertions(+) create mode 100644 exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt diff --git a/exporters/prometheus/exporter_test.go b/exporters/prometheus/exporter_test.go index 5b4d80c58ec..58286ffd808 100644 --- a/exporters/prometheus/exporter_test.go +++ b/exporters/prometheus/exporter_test.go @@ -11,6 +11,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "sync" "testing" "time" @@ -678,6 +679,25 @@ func TestPrometheusExporter(t *testing.T) { counter.Add(ctx, 5, otelmetric.WithAttributeSet(attrs2)) }, }, + { + name: "exemplar dropped attributes exceed 128 rune limit", + expectedFile: "testdata/exemplar_dropped_attrs_limit.txt", + recordMetrics: func(ctx context.Context, meter otelmetric.Meter) { + sc := trace.NewSpanContext(trace.SpanContextConfig{ + SpanID: [8]byte{0x01}, + TraceID: [16]byte{0x01}, + TraceFlags: trace.FlagsSampled, + }) + ctx = trace.ContextWithSpanContext(ctx, sc) + + hist, err:= meter.Float64Histogram("test_dropped_attrs_limit") + require.NoError(t, err) + longVal := strings.Repeat("A", 80) + hist.Record(ctx, 1.0, otelmetric.WithAttributes( + attribute.String("long_attribute", longVal), + )) + }, + }, } for _, tc := range testCases { @@ -720,6 +740,14 @@ func TestPrometheusExporter(t *testing.T) { MaxSize: 10, }}, ), + metric.NewView( + metric.Instrument{Name: "test_dropped_attrs_limit"}, + metric.Stream{ + AttributeFilter: func(kv attribute.KeyValue) bool { + return kv.Key != "long_attribute" + }, + }, + ), ), ) meter := provider.Meter( diff --git a/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt b/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt new file mode 100644 index 00000000000..8afa1188094 --- /dev/null +++ b/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt @@ -0,0 +1,10 @@ +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="unknown_service:something",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.40.0"} 1.0 +# HELP test_dropped_attrs_limit +# TYPE test_dropped_attrs_limit histogram +test_dropped_attrs_limit_bucket{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version="",le="64.0"} 1 # {span_id="0100000000000000",trace_id="01000000000000000000000000000000"} 1.0 1770543252 +test_dropped_attrs_limit_bucket{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version="",le="+Inf"} 1 +test_dropped_attrs_limit_sum{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version=""} 1.0 +test_dropped_attrs_limit_count{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version=""} 1 +# EOF From b58824d4daf31010d1c531c45778857114d2bce4 Mon Sep 17 00:00:00 2001 From: Yehor Nesterov Date: Thu, 12 Feb 2026 11:19:58 +0100 Subject: [PATCH 2/6] prevent exemplar drop when filtered labels exceed 128 runes --- exporters/prometheus/exporter.go | 32 ++++++++++++--- exporters/prometheus/exporter_test.go | 39 ++++++++++--------- exporters/prometheus/go.mod | 2 +- .../testdata/exemplar_dropped_attrs_limit.txt | 4 +- 4 files changed, 49 insertions(+), 28 deletions(-) diff --git a/exporters/prometheus/exporter.go b/exporters/prometheus/exporter.go index 0f2259af381..d220d0556c0 100644 --- a/exporters/prometheus/exporter.go +++ b/exporters/prometheus/exporter.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "sync" + "unicode/utf8" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" @@ -40,6 +41,8 @@ const ( // Registerer, rather than round-tripping them through the bridge and // exporter. bridgeScopeName = "go.opentelemetry.io/contrib/bridges/prometheus" + + exemplarRuneLimit = 128 ) var metricsPool = sync.Pool{ @@ -304,6 +307,7 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { addSumMetric(ch, v, m, name, kv, c.labelNamer, c.inst, ctx) case metricdata.Sum[float64]: addSumMetric(ch, v, m, name, kv, c.labelNamer, c.inst, ctx) + case metricdata.Gauge[int64]: addGaugeMetric(ch, v, m, name, kv, c.labelNamer, c.inst, ctx) case metricdata.Gauge[float64]: @@ -780,15 +784,23 @@ func addExemplars[N int64 | float64]( return m } promExemplars := make([]prometheus.Exemplar, len(exemplars)) + traceKeyLen := utf8.RuneCountInString(otlptranslator.ExemplarTraceIDKey) + spanKeyLen := utf8.RuneCountInString(otlptranslator.ExemplarSpanIDKey) + for i, exemplar := range exemplars { - labels, err := attributesToLabels(exemplar.FilteredAttributes, labelNamer) + traceId := hex.EncodeToString(exemplar.TraceID) + spanId := hex.EncodeToString(exemplar.SpanID) + + remaingSpace := exemplarRuneLimit - traceKeyLen - utf8.RuneCountInString(traceId) - spanKeyLen - utf8.RuneCountInString(spanId) + + labels, err := attributesToLabels(exemplar.FilteredAttributes, labelNamer, remaingSpace) if err != nil { otel.Handle(err) return m } // Overwrite any existing trace ID or span ID attributes - labels[otlptranslator.ExemplarTraceIDKey] = hex.EncodeToString(exemplar.TraceID) - labels[otlptranslator.ExemplarSpanIDKey] = hex.EncodeToString(exemplar.SpanID) + labels[otlptranslator.ExemplarTraceIDKey] = traceId + labels[otlptranslator.ExemplarSpanIDKey] = spanId promExemplars[i] = prometheus.Exemplar{ Value: float64(exemplar.Value), Timestamp: exemplar.Time, @@ -805,14 +817,22 @@ func addExemplars[N int64 | float64]( return metricWithExemplar } -func attributesToLabels(attrs []attribute.KeyValue, labelNamer otlptranslator.LabelNamer) (prometheus.Labels, error) { - labels := make(map[string]string) +func attributesToLabels(attrs []attribute.KeyValue, labelNamer otlptranslator.LabelNamer, remainingSpace int) (prometheus.Labels, error) { + labels := make(map[string]string, len(attrs)+2) + if remainingSpace <= 0 { + return labels, nil + } for _, attr := range attrs { name, err := labelNamer.Build(string(attr.Key)) if err != nil { return nil, err } - labels[name] = attr.Value.Emit() + val := attr.Value.Emit() + attrLength := utf8.RuneCountInString(val) + utf8.RuneCountInString(name) + if attrLength <= remainingSpace { + labels[name] = val + remainingSpace -= attrLength + } } return labels, nil } diff --git a/exporters/prometheus/exporter_test.go b/exporters/prometheus/exporter_test.go index 58286ffd808..f853380ba74 100644 --- a/exporters/prometheus/exporter_test.go +++ b/exporters/prometheus/exporter_test.go @@ -679,25 +679,6 @@ func TestPrometheusExporter(t *testing.T) { counter.Add(ctx, 5, otelmetric.WithAttributeSet(attrs2)) }, }, - { - name: "exemplar dropped attributes exceed 128 rune limit", - expectedFile: "testdata/exemplar_dropped_attrs_limit.txt", - recordMetrics: func(ctx context.Context, meter otelmetric.Meter) { - sc := trace.NewSpanContext(trace.SpanContextConfig{ - SpanID: [8]byte{0x01}, - TraceID: [16]byte{0x01}, - TraceFlags: trace.FlagsSampled, - }) - ctx = trace.ContextWithSpanContext(ctx, sc) - - hist, err:= meter.Float64Histogram("test_dropped_attrs_limit") - require.NoError(t, err) - longVal := strings.Repeat("A", 80) - hist.Record(ctx, 1.0, otelmetric.WithAttributes( - attribute.String("long_attribute", longVal), - )) - }, - }, } for _, tc := range testCases { @@ -1319,6 +1300,26 @@ func TestExemplars(t *testing.T) { expectedLabels: expectedNonEscapedLabels, strategy: otlptranslator.NoTranslation, }, + { + name: "exemplar overflow does not drop exemplar", + recordMetrics: func(ctx context.Context, meter otelmetric.Meter) { + hist, err := meter.Float64Histogram("exponential_histogram") + require.NoError(t, err) + + // Create attributes that exceed 128-rune limit after accounting for trace_id/span_id + // trace_id (32) + span_id (16) + "=" (1) = 49 characters + // Remaining space: 128 - 49 = 79 characters + // longVal (80 chars) + "=" = 81 chars ✓ + // 81 chars > 79, so long_value should be truncated + + longVal := strings.Repeat("B", 80) // 80 chars + + hist.Record(ctx, 0, otelmetric.WithAttributes( + attribute.String("long_value", longVal), + ), attrsOpt) + }, + expectedLabels: expectedEscapedLabels, + }, } { t.Run(tc.name, func(t *testing.T) { // initialize registry exporter diff --git a/exporters/prometheus/go.mod b/exporters/prometheus/go.mod index be4f9409b3f..3a7d034cfa1 100644 --- a/exporters/prometheus/go.mod +++ b/exporters/prometheus/go.mod @@ -9,6 +9,7 @@ retract v0.59.0 require ( github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 + github.com/prometheus/common v0.67.5 github.com/prometheus/otlptranslator v1.0.0 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.39.0 @@ -29,7 +30,6 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect diff --git a/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt b/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt index 8afa1188094..fa1dc2b14ff 100644 --- a/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt +++ b/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt @@ -1,9 +1,9 @@ # HELP target_info Target metadata # TYPE target_info gauge -target_info{service_name="unknown_service:something",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.40.0"} 1.0 +target_info{service_name="prometheus_test",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 # HELP test_dropped_attrs_limit # TYPE test_dropped_attrs_limit histogram -test_dropped_attrs_limit_bucket{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version="",le="64.0"} 1 # {span_id="0100000000000000",trace_id="01000000000000000000000000000000"} 1.0 1770543252 +test_dropped_attrs_limit_bucket{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version="",le="64.0"} 1 # {span_id="0100000000000000",trace_id="01000000000000000000000000000000"} 1.0 1.7705432527556e+09 test_dropped_attrs_limit_bucket{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version="",le="+Inf"} 1 test_dropped_attrs_limit_sum{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version=""} 1.0 test_dropped_attrs_limit_count{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version=""} 1 From c3bd99fc70f021c23ac1c4d2eda760739779ab4f Mon Sep 17 00:00:00 2001 From: Yehor Nesterov Date: Thu, 12 Feb 2026 11:34:01 +0100 Subject: [PATCH 3/6] update changelog --- CHANGELOG.md | 1 + exporters/prometheus/exporter.go | 12 ++++++++++-- exporters/prometheus/go.mod | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebe62c6988a..3575ef110fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Fix `SetAttributes` on `Record` in `go.opentelemetry.io/otel/sdk/log` to not log that attributes are dropped when they are actually not dropped. (#7662) - `WithHostID` detector in `go.opentelemetry.io/otel/sdk/resource` to use full path for `ioreg` command on Darwin (macOS). (#7818) - Fix missing `request.GetBody` in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp` to correctly handle HTTP2 GOAWAY frame. (#7794) +- Fix dropping entire exemplar when filtered attributes exceed the 128-rune limit.(#7883) ### Deprecated diff --git a/exporters/prometheus/exporter.go b/exporters/prometheus/exporter.go index d220d0556c0..625d766178c 100644 --- a/exporters/prometheus/exporter.go +++ b/exporters/prometheus/exporter.go @@ -791,7 +791,11 @@ func addExemplars[N int64 | float64]( traceId := hex.EncodeToString(exemplar.TraceID) spanId := hex.EncodeToString(exemplar.SpanID) - remaingSpace := exemplarRuneLimit - traceKeyLen - utf8.RuneCountInString(traceId) - spanKeyLen - utf8.RuneCountInString(spanId) + remaingSpace := exemplarRuneLimit - traceKeyLen - utf8.RuneCountInString( + traceId, + ) - spanKeyLen - utf8.RuneCountInString( + spanId, + ) labels, err := attributesToLabels(exemplar.FilteredAttributes, labelNamer, remaingSpace) if err != nil { @@ -817,7 +821,11 @@ func addExemplars[N int64 | float64]( return metricWithExemplar } -func attributesToLabels(attrs []attribute.KeyValue, labelNamer otlptranslator.LabelNamer, remainingSpace int) (prometheus.Labels, error) { +func attributesToLabels( + attrs []attribute.KeyValue, + labelNamer otlptranslator.LabelNamer, + remainingSpace int, +) (prometheus.Labels, error) { labels := make(map[string]string, len(attrs)+2) if remainingSpace <= 0 { return labels, nil diff --git a/exporters/prometheus/go.mod b/exporters/prometheus/go.mod index 3a7d034cfa1..be4f9409b3f 100644 --- a/exporters/prometheus/go.mod +++ b/exporters/prometheus/go.mod @@ -9,7 +9,6 @@ retract v0.59.0 require ( github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 - github.com/prometheus/common v0.67.5 github.com/prometheus/otlptranslator v1.0.0 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.39.0 @@ -30,6 +29,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect From c630174f36c17a44910fc246703795c7ea945741 Mon Sep 17 00:00:00 2001 From: Yehor Nesterov Date: Thu, 12 Feb 2026 11:40:02 +0100 Subject: [PATCH 4/6] removed unused test data --- .../testdata/exemplar_dropped_attrs_limit.txt | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt diff --git a/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt b/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt deleted file mode 100644 index fa1dc2b14ff..00000000000 --- a/exporters/prometheus/testdata/exemplar_dropped_attrs_limit.txt +++ /dev/null @@ -1,10 +0,0 @@ -# HELP target_info Target metadata -# TYPE target_info gauge -target_info{service_name="prometheus_test",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 -# HELP test_dropped_attrs_limit -# TYPE test_dropped_attrs_limit histogram -test_dropped_attrs_limit_bucket{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version="",le="64.0"} 1 # {span_id="0100000000000000",trace_id="01000000000000000000000000000000"} 1.0 1.7705432527556e+09 -test_dropped_attrs_limit_bucket{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version="",le="+Inf"} 1 -test_dropped_attrs_limit_sum{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version=""} 1.0 -test_dropped_attrs_limit_count{otel_scope_name="go.opentelemetry.io/contrib/examples/prometheus",otel_scope_url="",otel_scope_version=""} 1 -# EOF From e0e3d5ace2854165645ebfd0a1e6a324f8957811 Mon Sep 17 00:00:00 2001 From: Yehor Nesterov Date: Thu, 12 Feb 2026 15:07:14 +0100 Subject: [PATCH 5/6] move changelog entry under ## [Unreleased] --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b601ca6144..31840dcc55d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] + +### Fixed + +- Fix dropping entire exemplar when filtered attributes exceed the 128-rune limit.(#7883) + From b2e1eefab0e8ff6b05910660a6f9a56c611de32e Mon Sep 17 00:00:00 2001 From: Yehor Nesterov Date: Sat, 14 Feb 2026 13:07:19 +0100 Subject: [PATCH 6/6] update changelog --- CHANGELOG.md | 9 +++++++-- exporters/prometheus/exporter.go | 7 +++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31840dcc55d..d8530a763af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +This release is the last to support [Go 1.24]. +The next release will require at least [Go 1.25]. + +### Added + +- Support testing of [Go 1.26]. (#7902) ### Fixed @@ -47,8 +53,6 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Fix `SetAttributes` on `Record` in `go.opentelemetry.io/otel/sdk/log` to not log that attributes are dropped when they are actually not dropped. (#7662) - Fix missing `request.GetBody` in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp` to correctly handle HTTP/2 `GOAWAY` frame. (#7794) - `WithHostID` detector in `go.opentelemetry.io/otel/sdk/resource` to use full path for `ioreg` command on Darwin (macOS). (#7818) -- Fix missing `request.GetBody` in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp` to correctly handle HTTP2 GOAWAY frame. (#7794) -- Fix dropping entire exemplar when filtered attributes exceed the 128-rune limit.(#7883) ### Deprecated @@ -3642,6 +3646,7 @@ It contains api and sdk for trace and meter. +[Go 1.26]: https://go.dev/doc/go1.26 [Go 1.25]: https://go.dev/doc/go1.25 [Go 1.24]: https://go.dev/doc/go1.24 [Go 1.23]: https://go.dev/doc/go1.23 diff --git a/exporters/prometheus/exporter.go b/exporters/prometheus/exporter.go index 625d766178c..ac6c37f065b 100644 --- a/exporters/prometheus/exporter.go +++ b/exporters/prometheus/exporter.go @@ -45,6 +45,11 @@ const ( exemplarRuneLimit = 128 ) +var ( + traceKeyLen = utf8.RuneCountInString(otlptranslator.ExemplarTraceIDKey) + spanKeyLen = utf8.RuneCountInString(otlptranslator.ExemplarSpanIDKey) +) + var metricsPool = sync.Pool{ New: func() any { return &metricdata.ResourceMetrics{} @@ -784,8 +789,6 @@ func addExemplars[N int64 | float64]( return m } promExemplars := make([]prometheus.Exemplar, len(exemplars)) - traceKeyLen := utf8.RuneCountInString(otlptranslator.ExemplarTraceIDKey) - spanKeyLen := utf8.RuneCountInString(otlptranslator.ExemplarSpanIDKey) for i, exemplar := range exemplars { traceId := hex.EncodeToString(exemplar.TraceID)