Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
c924eb3
Fix comment about how to only run unit tests
AaronRM Jan 15, 2025
b2b4518
Remove extraneous lines from test_with_gzip_compression unit test
AaronRM Jan 15, 2025
9364bfa
Add retry_with_exponential_backoff method; Use in tonic logs client
AaronRM Feb 27, 2025
f848dd8
Add tests and comments to retry.rs
AaronRM Feb 27, 2025
63df3f8
Move retry to opentelemetry-sdk (WIP)
AaronRM Mar 13, 2025
f3a319f
Merge branch 'main' into aaronm-retry
AaronRM Mar 13, 2025
c5fd833
Fix build warning
AaronRM Mar 27, 2025
9c11248
Scope retry to just logs+tonic
AaronRM Mar 27, 2025
3a15eba
clean up after rebase
scottgerring Aug 11, 2025
d868a0d
migrate to the async runtime abstraction
scottgerring Aug 12, 2025
b4e3c2d
add retry to traces
scottgerring Aug 12, 2025
2e0bf3e
add retry to metrics
scottgerring Aug 12, 2025
5e49f2d
handle failure hints
scottgerring Aug 12, 2025
861343a
simplify retry interface
scottgerring Aug 12, 2025
f48d0b5
Implement retry for http trace
scottgerring Aug 12, 2025
4a701f5
Implement retry for http metrics and logs
scottgerring Aug 12, 2025
2877126
changelog
scottgerring Aug 12, 2025
6f8f023
chore: wrap headers in arc to save cloning per batch
scottgerring Sep 2, 2025
9bde68f
chore: simplify HttpExportError creation
scottgerring Sep 2, 2025
a8e7ad8
chore: factor retry logic out
scottgerring Sep 2, 2025
b55b369
chore: configurable retry policy with default
scottgerring Sep 5, 2025
c4fb3d9
chore: modify tonic exporters so we don't _need_ the async runtime if…
scottgerring Sep 5, 2025
27d303a
chore: fix missing dep
scottgerring Sep 5, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ thiserror = { version = "2", default-features = false }
tonic = { version = "0.14.1", default-features = false }
tonic-prost-build = "0.14.1"
tonic-prost = "0.14.1"
tonic-types = "0.14.1"
tokio = { version = "1", default-features = false }
tokio-stream = "0.1"
# Using `tracing 0.1.40` because 0.1.39 (which is yanked) introduces the ability to set event names in macros,
Expand Down
1 change: 1 addition & 0 deletions opentelemetry-otlp/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Released 2025-Sep-11

- Update `opentelemetry-proto` dependency version to 0.30.1
- Add HTTP compression support with `gzip-http` and `zstd-http` feature flags
- Add retry with exponential backoff and throttling support for HTTP and gRPC exporters

## 0.30.0

Expand Down
10 changes: 9 additions & 1 deletion opentelemetry-otlp/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ tracing = {workspace = true, optional = true}

prost = { workspace = true, optional = true }
tonic = { workspace = true, optional = true }
tonic-types = { workspace = true, optional = true }
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Needed for gRPC error type

tokio = { workspace = true, features = ["sync", "rt"], optional = true }

reqwest = { workspace = true, optional = true }
Expand Down Expand Up @@ -69,10 +70,13 @@ serialize = ["serde", "serde_json"]
default = ["http-proto", "reqwest-blocking-client", "trace", "metrics", "logs", "internal-logs"]

# grpc using tonic
grpc-tonic = ["tonic", "prost", "http", "tokio", "opentelemetry-proto/gen-tonic"]
grpc-tonic = ["tonic", "tonic-types", "prost", "http", "tokio", "opentelemetry-proto/gen-tonic"]
gzip-tonic = ["tonic/gzip"]
zstd-tonic = ["tonic/zstd"]

# grpc with retry support
experimental-grpc-retry = ["grpc-tonic", "opentelemetry_sdk/experimental_async_runtime", "opentelemetry_sdk/rt-tokio"]

# http compression
gzip-http = ["flate2"]
zstd-http = ["zstd"]
Expand All @@ -82,6 +86,10 @@ tls-webpki-roots = ["tls", "tonic/tls-webpki-roots"]

# http binary
http-proto = ["prost", "opentelemetry-http", "opentelemetry-proto/gen-tonic-messages", "http", "trace", "metrics"]

# http with retry support.
experimental-http-retry = ["opentelemetry_sdk/experimental_async_runtime", "opentelemetry_sdk/rt-tokio", "tokio"]

http-json = ["serde_json", "prost", "opentelemetry-http", "opentelemetry-proto/gen-tonic-messages", "opentelemetry-proto/with-serde", "http", "trace", "metrics"]
reqwest-blocking-client = ["reqwest/blocking", "opentelemetry-http/reqwest-blocking"]
reqwest-client = ["reqwest", "opentelemetry-http/reqwest"]
Expand Down
3 changes: 3 additions & 0 deletions opentelemetry-otlp/allowed-external-types.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,7 @@ allowed_external_types = [
"tonic::transport::tls::Identity",
"tonic::transport::channel::Channel",
"tonic::service::interceptor::Interceptor",

# For retries
"tonic::status::Status"
]
57 changes: 6 additions & 51 deletions opentelemetry-otlp/src/exporter/http/logs.rs
Original file line number Diff line number Diff line change
@@ -1,61 +1,16 @@
use super::OtlpHttpClient;
use http::header::CONTENT_ENCODING;
use http::{header::CONTENT_TYPE, Method};
use opentelemetry::otel_debug;
use opentelemetry_sdk::error::{OTelSdkError, OTelSdkResult};
use opentelemetry_sdk::logs::{LogBatch, LogExporter};
use std::time;

impl LogExporter for OtlpHttpClient {
async fn export(&self, batch: LogBatch<'_>) -> OTelSdkResult {
let client = self
.client
.lock()
.map_err(|e| OTelSdkError::InternalFailure(format!("Mutex lock failed: {e}")))?
.clone()
.ok_or(OTelSdkError::AlreadyShutdown)?;

let (body, content_type, content_encoding) = self
.build_logs_export_body(batch)
.map_err(OTelSdkError::InternalFailure)?;

let mut request_builder = http::Request::builder()
.method(Method::POST)
.uri(&self.collector_endpoint)
.header(CONTENT_TYPE, content_type);

if let Some(encoding) = content_encoding {
request_builder = request_builder.header(CONTENT_ENCODING, encoding);
}

let mut request = request_builder
.body(body.into())
.map_err(|e| OTelSdkError::InternalFailure(e.to_string()))?;

for (k, v) in &self.headers {
request.headers_mut().insert(k.clone(), v.clone());
}

let request_uri = request.uri().to_string();
otel_debug!(name: "HttpLogsClient.ExportStarted");
let response = client
.send_bytes(request)
.await
.map_err(|e| OTelSdkError::InternalFailure(format!("{e:?}")))?;

if !response.status().is_success() {
let error = format!(
"OpenTelemetry logs export failed. Url: {}, Status Code: {}, Response: {:?}",
request_uri,
response.status().as_u16(),
response.body()
);
otel_debug!(name: "HttpLogsClient.ExportFailed", error = &error);
return Err(OTelSdkError::InternalFailure(error));
}

otel_debug!(name: "HttpLogsClient.ExportSucceeded");
Ok(())
self.export_http_with_retry(
batch,
OtlpHttpClient::build_logs_export_body,
"HttpLogsClient.Export",
)
.await
}

fn shutdown_with_timeout(&self, _timeout: time::Duration) -> OTelSdkResult {
Expand Down
67 changes: 8 additions & 59 deletions opentelemetry-otlp/src/exporter/http/metrics.rs
Original file line number Diff line number Diff line change
@@ -1,70 +1,19 @@
use std::sync::Arc;

use crate::metric::MetricsClient;
use http::{header::CONTENT_TYPE, Method};
use opentelemetry::otel_debug;
use opentelemetry_sdk::error::{OTelSdkError, OTelSdkResult};
use opentelemetry_sdk::metrics::data::ResourceMetrics;

use super::OtlpHttpClient;

impl MetricsClient for OtlpHttpClient {
async fn export(&self, metrics: &ResourceMetrics) -> OTelSdkResult {
let client = self
.client
.lock()
.map_err(|e| OTelSdkError::InternalFailure(format!("Failed to acquire lock: {e:?}")))
.and_then(|g| match &*g {
Some(client) => Ok(Arc::clone(client)),
_ => Err(OTelSdkError::AlreadyShutdown),
})?;

let (body, content_type, content_encoding) =
self.build_metrics_export_body(metrics).ok_or_else(|| {
OTelSdkError::InternalFailure("Failed to serialize metrics".to_string())
})?;

let mut request_builder = http::Request::builder()
.method(Method::POST)
.uri(&self.collector_endpoint)
.header(CONTENT_TYPE, content_type);

if let Some(encoding) = content_encoding {
request_builder = request_builder.header("Content-Encoding", encoding);
}

let mut request = request_builder
.body(body.into())
.map_err(|e| OTelSdkError::InternalFailure(format!("{e:?}")))?;

for (k, v) in &self.headers {
request.headers_mut().insert(k.clone(), v.clone());
}

otel_debug!(name: "HttpMetricsClient.ExportStarted");
let result = client.send_bytes(request).await;

match result {
Ok(response) => {
if response.status().is_success() {
otel_debug!(name: "HttpMetricsClient.ExportSucceeded");
Ok(())
} else {
let error = format!(
"OpenTelemetry metrics export failed. Status Code: {}, Response: {:?}",
response.status().as_u16(),
response.body()
);
otel_debug!(name: "HttpMetricsClient.ExportFailed", error = &error);
Err(OTelSdkError::InternalFailure(error))
}
}
Err(e) => {
let error = format!("{e:?}");
otel_debug!(name: "HttpMetricsClient.ExportFailed", error = &error);
Err(OTelSdkError::InternalFailure(error))
}
}
let build_body_wrapper = |client: &OtlpHttpClient, metrics: &ResourceMetrics| {
client
.build_metrics_export_body(metrics)
.ok_or_else(|| "Failed to serialize metrics".to_string())
};

self.export_http_with_retry(metrics, build_body_wrapper, "HttpMetricsClient.Export")
.await
}

fn shutdown(&self) -> OTelSdkResult {
Expand Down
Loading
Loading