feat(tracing-otlp): make trace id ratio sample customizable with --tracing-otlp.sample-ratio arg (#19438)

Author: lean-apple

Cargo.lock

@@ -3,7 +3,7 @@
 use clap::Parser;
 use eyre::WrapErr;
 use reth_tracing::{tracing_subscriber::EnvFilter, Layers};
-use reth_tracing_otlp::OtlpProtocol;
+use reth_tracing_otlp::{OtlpConfig, OtlpProtocol};
 use url::Url;
 
 /// CLI arguments for configuring `Opentelemetry` trace and span export.
@@ -78,6 +78,23 @@ pub struct TraceArgs {
         help_heading = "Tracing"
     )]
     pub service_name: String,
+
+    /// Trace sampling ratio to control the percentage of traces to export.
+    ///
+    /// Valid range: 0.0 to 1.0
+    /// - 1.0, default: Sample all traces
+    /// - 0.01: Sample 1% of traces
+    /// - 0.0: Disable sampling
+    ///
+    /// Example: --tracing-otlp.sample-ratio=0.0.
+    #[arg(
+        long = "tracing-otlp.sample-ratio",
+        env = "OTEL_TRACES_SAMPLER_ARG",
+        global = true,
+        value_name = "RATIO",
+        help_heading = "Tracing"
+    )]
+    pub sample_ratio: Option<f64>,
 }
 
 impl Default for TraceArgs {
@@ -86,6 +103,7 @@ impl Default for TraceArgs {
             otlp: None,
             protocol: OtlpProtocol::Http,
             otlp_filter: EnvFilter::from_default_env(),
+            sample_ratio: None,
             service_name: "reth".to_string(),
         }
     }
@@ -102,22 +120,24 @@ impl TraceArgs {
     /// Note: even though this function is async, it does not actually perform any async operations.
     /// It's needed only to be able to initialize the gRPC transport of OTLP tracing that needs to
     /// be called inside a tokio runtime context.
-    pub async fn init_otlp_tracing(
-        &mut self,
-        _layers: &mut Layers,
-    ) -> eyre::Result<OtlpInitStatus> {
+    pub async fn init_otlp_tracing(&mut self, layers: &mut Layers) -> eyre::Result<OtlpInitStatus> {
         if let Some(endpoint) = self.otlp.as_mut() {
             self.protocol.validate_endpoint(endpoint)?;
 
             #[cfg(feature = "otlp")]
             {
-                _layers.with_span_layer(
-                    self.service_name.clone(),
-                    endpoint.clone(),
-                    self.otlp_filter.clone(),
-                    self.protocol,
-                )?;
-                Ok(OtlpInitStatus::Started(endpoint.clone()))
+                {
+                    let config = OtlpConfig::new(
+                        self.service_name.clone(),
+                        endpoint.clone(),
+                        self.protocol,
+                        self.sample_ratio,
+                    )?;
+
+                    layers.with_span_layer(config.clone(), self.otlp_filter.clone())?;
+
+                    Ok(OtlpInitStatus::Started(config.endpoint().clone()))
+                }
             }
             #[cfg(not(feature = "otlp"))]
             {

crates/node/core/src/args/trace.rs

@@ -12,7 +12,7 @@ use opentelemetry::{global, trace::TracerProvider, KeyValue, Value};
 use opentelemetry_otlp::{SpanExporter, WithExportConfig};
 use opentelemetry_sdk::{
     propagation::TraceContextPropagator,
-    trace::{SdkTracer, SdkTracerProvider},
+    trace::{Sampler, SdkTracer, SdkTracerProvider},
     Resource,
 };
 use opentelemetry_semantic_conventions::{attribute::SERVICE_VERSION, SCHEMA_URL};
@@ -29,36 +29,92 @@ const HTTP_TRACE_ENDPOINT: &str = "/v1/traces";
 ///
 /// This layer can be added to a [`tracing_subscriber::Registry`] to enable `OpenTelemetry` tracing
 /// with OTLP export to an url.
-pub fn span_layer<S>(
-    service_name: impl Into<Value>,
-    endpoint: &Url,
-    protocol: OtlpProtocol,
-) -> eyre::Result<OpenTelemetryLayer<S, SdkTracer>>
+pub fn span_layer<S>(otlp_config: OtlpConfig) -> eyre::Result<OpenTelemetryLayer<S, SdkTracer>>
 where
     for<'span> S: Subscriber + LookupSpan<'span>,
 {
     global::set_text_map_propagator(TraceContextPropagator::new());
 
-    let resource = build_resource(service_name);
+    let resource = build_resource(otlp_config.service_name.clone());
 
     let span_builder = SpanExporter::builder();
 
-    let span_exporter = match protocol {
-        OtlpProtocol::Http => span_builder.with_http().with_endpoint(endpoint.as_str()).build()?,
-        OtlpProtocol::Grpc => span_builder.with_tonic().with_endpoint(endpoint.as_str()).build()?,
+    let span_exporter = match otlp_config.protocol {
+        OtlpProtocol::Http => {
+            span_builder.with_http().with_endpoint(otlp_config.endpoint.as_str()).build()?
+        }
+        OtlpProtocol::Grpc => {
+            span_builder.with_tonic().with_endpoint(otlp_config.endpoint.as_str()).build()?
+        }
     };
 
+    let sampler = build_sampler(otlp_config.sample_ratio)?;
+
     let tracer_provider = SdkTracerProvider::builder()
         .with_resource(resource)
+        .with_sampler(sampler)
         .with_batch_exporter(span_exporter)
         .build();
 
     global::set_tracer_provider(tracer_provider.clone());
 
-    let tracer = tracer_provider.tracer("reth");
+    let tracer = tracer_provider.tracer(otlp_config.service_name);
     Ok(tracing_opentelemetry::layer().with_tracer(tracer))
 }
 
+/// Configuration for OTLP trace export.
+#[derive(Debug, Clone)]
+pub struct OtlpConfig {
+    /// Service name for trace identification
+    service_name: String,
+    /// Otlp endpoint URL
+    endpoint: Url,
+    /// Transport protocol, HTTP or gRPC
+    protocol: OtlpProtocol,
+    /// Optional sampling ratio, from 0.0 to 1.0
+    sample_ratio: Option<f64>,
+}
+
+impl OtlpConfig {
+    /// Creates a new OTLP configuration.
+    pub fn new(
+        service_name: impl Into<String>,
+        endpoint: Url,
+        protocol: OtlpProtocol,
+        sample_ratio: Option<f64>,
+    ) -> eyre::Result<Self> {
+        if let Some(ratio) = sample_ratio {
+            ensure!(
+                (0.0..=1.0).contains(&ratio),
+                "Sample ratio must be between 0.0 and 1.0, got: {}",
+                ratio
+            );
+        }
+
+        Ok(Self { service_name: service_name.into(), endpoint, protocol, sample_ratio })
+    }
+
+    /// Returns the service name.
+    pub fn service_name(&self) -> &str {
+        &self.service_name
+    }
+
+    /// Returns the OTLP endpoint URL.
+    pub const fn endpoint(&self) -> &Url {
+        &self.endpoint
+    }
+
+    /// Returns the transport protocol.
+    pub const fn protocol(&self) -> OtlpProtocol {
+        self.protocol
+    }
+
+    /// Returns the sampling ratio.
+    pub const fn sample_ratio(&self) -> Option<f64> {
+        self.sample_ratio
+    }
+}
+
 // Builds OTLP resource with service information.
 fn build_resource(service_name: impl Into<Value>) -> Resource {
     Resource::builder()
@@ -67,6 +123,18 @@ fn build_resource(service_name: impl Into<Value>) -> Resource {
         .build()
 }
 
+/// Builds the appropriate sampler based on the sample ratio.
+fn build_sampler(sample_ratio: Option<f64>) -> eyre::Result<Sampler> {
+    match sample_ratio {
+        // Default behavior: sample all traces
+        None | Some(1.0) => Ok(Sampler::ParentBased(Box::new(Sampler::AlwaysOn))),
+        // Don't sample anything
+        Some(0.0) => Ok(Sampler::ParentBased(Box::new(Sampler::AlwaysOff))),
+        // Sample based on trace ID ratio
+        Some(ratio) => Ok(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(ratio)))),
+    }
+}
+
 /// OTLP transport protocol type
 #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
 pub enum OtlpProtocol {

crates/tracing-otlp/src/lib.rs

@@ -26,8 +26,7 @@ tracing-logfmt.workspace = true
 clap = { workspace = true, features = ["derive"] }
 eyre.workspace = true
 rolling-file.workspace = true
-url = { workspace = true, optional = true }
 
 [features]
 default = ["otlp"]
-otlp = ["reth-tracing-otlp", "dep:url"]
+otlp = ["reth-tracing-otlp"]

crates/tracing/Cargo.toml

@@ -1,16 +1,13 @@
 use crate::formatter::LogFormat;
+#[cfg(feature = "otlp")]
+use reth_tracing_otlp::{span_layer, OtlpConfig};
 use rolling_file::{RollingConditionBasic, RollingFileAppender};
 use std::{
     fmt,
     path::{Path, PathBuf},
 };
 use tracing_appender::non_blocking::WorkerGuard;
 use tracing_subscriber::{filter::Directive, EnvFilter, Layer, Registry};
-#[cfg(feature = "otlp")]
-use {
-    reth_tracing_otlp::{span_layer, OtlpProtocol},
-    url::Url,
-};
 
 /// A worker guard returned by the file layer.
 ///
@@ -137,14 +134,12 @@ impl Layers {
     #[cfg(feature = "otlp")]
     pub fn with_span_layer(
         &mut self,
-        service_name: String,
-        endpoint_exporter: Url,
+        otlp_config: OtlpConfig,
         filter: EnvFilter,
-        otlp_protocol: OtlpProtocol,
     ) -> eyre::Result<()> {
         // Create the span provider
 
-        let span_layer = span_layer(service_name, &endpoint_exporter, otlp_protocol)
+        let span_layer = span_layer(otlp_config)
             .map_err(|e| eyre::eyre!("Failed to build OTLP span exporter {}", e))?
             .with_filter(filter);
 

crates/tracing/src/layers.rs

@@ -146,4 +146,13 @@ Tracing:
           Defaults to TRACE if not specified.
 
           [default: debug]
+
+      --tracing-otlp.sample-ratio <RATIO>
+          Trace sampling ratio to control the percentage of traces to export.
+
+          Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
+
+          Example: --tracing-otlp.sample-ratio=0.0.
+
+          [env: OTEL_TRACES_SAMPLER_ARG=]
 ```

docs/vocs/docs/pages/cli/reth.mdx

@@ -132,4 +132,13 @@ Tracing:
           Defaults to TRACE if not specified.
 
           [default: debug]
+
+      --tracing-otlp.sample-ratio <RATIO>
+          Trace sampling ratio to control the percentage of traces to export.
+
+          Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
+
+          Example: --tracing-otlp.sample-ratio=0.0.
+
+          [env: OTEL_TRACES_SAMPLER_ARG=]
 ```

docs/vocs/docs/pages/cli/reth/config.mdx

@@ -242,4 +242,13 @@ Tracing:
           Defaults to TRACE if not specified.
 
           [default: debug]
+
+      --tracing-otlp.sample-ratio <RATIO>
+          Trace sampling ratio to control the percentage of traces to export.
+
+          Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
+
+          Example: --tracing-otlp.sample-ratio=0.0.
+
+          [env: OTEL_TRACES_SAMPLER_ARG=]
 ```

docs/vocs/docs/pages/cli/reth/db.mdx

@@ -149,4 +149,13 @@ Tracing:
           Defaults to TRACE if not specified.
 
           [default: debug]
+
+      --tracing-otlp.sample-ratio <RATIO>
+          Trace sampling ratio to control the percentage of traces to export.
+
+          Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
+
+          Example: --tracing-otlp.sample-ratio=0.0.
+
+          [env: OTEL_TRACES_SAMPLER_ARG=]
 ```

docs/vocs/docs/pages/cli/reth/db/checksum.mdx

@@ -141,4 +141,13 @@ Tracing:
           Defaults to TRACE if not specified.
 
           [default: debug]
+
+      --tracing-otlp.sample-ratio <RATIO>
+          Trace sampling ratio to control the percentage of traces to export.
+
+          Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
+
+          Example: --tracing-otlp.sample-ratio=0.0.
+
+          [env: OTEL_TRACES_SAMPLER_ARG=]
 ```