From d50452e60b77960256453df5d1e6d3a69ff4184c Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Tue, 30 Jun 2026 11:37:58 -0400 Subject: [PATCH 1/6] feat(relay): migrate telemetry to OpenTelemetry with OTLP and Prometheus export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace metrics-rs/metrics-exporter-prometheus with OpenTelemetry native instruments backed by both a Prometheus text endpoint (:9102) and an OTLP gRPC exporter. Add distributed tracing via tracing-opentelemetry. Add DB and Redis pool metrics. ## What changed ### Metrics - Rewrote metrics.rs as an OTEL setup module: SdkMeterProvider with a PrometheusExporter (pull-based, same /metrics endpoint) and an optional PeriodicReader+OTLP exporter gated on OTEL_EXPORTER_OTLP_ENDPOINT. - Migrated all 41 metrics::counter!/histogram!/gauge! call sites across connection.rs, subscription.rs, state.rs, handlers/, and api/ to the pre-built Metrics struct (OnceLock, zero per-call-site allocation). - Preserved every metric name, type, label set, and histogram bucket from the prior implementation so existing Prometheus scrapers (including the Datadog Agent openmetrics annotation) need no changes. - Instruments lazy-init to OTEL noop meter when install() hasn't been called, matching prior metrics-rs behaviour in unit tests. ### Tracing - Added telemetry.rs: try_init_tracer() initialises an OTLP gRPC span exporter + SdkTracerProvider when OTEL_EXPORTER_OTLP_ENDPOINT is set; returns None (zero overhead) when unset. - Wired OpenTelemetryLayer into the tracing_subscriber stack in main.rs alongside the existing JSON fmt layer (stdout logs unchanged). - Added #[instrument] spans on hot paths: handle_event, fan_out_pubsub_event, handle_auth, SubRegistry::fan_out_scoped. ### DB and Redis pool metrics - Added Db::pool_stats() -> DbPoolStats in buzz-db (exposes sqlx pool size and num_idle). - Added background task in main.rs polling pool stats every 10 s (configurable via BUZZ_POOL_METRICS_INTERVAL_SECS) and emitting buzz_db_pool_{size,idle,active} and buzz_redis_pool_{available,size, max,waiting} gauges. ### Graceful shutdown - SdkMeterProvider and optional SdkTracerProvider flushed on SIGTERM drain. ## Environment variables (all optional) - OTEL_EXPORTER_OTLP_ENDPOINT — unset disables OTEL entirely - OTEL_SERVICE_NAME — defaults to buzz-relay - OTEL_RESOURCE_ATTRIBUTES — extra resource attributes - OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG — sampling strategy - BUZZ_POOL_METRICS_INTERVAL_SECS — pool poll interval (default 10) Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- Cargo.lock | 302 ++++++++------- Cargo.toml | 12 +- crates/buzz-db/src/lib.rs | 20 + crates/buzz-relay/Cargo.toml | 9 +- crates/buzz-relay/src/api/bridge.rs | 8 +- crates/buzz-relay/src/api/media.rs | 16 +- crates/buzz-relay/src/connection.rs | 10 +- crates/buzz-relay/src/handlers/auth.rs | 19 +- crates/buzz-relay/src/handlers/count.rs | 8 +- crates/buzz-relay/src/handlers/event.rs | 36 +- crates/buzz-relay/src/lib.rs | 2 + crates/buzz-relay/src/main.rs | 68 +++- crates/buzz-relay/src/metrics.rs | 490 ++++++++++++++++++++++-- crates/buzz-relay/src/state.rs | 16 +- crates/buzz-relay/src/subscription.rs | 9 +- crates/buzz-relay/src/telemetry.rs | 57 +++ 16 files changed, 849 insertions(+), 233 deletions(-) create mode 100644 crates/buzz-relay/src/telemetry.rs diff --git a/Cargo.lock b/Cargo.lock index 09108807c6..a5c1a112d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1053,10 +1053,14 @@ dependencies = [ "infer", "mesh-llm-host-runtime", "mesh-llm-sdk", - "metrics", - "metrics-exporter-prometheus", "moka", "nostr", + "opentelemetry 0.32.0", + "opentelemetry-otlp 0.32.0", + "opentelemetry-prometheus", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk 0.32.1", + "prometheus", "rand 0.10.1", "redis", "reqwest 0.13.3", @@ -1074,6 +1078,7 @@ dependencies = [ "tower", "tower-http", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "url", "uuid", @@ -2394,17 +2399,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "evmap" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" -dependencies = [ - "hashbag", - "left-right", - "smallvec", -] - [[package]] name = "fastrand" version = "2.4.1" @@ -2844,12 +2838,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbag" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" - [[package]] name = "hashbrown" version = "0.14.5" @@ -3197,7 +3185,6 @@ dependencies = [ "hyper", "hyper-util", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -3868,17 +3855,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" -[[package]] -name = "left-right" -version = "0.11.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0c21e4c8ff95f487fb34e6f9182875f42c84cef966d29216bf115d9bba835a" -dependencies = [ - "crossbeam-utils", - "loom", - "slab", -] - [[package]] name = "libc" version = "0.2.186" @@ -4250,9 +4226,9 @@ dependencies = [ "model-resolver", "nostr-sdk", "openai-frontend", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry_sdk", + "opentelemetry 0.31.0", + "opentelemetry-otlp 0.31.1", + "opentelemetry_sdk 0.31.0", "prost", "rand 0.10.1", "regex-lite", @@ -4489,56 +4465,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "metrics" -version = "0.24.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" -dependencies = [ - "portable-atomic", - "rapidhash", -] - -[[package]] -name = "metrics-exporter-prometheus" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" -dependencies = [ - "base64", - "evmap", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "indexmap", - "ipnet", - "metrics", - "metrics-util", - "quanta", - "rustls", - "thiserror 2.0.18", - "tokio", - "tracing", -] - -[[package]] -name = "metrics-util" -version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", - "hashbrown 0.16.1", - "metrics", - "quanta", - "rand 0.9.4", - "rand_xoshiro", - "rapidhash", - "sketches-ddsketch", -] - [[package]] name = "mime" version = "0.3.17" @@ -5425,6 +5351,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.18", + "tracing", +] + [[package]] name = "opentelemetry-http" version = "0.31.0" @@ -5434,7 +5374,7 @@ dependencies = [ "async-trait", "bytes", "http", - "opentelemetry", + "opentelemetry 0.31.0", "reqwest 0.12.28", ] @@ -5445,15 +5385,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ "http", - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry-http", - "opentelemetry-proto", - "opentelemetry_sdk", + "opentelemetry-proto 0.31.0", + "opentelemetry_sdk 0.31.0", "prost", "reqwest 0.12.28", "thiserror 2.0.18", ] +[[package]] +name = "opentelemetry-otlp" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +dependencies = [ + "http", + "opentelemetry 0.32.0", + "opentelemetry-proto 0.32.0", + "opentelemetry_sdk 0.32.1", + "prost", + "thiserror 2.0.18", + "tokio", + "tonic", + "tonic-types", +] + +[[package]] +name = "opentelemetry-prometheus" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c0359983e7f79cf33c9abd89e5d7ddf67c46c419d0148598022d70e70c01aba" +dependencies = [ + "once_cell", + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.1", + "prometheus", + "tracing", +] + [[package]] name = "opentelemetry-proto" version = "0.31.0" @@ -5462,8 +5432,8 @@ checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "base64", "const-hex", - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.31.0", + "opentelemetry_sdk 0.31.0", "prost", "serde", "serde_json", @@ -5471,6 +5441,25 @@ dependencies = [ "tonic-prost", ] +[[package]] +name = "opentelemetry-proto" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.1", + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" + [[package]] name = "opentelemetry_sdk" version = "0.31.0" @@ -5480,10 +5469,28 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.31.0", + "percent-encoding", + "rand 0.9.4", + "thiserror 2.0.18", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.32.0", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", + "tokio", + "tokio-stream", ] [[package]] @@ -5932,6 +5939,21 @@ dependencies = [ "windows 0.62.2", ] +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if 1.0.4", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror 2.0.18", +] + [[package]] name = "proptest" version = "1.11.0" @@ -6002,6 +6024,26 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "protoc-bin-vendored" version = "3.2.0" @@ -6072,21 +6114,6 @@ version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" -[[package]] -name = "quanta" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" -dependencies = [ - "crossbeam-utils", - "libc", - "once_cell", - "raw-cpuid", - "wasi 0.11.1+wasi-snapshot-preview1", - "web-sys", - "winapi", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -6289,33 +6316,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_xoshiro" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" -dependencies = [ - "rand_core 0.9.5", -] - -[[package]] -name = "rapidhash" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" -dependencies = [ - "rustversion", -] - -[[package]] -name = "raw-cpuid" -version = "11.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" -dependencies = [ - "bitflags", -] - [[package]] name = "redb" version = "3.1.3" @@ -7316,12 +7316,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" -[[package]] -name = "sketches-ddsketch" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" - [[package]] name = "skippy-cache" version = "0.68.0" @@ -7392,7 +7386,7 @@ dependencies = [ "futures-util", "libc", "openai-frontend", - "opentelemetry-proto", + "opentelemetry-proto 0.31.0", "serde", "serde_json", "sha2 0.10.9", @@ -8254,6 +8248,7 @@ dependencies = [ "socket2", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", @@ -8272,6 +8267,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -8389,6 +8395,22 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +dependencies = [ + "js-sys", + "opentelemetry 0.32.0", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 03cba4ee7c..fd5a0f7d6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,10 +65,14 @@ serde_yaml = "0.9" evalexpr = "11" cron = "0.16" # Observability -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } -metrics = "0.24" -metrics-exporter-prometheus = "0.18" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +tracing-opentelemetry = { version = "0.33", features = ["metrics"] } +opentelemetry = { version = "0.32", features = ["trace", "metrics", "logs"] } +opentelemetry_sdk = { version = "0.32", features = ["trace", "metrics", "rt-tokio"] } +opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "metrics", "grpc-tonic", "tls-ring"] } +opentelemetry-prometheus = { version = "0.32" } +opentelemetry-semantic-conventions = { version = "0.32" } # Error handling thiserror = "2" diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index f3f0bc25d0..81881c70b1 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -131,6 +131,15 @@ pub struct Db { pub(crate) pool: PgPool, } +/// Snapshot of Postgres connection pool utilisation. +#[derive(Debug, Clone, Copy)] +pub struct DbPoolStats { + /// Total connections currently in the pool (idle + active). + pub size: u32, + /// Connections available for immediate reuse. + pub idle: u32, +} + /// Configuration for the Postgres connection pool. #[derive(Debug, Clone)] pub struct DbConfig { @@ -219,6 +228,17 @@ impl Db { sqlx::query("SELECT 1").execute(&self.pool).await.is_ok() } + /// Returns pool utilisation stats for metrics emission. + /// + /// `size` — total connections (idle + active) + /// `idle` — connections available for immediate reuse + pub fn pool_stats(&self) -> DbPoolStats { + DbPoolStats { + size: self.pool.size(), + idle: self.pool.num_idle() as u32, + } + } + /// Begin a database transaction for atomic multi-statement operations. /// /// Returns a `'static` transaction because `PgPool` is `Arc`-backed internally. diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index c9b80d53fc..594863099c 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -36,6 +36,13 @@ serde = { workspace = true } serde_json = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +tracing-opentelemetry = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry-prometheus = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } +prometheus = "0.14" thiserror = { workspace = true } anyhow = { workspace = true } uuid = { workspace = true } @@ -61,8 +68,6 @@ rand = { workspace = true } hex = { workspace = true } url = { workspace = true } moka = { workspace = true } -metrics = { workspace = true } -metrics-exporter-prometheus = { workspace = true } [features] dev = ["buzz-auth/dev"] diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 3882363d80..da82ff5efc 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -744,7 +744,9 @@ pub async fn count_events( match state.db.query_events(&q).await { Ok(stored_events) => { if crate::handlers::req::count_fallback_exceeded(stored_events.len()) { - metrics::counter!("buzz_count_fallback_rejections_total").increment(1); + crate::metrics::metrics() + .count_fallback_rejections_total + .add(1, &[]); return Err(api_error( StatusCode::BAD_REQUEST, "count filter requires narrower constraints", @@ -801,7 +803,9 @@ pub async fn count_events( match state.db.query_events(&query).await { Ok(stored_events) => { if crate::handlers::req::count_fallback_exceeded(stored_events.len()) { - metrics::counter!("buzz_count_fallback_rejections_total").increment(1); + crate::metrics::metrics() + .count_fallback_rejections_total + .add(1, &[]); return Err(api_error( StatusCode::BAD_REQUEST, "count filter requires narrower constraints", diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index bc7d9ed544..4e421b6ba4 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -193,13 +193,15 @@ impl FromRequestParts> for AuthenticatedUpload { .map_err(|_| MediaError::RelayMembershipRequired)?; if upload_rate_limited(state, &auth_event.pubkey) { - metrics::counter!("buzz_media_upload_rejections_total", "reason" => "rate_limit") - .increment(1); + crate::metrics::metrics() + .media_upload_rejections_total + .add(1, &[opentelemetry::KeyValue::new("reason", "rate_limit")]); return Err(MediaError::UploadRateLimitExceeded); } let upload_permit = acquire_upload_permit(state, &auth_event.pubkey).inspect_err(|_| { - metrics::counter!("buzz_media_upload_rejections_total", "reason" => "concurrency") - .increment(1); + crate::metrics::metrics() + .media_upload_rejections_total + .add(1, &[opentelemetry::KeyValue::new("reason", "concurrency")]); })?; Ok(AuthenticatedUpload { @@ -310,7 +312,9 @@ pub async fn upload_blob( } _ => "other", }; - metrics::counter!("buzz_media_uploads_total", "mime" => mime_label.to_owned()).increment(1); + crate::metrics::metrics() + .media_uploads_total + .add(1, &[opentelemetry::KeyValue::new("mime", mime_label.to_string())]); // Audit via bounded channel — same pattern as event audit. let desc = descriptor.clone(); @@ -330,7 +334,7 @@ pub async fn upload_blob( .await { tracing::error!("Media audit channel closed — entry lost: {e}"); - metrics::counter!("buzz_audit_send_errors_total").increment(1); + crate::metrics::metrics().audit_send_errors_total.add(1, &[]); } Ok(Json(descriptor)) diff --git a/crates/buzz-relay/src/connection.rs b/crates/buzz-relay/src/connection.rs index 58a183acb5..cbf5117782 100644 --- a/crates/buzz-relay/src/connection.rs +++ b/crates/buzz-relay/src/connection.rs @@ -92,7 +92,7 @@ impl ConnectionState { let count = self.backpressure_count.fetch_add(1, Ordering::Relaxed) + 1; if count >= self.grace_limit { warn!(conn_id = %self.conn_id, count, "sustained backpressure — closing slow client"); - metrics::counter!("buzz_ws_backpressure_disconnects_total").increment(1); + crate::metrics::metrics().ws_backpressure_disconnects_total.add(1, &[]); self.cancel.cancel(); } else { warn!(conn_id = %self.conn_id, count, grace = self.grace_limit, "send buffer full — grace {count}/{}", self.grace_limit); @@ -153,7 +153,7 @@ pub async fn handle_connection( }); info!(conn_id = %conn_id, addr = %addr, "WebSocket connection established"); - metrics::counter!("buzz_ws_connections_total").increment(1); + crate::metrics::metrics().ws_connections_total.add(1, &[]); let challenge_msg = RelayMessage::auth_challenge(&challenge); if tx @@ -167,7 +167,7 @@ pub async fn handle_connection( // Gauge incremented AFTER challenge send succeeds — early disconnects // don't leak. Decremented in the cleanup path below. - metrics::gauge!("buzz_ws_connections_active").increment(1.0); + crate::metrics::metrics().ws_connections_active.add(1, &[]); // Register after challenge succeeds — avoids leaked entries on early disconnect. state.conn_manager.register( @@ -208,7 +208,7 @@ pub async fn handle_connection( timeout_secs = AUTH_TIMEOUT.as_secs(), "NIP-42 auth timeout — closing connection" ); - metrics::counter!("buzz_ws_auth_timeouts_total").increment(1); + crate::metrics::metrics().ws_auth_timeouts_total.add(1, &[]); auth_timeout_cancel.cancel(); } } @@ -248,7 +248,7 @@ pub async fn handle_connection( .await; } } - metrics::gauge!("buzz_ws_connections_active").decrement(1.0); + crate::metrics::metrics().ws_connections_active.add(-1, &[]); info!(conn_id = %conn_id, addr = %addr, "WebSocket connection closed"); drop(permit); diff --git a/crates/buzz-relay/src/handlers/auth.rs b/crates/buzz-relay/src/handlers/auth.rs index 143dfbee9f..a6038d9f26 100644 --- a/crates/buzz-relay/src/handlers/auth.rs +++ b/crates/buzz-relay/src/handlers/auth.rs @@ -38,6 +38,7 @@ pub fn extract_auth_tag_json(event: &nostr::Event) -> Option { /// the connection to authenticated state. /// /// Pure crypto verification — no API tokens, no JWT, no DB token lookups. +#[tracing::instrument(skip_all, fields(event_id, conn_id))] pub async fn handle_auth(event: nostr::Event, conn: Arc, state: Arc) { let event_id_hex = event.id.to_hex(); let (challenge, conn_id) = { @@ -74,7 +75,9 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: crate::api::bridge::nip42_expected_relay_url(&state.config.relay_url, &conn.tenant); let auth_svc = Arc::clone(&state.auth); - metrics::counter!("buzz_auth_attempts_total", "method" => "nip42").increment(1); + crate::metrics::metrics() + .auth_attempts_total + .add(1, &[opentelemetry::KeyValue::new("method", "nip42")]); // Pure NIP-42 verification — crypto only, no DB lookups. match auth_svc @@ -102,8 +105,9 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: }; if !allowed { warn!(conn_id = %conn_id, pubkey = %pubkey.to_hex(), "pubkey not in allowlist"); - metrics::counter!("buzz_auth_failures_total", "reason" => "allowlist_denied") - .increment(1); + crate::metrics::metrics() + .auth_failures_total + .add(1, &[opentelemetry::KeyValue::new("reason", "allowlist_denied")]); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, @@ -126,8 +130,9 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: Ok(owner) => owner, Err(e) => { warn!(conn_id = %conn_id, pubkey = %pubkey.to_hex(), error = ?e, "not a relay member"); - metrics::counter!("buzz_auth_failures_total", "reason" => "not_relay_member") - .increment(1); + crate::metrics::metrics() + .auth_failures_total + .add(1, &[opentelemetry::KeyValue::new("reason", "not_relay_member")]); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, @@ -241,7 +246,9 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: } Err(e) => { warn!(conn_id = %conn_id, error = %e, "NIP-42 auth failed"); - metrics::counter!("buzz_auth_failures_total", "reason" => "nip42_invalid").increment(1); + crate::metrics::metrics() + .auth_failures_total + .add(1, &[opentelemetry::KeyValue::new("reason", "nip42_invalid")]); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, diff --git a/crates/buzz-relay/src/handlers/count.rs b/crates/buzz-relay/src/handlers/count.rs index 7cb4882185..f92dfdfc63 100644 --- a/crates/buzz-relay/src/handlers/count.rs +++ b/crates/buzz-relay/src/handlers/count.rs @@ -164,7 +164,9 @@ pub async fn handle_count( match state.db.query_events(&q).await { Ok(stored_events) => { if super::req::count_fallback_exceeded(stored_events.len()) { - metrics::counter!("buzz_count_fallback_rejections_total").increment(1); + crate::metrics::metrics() + .count_fallback_rejections_total + .add(1, &[]); conn.send(RelayMessage::closed( &sub_id, "restricted: count filter requires narrower constraints", @@ -227,7 +229,9 @@ pub async fn handle_count( match state.db.query_events(&query).await { Ok(stored_events) => { if super::req::count_fallback_exceeded(stored_events.len()) { - metrics::counter!("buzz_count_fallback_rejections_total").increment(1); + crate::metrics::metrics() + .count_fallback_rejections_total + .add(1, &[]); conn.send(RelayMessage::closed( &sub_id, "restricted: count filter requires narrower constraints", diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index 61fd196dbc..f0ee1044b3 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -27,7 +27,9 @@ use super::ingest::{IngestAuth, IngestError}; /// Increment the rejection counter with a bounded reason label. fn reject(reason: &'static str) { - metrics::counter!("buzz_events_rejected_total", "reason" => reason).increment(1); + crate::metrics::metrics() + .events_rejected_total + .add(1, &[opentelemetry::KeyValue::new("reason", reason)]); } /// Bound the `kind` label to prevent cardinality explosion from arbitrary Nostr kinds. @@ -157,7 +159,9 @@ pub(crate) async fn fan_out_event_to_local_subscribers( ) { let matches = state.sub_registry.fan_out_scoped(community_id, stored); let matches = filter_fanout_by_access(state, community_id, stored, matches).await; - metrics::histogram!("buzz_fanout_recipients").record(matches.len() as f64); + crate::metrics::metrics() + .fanout_recipients + .record(matches.len() as f64, &[]); if matches.is_empty() { return; } @@ -186,6 +190,7 @@ pub(crate) async fn fan_out_event_to_local_subscribers( } /// Fan out one event received from Redis pub/sub to this relay's local subscribers. +#[tracing::instrument(skip_all)] pub async fn fan_out_pubsub_event(state: &Arc, channel_event: buzz_pubsub::ChannelEvent) { // The Redis topic carries the tenant-local routing scope explicitly: // `Channel(id)` for a per-channel event, `Global` for a channel-less one. @@ -212,7 +217,7 @@ pub async fn fan_out_pubsub_event(state: &Arc, channel_event: buzz_pub let matches = state.sub_registry.fan_out_scoped(community_id, &stored); let matches = filter_fanout_by_access(state, community_id, &stored, matches).await; - metrics::counter!("buzz_multinode_fanout_total").increment(1); + crate::metrics::metrics().multinode_fanout_total.add(1, &[]); if matches.is_empty() { return; } @@ -276,7 +281,9 @@ pub(crate) async fn dispatch_persistent_event( .sub_registry .fan_out_scoped(tenant.community(), stored_event); let matches = filter_fanout_by_access(state, tenant.community(), stored_event, matches).await; - metrics::histogram!("buzz_fanout_recipients").record(matches.len() as f64); + crate::metrics::metrics() + .fanout_recipients + .record(matches.len() as f64, &[]); debug!( event_id = %event_id_hex, channel_id = ?stored_event.channel_id, @@ -356,7 +363,7 @@ pub(crate) async fn dispatch_persistent_event( }; if let Err(e) = state.audit_tx.send(audit_entry).await { error!(event_id = %event_id_hex, "Audit channel closed — entry lost: {e}"); - metrics::counter!("buzz_audit_send_errors_total").increment(1); + crate::metrics::metrics().audit_send_errors_total.add(1, &[]); } // Skip workflow triggering for workflow-execution kinds and relay-signed workflow messages. @@ -388,8 +395,9 @@ pub(crate) async fn dispatch_persistent_event( { tracing::error!(event_id = ?workflow_event.event.id, "Workflow trigger failed: {e}"); } else { - metrics::counter!("buzz_workflow_runs_total", "trigger" => trigger_kind) - .increment(1); + crate::metrics::metrics() + .workflow_runs_total + .add(1, &[opentelemetry::KeyValue::new("trigger", trigger_kind)]); } }); } @@ -401,13 +409,16 @@ pub(crate) async fn dispatch_persistent_event( /// /// Extracts auth from the WS connection, dispatches ephemeral events locally, /// and delegates persistent events to [`super::ingest::ingest_event`]. +#[tracing::instrument(skip_all, fields(event_id, kind))] pub async fn handle_event(event: Event, conn: Arc, state: Arc) { let start = std::time::Instant::now(); let event_id_hex = event.id.to_hex(); let kind_u32 = event_kind_u32(&event); let kind_str = bounded_kind_label(kind_u32); debug!(event_id = %event_id_hex, kind = kind_u32, "EVENT"); - metrics::counter!("buzz_events_received_total", "kind" => kind_str.clone()).increment(1); + crate::metrics::metrics() + .events_received_total + .add(1, &[opentelemetry::KeyValue::new("kind", kind_str.to_string())]); let (conn_id, pubkey_bytes, auth_pubkey, scopes, channel_ids) = { let auth = conn.auth_state.read().await; @@ -506,7 +517,9 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc { if result.accepted { - metrics::counter!("buzz_events_stored_total", "kind" => kind_str).increment(1); + crate::metrics::metrics() + .events_stored_total + .add(1, &[opentelemetry::KeyValue::new("kind", kind_str.to_string())]); info!( event_id = %result.event_id, kind = kind_u32, @@ -514,8 +527,9 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc) -> bool { @@ -28,9 +29,18 @@ fn buzz_auto_migrate_enabled(value: Option<&str>) -> bool { #[tokio::main] async fn main() -> anyhow::Result<()> { // JSON-only structured logs — simple, machine-parseable, CAKE-compatible. + // If OTEL_EXPORTER_OTLP_ENDPOINT is set, also attach an OpenTelemetry tracing + // layer that exports spans via OTLP gRPC alongside the JSON stdout logs. + let tracer_provider = telemetry::try_init_tracer(); + let otel_layer = tracer_provider.as_ref().map(|p| { + use opentelemetry::trace::TracerProvider as _; + tracing_opentelemetry::layer().with_tracer(p.tracer("buzz-relay")) + }); + tracing_subscriber::registry() .with(fmt::layer().json().flatten_event(true)) .with(EnvFilter::from_default_env().add_directive("buzz_relay=info".parse()?)) + .with(otel_layer) .init(); info!("Starting buzz-relay"); @@ -48,7 +58,7 @@ async fn main() -> anyhow::Result<()> { "Config loaded" ); - relay_metrics::install(config.metrics_port); + let meter_provider = relay_metrics::install(config.metrics_port); info!( port = config.metrics_port, "Prometheus metrics exporter started" @@ -646,6 +656,9 @@ async fn main() -> anyhow::Result<()> { let state_for_sub = Arc::clone(&state); let mut rx = state_for_sub.pubsub.subscribe_local(); tokio::spawn(async move { + let fanout_lag = relay_metrics::meter() + .u64_counter("buzz_multinode_fanout_lag_total") + .build(); loop { match rx.recv().await { Ok(channel_event) => { @@ -656,7 +669,7 @@ async fn main() -> anyhow::Result<()> { .await; } Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - metrics::counter!("buzz_multinode_fanout_lag_total").increment(n); + fanout_lag.add(n, &[]); tracing::warn!("Multi-node fan-out lagged by {n} messages"); } Err(tokio::sync::broadcast::error::RecvError::Closed) => { @@ -676,6 +689,9 @@ async fn main() -> anyhow::Result<()> { let state_for_cache = Arc::clone(&state); let mut rx = state_for_cache.pubsub.subscribe_cache_invalidations(); tokio::spawn(async move { + let cache_inval_lag = relay_metrics::meter() + .u64_counter("buzz_cache_invalidation_lag_total") + .build(); loop { match rx.recv().await { Ok(scoped) => { @@ -687,7 +703,7 @@ async fn main() -> anyhow::Result<()> { .apply_cache_invalidation(scoped.community_id, scoped.invalidation); } Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - metrics::counter!("buzz_cache_invalidation_lag_total").increment(n); + cache_inval_lag.add(n, &[]); tracing::warn!("Cache-invalidation consumer lagged by {n} messages"); } Err(tokio::sync::broadcast::error::RecvError::Closed) => { @@ -702,6 +718,42 @@ async fn main() -> anyhow::Result<()> { let router = build_router(Arc::clone(&state)); let health_router = build_health_router(Arc::clone(&state)); + // Pool metrics: periodic background task polling DB + Redis pool stats. + { + let pool_state = Arc::clone(&state); + let interval_secs = std::env::var("BUZZ_POOL_METRICS_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(10); + tokio::spawn(async move { + let m = relay_metrics::meter(); + let db_pool_size = m.u64_gauge("buzz_db_pool_size").build(); + let db_pool_idle = m.u64_gauge("buzz_db_pool_idle").build(); + let db_pool_active = m.u64_gauge("buzz_db_pool_active").build(); + let redis_pool_available = m.u64_gauge("buzz_redis_pool_available").build(); + let redis_pool_size = m.u64_gauge("buzz_redis_pool_size").build(); + let redis_pool_max = m.u64_gauge("buzz_redis_pool_max").build(); + let redis_pool_waiting = m.u64_gauge("buzz_redis_pool_waiting").build(); + + let mut interval = + tokio::time::interval(std::time::Duration::from_secs(interval_secs)); + loop { + interval.tick().await; + let db_stats = pool_state.db.pool_stats(); + let active = db_stats.size.saturating_sub(db_stats.idle); + db_pool_size.record(db_stats.size as u64, &[]); + db_pool_idle.record(db_stats.idle as u64, &[]); + db_pool_active.record(active as u64, &[]); + + let rs = pool_state.redis_pool.status(); + redis_pool_available.record(rs.available as u64, &[]); + redis_pool_size.record(rs.size as u64, &[]); + redis_pool_max.record(rs.max_size as u64, &[]); + redis_pool_waiting.record(rs.waiting as u64, &[]); + } + }); + } + serve(router, health_router, Arc::clone(&state)).await?; // Signal the audit worker to stop accepting, flush buffered entries, and @@ -711,6 +763,16 @@ async fn main() -> anyhow::Result<()> { .drain(std::time::Duration::from_secs(5)) .await; + // Flush pending OTEL spans and metrics before exit. + if let Err(e) = meter_provider.shutdown() { + tracing::warn!(error = %e, "OTEL meter provider shutdown error"); + } + if let Some(tp) = tracer_provider { + if let Err(e) = tp.shutdown() { + tracing::warn!(error = %e, "OTEL tracer provider shutdown error"); + } + } + Ok(()) } diff --git a/crates/buzz-relay/src/metrics.rs b/crates/buzz-relay/src/metrics.rs index 954e78434d..0aa684625a 100644 --- a/crates/buzz-relay/src/metrics.rs +++ b/crates/buzz-relay/src/metrics.rs @@ -1,19 +1,25 @@ -//! Prometheus metrics: recorder setup, upkeep task, and HTTP middleware. +//! OpenTelemetry metrics: provider setup, instrument handles, and HTTP middleware. //! //! ```text -//! ┌──────────────────────────────────────────────────────────┐ -//! │ metrics-rs facade (metrics::counter!, histogram!, etc.) │ -//! │ ↓ │ -//! │ PrometheusBuilder → HTTP listener on :9102 │ -//! │ ↓ │ -//! │ GET /metrics → Prometheus text format │ -//! └──────────────────────────────────────────────────────────┘ +//! ┌──────────────────────────────────────────────────────────────────────┐ +//! │ OTEL Meter API via global Metrics struct (pre-built instrument │ +//! │ handles — one allocation at startup, zero per call-site) │ +//! │ ↓ │ +//! │ SdkMeterProvider │ +//! │ ├── PrometheusExporter → prometheus::Registry → HTTP :9102 │ +//! │ └── (if OTEL_EXPORTER_OTLP_ENDPOINT set) │ +//! │ PeriodicReader + OTLP MetricExporter → collector/DD agent │ +//! └──────────────────────────────────────────────────────────────────────┘ //! ``` //! -//! Framework metrics (`http_requests_total`, `http_request_latency_ms`) are -//! recorded by [`track_metrics`] middleware on the app router. Buzz-specific -//! metrics are recorded inline at their call sites. +//! All metric names, types, and label sets are preserved from the prior +//! `metrics-rs` implementation so existing Prometheus scrapers and Datadog +//! dashboards need no changes. +//! +//! Custom histogram bucket boundaries are registered as OTEL Views before the +//! provider is built. +use std::sync::OnceLock; use std::time::Instant; use axum::{ @@ -21,49 +27,343 @@ use axum::{ middleware::Next, response::Response, }; -use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; +use opentelemetry::{ + metrics::{Counter, Histogram, Meter, UpDownCounter}, + KeyValue, +}; +use opentelemetry_sdk::metrics::{Aggregation, Instrument, PeriodicReader, SdkMeterProvider, Stream}; +use prometheus::Registry; + +// ─── Bucket boundaries ─────────────────────────────────────────────────────── /// HTTP latency buckets (milliseconds) — only for `http_request_latency_ms`. -const LATENCY_BUCKETS_MS: [f64; 11] = [ +pub const LATENCY_BUCKETS_MS: &[f64] = &[ 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0, 10000.0, ]; /// Seconds-scale buckets for internal processing histograms (event, search, audit). -const DURATION_BUCKETS_S: [f64; 10] = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0]; +pub const DURATION_BUCKETS_S: &[f64] = + &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0]; /// Integer-count buckets for fan-out recipient histograms. -const FANOUT_BUCKETS: [f64; 9] = [0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0]; +pub const FANOUT_BUCKETS: &[f64] = &[0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0]; + +/// Scope name used when obtaining meters. +pub const METER_SCOPE: &str = "buzz-relay"; + +// ─── Global instrument handles ──────────────────────────────────────────────── + +/// Pre-built OTEL instrument handles. One global instance; zero per-call-site +/// allocation. Initialized by [`install`]; panics if accessed before that. +#[allow(missing_docs)] // field names are self-documenting metric names +pub struct Metrics { + // HTTP framework + pub http_requests_total: Counter, + pub http_request_latency_ms: Histogram, + + // WebSocket connections + pub ws_connections_total: Counter, + pub ws_connections_active: UpDownCounter, + pub ws_backpressure_disconnects_total: Counter, + pub ws_auth_timeouts_total: Counter, + + // Subscriptions + pub subscriptions_active: UpDownCounter, + + // Events + pub events_received_total: Counter, + pub events_stored_total: Counter, + pub events_rejected_total: Counter, + pub event_processing_seconds: Histogram, + + // Fan-out + pub fanout_recipients: Histogram, + pub multinode_fanout_total: Counter, + pub multinode_fanout_lag_total: Counter, + pub cache_invalidation_lag_total: Counter, + + // Search + pub search_index_seconds: Histogram, + pub search_index_errors_total: Counter, + + // Audit + pub audit_log_seconds: Histogram, + pub audit_log_errors_total: Counter, + pub audit_send_errors_total: Counter, -/// Install the global metrics recorder and spawn the Prometheus HTTP exporter. + // Auth + pub auth_attempts_total: Counter, + pub auth_failures_total: Counter, + + // Media + pub media_uploads_total: Counter, + pub media_upload_rejections_total: Counter, + + // Workflows + pub workflow_runs_total: Counter, + + // Cache + pub membership_cache_hits_total: Counter, + pub membership_cache_misses_total: Counter, + pub accessible_channels_cache_hits_total: Counter, + pub accessible_channels_cache_misses_total: Counter, + + // Count fallback + pub count_fallback_rejections_total: Counter, +} + +static METRICS: OnceLock = OnceLock::new(); + +/// Access the global [`Metrics`] instance. /// -/// `build()` returns the recorder + exporter future and internally spawns -/// the upkeep task, so no separate upkeep call is needed. +/// If [`install`] has not yet been called (e.g. in unit tests that don't +/// start a full relay), returns a lazily-initialised set of no-op instruments +/// built from the global meter (which is a no-op provider by default). +/// This matches the prior `metrics-rs` behaviour where macros silently +/// did nothing without explicit initialisation. +pub fn metrics() -> &'static Metrics { + METRICS.get_or_init(build_metrics) +} + +/// Build the [`Metrics`] instrument handles from the current global meter. +/// +/// Called once — either from [`install`] (real provider set) or lazily from +/// [`metrics`] (falls back to OTEL's built-in no-op meter). +fn build_metrics() -> Metrics { + let m = opentelemetry::global::meter(METER_SCOPE); + Metrics { + http_requests_total: m.u64_counter("http_requests_total").build(), + http_request_latency_ms: m.f64_histogram("http_request_latency_ms").build(), + + ws_connections_total: m.u64_counter("buzz_ws_connections_total").build(), + ws_connections_active: m.i64_up_down_counter("buzz_ws_connections_active").build(), + ws_backpressure_disconnects_total: m + .u64_counter("buzz_ws_backpressure_disconnects_total") + .build(), + ws_auth_timeouts_total: m.u64_counter("buzz_ws_auth_timeouts_total").build(), + + subscriptions_active: m.i64_up_down_counter("buzz_subscriptions_active").build(), + + events_received_total: m.u64_counter("buzz_events_received_total").build(), + events_stored_total: m.u64_counter("buzz_events_stored_total").build(), + events_rejected_total: m.u64_counter("buzz_events_rejected_total").build(), + event_processing_seconds: m.f64_histogram("buzz_event_processing_seconds").build(), + + fanout_recipients: m.f64_histogram("buzz_fanout_recipients").build(), + multinode_fanout_total: m.u64_counter("buzz_multinode_fanout_total").build(), + multinode_fanout_lag_total: m.u64_counter("buzz_multinode_fanout_lag_total").build(), + cache_invalidation_lag_total: m + .u64_counter("buzz_cache_invalidation_lag_total") + .build(), + + search_index_seconds: m.f64_histogram("buzz_search_index_seconds").build(), + search_index_errors_total: m.u64_counter("buzz_search_index_errors_total").build(), + + audit_log_seconds: m.f64_histogram("buzz_audit_log_seconds").build(), + audit_log_errors_total: m.u64_counter("buzz_audit_log_errors_total").build(), + audit_send_errors_total: m.u64_counter("buzz_audit_send_errors_total").build(), + + auth_attempts_total: m.u64_counter("buzz_auth_attempts_total").build(), + auth_failures_total: m.u64_counter("buzz_auth_failures_total").build(), + + media_uploads_total: m.u64_counter("buzz_media_uploads_total").build(), + media_upload_rejections_total: m + .u64_counter("buzz_media_upload_rejections_total") + .build(), + + workflow_runs_total: m.u64_counter("buzz_workflow_runs_total").build(), + + membership_cache_hits_total: m.u64_counter("buzz_membership_cache_hits_total").build(), + membership_cache_misses_total: m + .u64_counter("buzz_membership_cache_misses_total") + .build(), + accessible_channels_cache_hits_total: m + .u64_counter("buzz_accessible_channels_cache_hits_total") + .build(), + accessible_channels_cache_misses_total: m + .u64_counter("buzz_accessible_channels_cache_misses_total") + .build(), + + count_fallback_rejections_total: m + .u64_counter("buzz_count_fallback_rejections_total") + .build(), + } +} + +/// Returns a [`Meter`] scoped to the relay. Useful for one-off or dynamic +/// instruments (e.g. pool gauge task). +pub fn meter() -> Meter { + opentelemetry::global::meter(METER_SCOPE) +} + +// ─── Provider setup ─────────────────────────────────────────────────────────── + +/// Install the global OTEL meter provider and spawn the Prometheus HTTP exporter. +/// +/// Returns the [`SdkMeterProvider`] so the caller can shut it down gracefully +/// on SIGTERM. +/// +/// If `OTEL_EXPORTER_OTLP_ENDPOINT` is set, a second reader is attached that +/// pushes metrics via OTLP gRPC on the configured interval (default 60 s). +/// +/// # Panics +/// Panics if called more than once or if the HTTP listener cannot bind to `port`. +pub fn install(port: u16) -> SdkMeterProvider { + let registry = prometheus::Registry::new(); + + // Build the Prometheus exporter (pull-based: no periodic push needed). + let prom_exporter = opentelemetry_prometheus::exporter() + .with_registry(registry.clone()) + // Don't add unit suffixes — keep names identical to the old metrics-rs names. + .without_units() + // Don't add `_total` suffix — our counter names already end in `_total`. + .without_counter_suffixes() + // otel_scope_* labels add noise; the relay has a single scope. + .without_scope_info() + .build() + .expect("Prometheus exporter must build exactly once"); + + let mut provider_builder = SdkMeterProvider::builder() + .with_reader(prom_exporter) + .with_view(explicit_bucket_view); + + // Attach OTLP metric exporter only when the endpoint env var is set. + if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok() { + match opentelemetry_otlp::MetricExporter::builder() + .with_tonic() + .build() + { + Ok(exporter) => { + let periodic = PeriodicReader::builder(exporter).build(); + provider_builder = provider_builder.with_reader(periodic); + } + Err(e) => { + tracing::warn!(error = %e, "Failed to build OTLP metric exporter; OTLP metrics disabled"); + } + } + } + + let provider = provider_builder.build(); + + // Store globally so `global::meter(...)` works throughout the codebase. + // Must happen BEFORE build_metrics() so the real provider backs all handles. + opentelemetry::global::set_meter_provider(provider.clone()); + + // Build all instrument handles from the now-installed global meter. + // `get_or_init` is safe here: in production `install()` runs before any + // metric emission, so this is always the first init. If somehow called + // after a lazy noop init in tests, the noop handles are kept (acceptable + // for test isolation — the Prometheus endpoint test calls install() first). + METRICS.get_or_init(build_metrics); + + // Spawn the Prometheus HTTP listener on the metrics port. + tokio::spawn(serve_prometheus(port, registry)); + + provider +} + +// ─── View ───────────────────────────────────────────────────────────────────── + +/// OTEL View mapping named histograms to their explicit bucket boundaries. /// -/// Must be called from within a Tokio runtime. -/// Panics if a recorder is already installed or the port is in use. -pub fn install(port: u16) { - let (recorder, exporter) = PrometheusBuilder::new() - .with_http_listener(([0, 0, 0, 0], port)) - // Per-metric buckets: ms for HTTP latency, seconds for internal processing. - .set_buckets_for_metric( - Matcher::Full("http_request_latency_ms".to_owned()), - &LATENCY_BUCKETS_MS, - ) - .expect("valid ms bucket boundaries") - .set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), &DURATION_BUCKETS_S) - .expect("valid seconds bucket boundaries") - .set_buckets_for_metric( - Matcher::Full("buzz_fanout_recipients".to_owned()), - &FANOUT_BUCKETS, - ) - .expect("valid fanout bucket boundaries") +/// Any histogram whose name doesn't match uses the SDK default buckets. +fn explicit_bucket_view(inst: &Instrument) -> Option { + let boundaries: &[f64] = if inst.name() == "http_request_latency_ms" { + LATENCY_BUCKETS_MS + } else if inst.name() == "buzz_event_processing_seconds" + || inst.name() == "buzz_search_index_seconds" + || inst.name() == "buzz_audit_log_seconds" + { + DURATION_BUCKETS_S + } else if inst.name() == "buzz_fanout_recipients" { + FANOUT_BUCKETS + } else { + return None; // use SDK default + }; + + Stream::builder() + .with_aggregation(Aggregation::ExplicitBucketHistogram { + boundaries: boundaries.to_vec(), + record_min_max: false, + }) .build() - .expect("metrics exporter must build exactly once"); + .ok() +} + +// ─── Prometheus HTTP endpoint ───────────────────────────────────────────────── + +/// Serve the Prometheus `/metrics` endpoint on a bare TCP listener. +/// +/// This is intentionally minimal — no middleware, no auth. Port access controls +/// are expected to be enforced at the network/mesh level (Istio excludes this +/// port from the mesh by default in the Blox deployment). +async fn serve_prometheus(port: u16, registry: Registry) { + use axum::{routing::get, Router}; + use prometheus::{Encoder, TextEncoder}; + use std::net::SocketAddr; + + let app = Router::new().route( + "/metrics", + get(move || { + let reg = registry.clone(); + async move { + let encoder = TextEncoder::new(); + let families = reg.gather(); + let mut buf = Vec::new(); + encoder + .encode(&families, &mut buf) + .expect("encode prometheus metrics"); + let content_type = encoder.format_type().to_owned(); + ( + [( + axum::http::header::CONTENT_TYPE, + content_type, + )], + buf, + ) + } + }), + ); - metrics::set_global_recorder(recorder).expect("global recorder must be set exactly once"); - tokio::spawn(exporter); + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + let listener = tokio::net::TcpListener::bind(addr) + .await + .unwrap_or_else(|e| panic!("metrics listener failed to bind :{port}: {e}")); + axum::serve(listener, app).await.ok(); +} + +// ─── Cardinality control ────────────────────────────────────────────────────── + +/// Map arbitrary Nostr event kinds to a bounded label value. +/// +/// The allow-list matches the metrics observed in production; all other kinds +/// collapse to `"other"` to avoid cardinality explosion. +pub fn bounded_kind_label(kind: u16) -> &'static str { + match kind { + 0 => "0", // NIP-01 metadata + 1 => "1", // short text note + 3 => "3", // contact list + 4 => "4", // encrypted DM + 5 => "5", // deletion + 6 => "6", // repost + 7 => "7", // reaction + 40 => "40", // channel create + 41 => "41", // channel metadata + 42 => "42", // channel message + 43 => "43", // channel hide + 44 => "44", // channel mute + 1059 => "1059", // NIP-44 gift wrap + 1984 => "1984", // report + 9734 => "9734", // zap request + 9735 => "9735", // zap + 10000 => "10000", // mute list + 10001 => "10001", // pin list + _ => "other", + } } +// ─── HTTP metrics middleware ─────────────────────────────────────────────────── + /// Axum middleware that records CAKE framework HTTP metrics. /// /// Emits: @@ -117,9 +417,115 @@ pub async fn track_metrics(req: Request, next: Next) -> Response { let status = response.status().as_u16().to_string(); let latency_ms = start.elapsed().as_secs_f64() * 1000.0; - let labels = [("code", status), ("caller", caller), ("action", action)]; - metrics::counter!("http_requests_total", &labels).increment(1); - metrics::histogram!("http_request_latency_ms", &labels).record(latency_ms); + let labels = [ + KeyValue::new("code", status), + KeyValue::new("caller", caller), + KeyValue::new("action", action), + ]; + let m = metrics(); + m.http_requests_total.add(1, &labels); + m.http_request_latency_ms.record(latency_ms, &labels); response } + +// ─── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bounded_kind_label_known_kinds_are_stable() { + assert_eq!(bounded_kind_label(1), "1"); + assert_eq!(bounded_kind_label(42), "42"); + assert_eq!(bounded_kind_label(9735), "9735"); + } + + #[test] + fn bounded_kind_label_unknown_collapses_to_other() { + assert_eq!(bounded_kind_label(12345), "other"); + assert_eq!(bounded_kind_label(0xFFFF), "other"); + } + + /// Verify that the Prometheus HTTP endpoint returns 200 and includes + /// known metric names after installation. + /// + /// This test builds its own isolated Prometheus registry and meter + /// provider (not the global one) so it is fully independent of any + /// other test's metric state. + #[tokio::test] + async fn prometheus_endpoint_serves_known_metric_names() { + use opentelemetry_sdk::metrics::SdkMeterProvider; + use prometheus::Registry; + + // 1. Build an isolated registry + OTEL provider. + let registry = Registry::new(); + let exporter = opentelemetry_prometheus::exporter() + .with_registry(registry.clone()) + .without_units() + .without_counter_suffixes() + .without_scope_info() + .build() + .expect("build test prometheus exporter"); + + let provider = SdkMeterProvider::builder() + .with_reader(exporter) + .build(); + + // 2. Create a meter from this isolated provider (not the global one). + let meter = opentelemetry::metrics::MeterProvider::meter(&provider, METER_SCOPE); + + // 3. Build and record values for a sample of the relay's named instruments. + // These must appear in the Prometheus output. + let ws_conn = meter.u64_counter("buzz_ws_connections_total").build(); + let events_recv = meter.u64_counter("buzz_events_received_total").build(); + let events_stored = meter.u64_counter("buzz_events_stored_total").build(); + let auth_attempts = meter.u64_counter("buzz_auth_attempts_total").build(); + + ws_conn.add(1, &[]); + events_recv.add(1, &[KeyValue::new("kind", "1")]); + events_stored.add(1, &[KeyValue::new("kind", "42")]); + auth_attempts.add(1, &[KeyValue::new("method", "nip42")]); + + // 4. Bind port 0 → let OS pick a free port, then release so the + // listener can bind it. (Tiny TOCTOU gap acceptable in tests.) + let port = { + let sock = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral"); + sock.local_addr().expect("local_addr").port() + }; + + // 5. Spawn the Prometheus HTTP server with the isolated registry. + tokio::spawn(serve_prometheus(port, registry)); + + // Give the listener a moment to finish binding. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // 6. Fetch /metrics and verify the expected names are present. + let url = format!("http://127.0.0.1:{port}/metrics"); + let body = reqwest::get(&url) + .await + .expect("GET /metrics") + .error_for_status() + .expect("HTTP 200 from /metrics") + .text() + .await + .expect("read response body"); + + for expected in &[ + "buzz_ws_connections_total", + "buzz_events_received_total", + "buzz_events_stored_total", + "buzz_auth_attempts_total", + ] { + assert!( + body.contains(expected), + "/metrics body is missing '{expected}';\nbody:\n{body}", + ); + } + + // Keep the provider alive until the assertions complete so metrics + // aren't flushed/dropped before the HTTP response arrives. + drop(provider); + } +} diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index 51b98ddef1..963990aa8b 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -166,7 +166,7 @@ impl ConnectionManager { let count = conn.backpressure_count.fetch_add(1, Ordering::Relaxed) + 1; if count >= conn.grace_limit { tracing::warn!(conn_id = %conn_id, count, "fan-out: sustained backpressure — cancelling slow client"); - metrics::counter!("buzz_ws_backpressure_disconnects_total").increment(1); + crate::metrics::metrics().ws_backpressure_disconnects_total.add(1, &[]); conn.cancel.cancel(); } else { tracing::warn!(conn_id = %conn_id, count, grace = conn.grace_limit, "fan-out: send buffer full — grace {count}/{}", conn.grace_limit); @@ -466,10 +466,10 @@ impl AppState { ) -> Result { let key = (community_id, channel_id, pubkey.to_vec()); if let Some(cached) = self.membership_cache.get(&key) { - metrics::counter!("buzz_membership_cache_hits_total").increment(1); + crate::metrics::metrics().membership_cache_hits_total.add(1, &[]); return Ok(cached); } - metrics::counter!("buzz_membership_cache_misses_total").increment(1); + crate::metrics::metrics().membership_cache_misses_total.add(1, &[]); let result = self.db.is_member(community_id, channel_id, pubkey).await?; self.membership_cache.insert(key, result); Ok(result) @@ -598,10 +598,10 @@ impl AppState { ) -> Result, buzz_db::DbError> { let key = (community_id, pubkey.to_vec()); if let Some(cached) = self.accessible_channels_cache.get(&key) { - metrics::counter!("buzz_accessible_channels_cache_hits_total").increment(1); + crate::metrics::metrics().accessible_channels_cache_hits_total.add(1, &[]); return Ok(cached); } - metrics::counter!("buzz_accessible_channels_cache_misses_total").increment(1); + crate::metrics::metrics().accessible_channels_cache_misses_total.add(1, &[]); let result = self .db .get_accessible_channel_ids(community_id, pubkey) @@ -673,10 +673,12 @@ impl AuditShutdownHandle { async fn log_audit_entry(audit: &buzz_audit::AuditService, entry: buzz_audit::NewAuditEntry) { let t = std::time::Instant::now(); if let Err(e) = audit.log(entry).await { - metrics::counter!("buzz_audit_log_errors_total").increment(1); + crate::metrics::metrics().audit_log_errors_total.add(1, &[]); tracing::error!("Audit log failed: {e}"); } else { - metrics::histogram!("buzz_audit_log_seconds").record(t.elapsed().as_secs_f64()); + crate::metrics::metrics() + .audit_log_seconds + .record(t.elapsed().as_secs_f64(), &[]); } } diff --git a/crates/buzz-relay/src/subscription.rs b/crates/buzz-relay/src/subscription.rs index a46f593085..ff0ae32a53 100644 --- a/crates/buzz-relay/src/subscription.rs +++ b/crates/buzz-relay/src/subscription.rs @@ -80,7 +80,7 @@ impl SubscriptionRegistry { .entry(conn_id) .or_default() .insert(sub_id.clone(), (filters.clone(), community_id, channel_id)); - metrics::gauge!("buzz_subscriptions_active").increment(1.0); + crate::metrics::metrics().subscriptions_active.add(1, &[]); if let Some(ch_id) = channel_id { match extract_kinds_from_filters(&filters) { @@ -167,7 +167,7 @@ impl SubscriptionRegistry { if let Some(mut conn_subs) = self.subs.get_mut(&conn_id) { if let Some((filters, community_id, channel_id)) = conn_subs.remove(sub_id) { self.remove_from_index(conn_id, sub_id, &filters, community_id, channel_id); - metrics::gauge!("buzz_subscriptions_active").decrement(1.0); + crate::metrics::metrics().subscriptions_active.add(-1, &[]); return Some(RemovedSubscription { community_id, channel_id, @@ -189,7 +189,9 @@ impl SubscriptionRegistry { channel_id: *channel_id, }); } - metrics::gauge!("buzz_subscriptions_active").decrement(count as f64); + crate::metrics::metrics() + .subscriptions_active + .add(-(count as i64), &[]); } removed } @@ -261,6 +263,7 @@ impl SubscriptionRegistry { /// Return all (conn_id, sub_id) pairs whose filters match the given event in /// one server-resolved community. + #[tracing::instrument(skip_all)] pub fn fan_out_scoped( &self, community_id: CommunityId, diff --git a/crates/buzz-relay/src/telemetry.rs b/crates/buzz-relay/src/telemetry.rs new file mode 100644 index 0000000000..51bdbbd807 --- /dev/null +++ b/crates/buzz-relay/src/telemetry.rs @@ -0,0 +1,57 @@ +//! OpenTelemetry tracing initialisation. +//! +//! ```text +//! ┌────────────────────────────────────────────────────────────────────┐ +//! │ tracing crate (spans + events from #[instrument] and macros) │ +//! │ │ │ +//! │ ├── fmt::layer().json() → stdout (always on) │ +//! │ └── OpenTelemetryLayer (only when endpoint env var set) │ +//! │ ↓ │ +//! │ SdkTracerProvider + OTLP batch exporter │ +//! │ ↓ │ +//! │ OTLP gRPC → collector / Datadog Agent │ +//! └────────────────────────────────────────────────────────────────────┘ +//! ``` +//! +//! When `OTEL_EXPORTER_OTLP_ENDPOINT` is **unset** this module is a no-op: +//! the JSON stdout logs continue to work exactly as before and no OTLP +//! connection is attempted. +//! +//! Standard OTEL env vars honoured automatically by the SDK: +//! - `OTEL_SERVICE_NAME` (default: `buzz-relay`) +//! - `OTEL_RESOURCE_ATTRIBUTES` +//! - `OTEL_TRACES_SAMPLER` (default: `parentbased_always_on`) +//! - `OTEL_TRACES_SAMPLER_ARG` + +use opentelemetry_sdk::trace::SdkTracerProvider; + +/// Build and install the OTEL tracer provider, returning it so the caller +/// can register a shutdown hook. +/// +/// Returns `None` when `OTEL_EXPORTER_OTLP_ENDPOINT` is unset — in that +/// case the tracing subscriber stack is unchanged. +pub fn try_init_tracer() -> Option { + if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_err() { + return None; + } + + match opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .build() + { + Ok(exporter) => { + let provider = SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .build(); + opentelemetry::global::set_tracer_provider(provider.clone()); + Some(provider) + } + Err(e) => { + tracing::warn!( + error = %e, + "Failed to build OTLP trace exporter; distributed tracing disabled" + ); + None + } + } +} From 48de024ff9389ea2f636ee03ddbb5c2b3fda8b81 Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Tue, 30 Jun 2026 12:02:11 -0400 Subject: [PATCH 2/6] =?UTF-8?q?fix(relay):=20address=20Thufir=20review=20p?= =?UTF-8?q?ass=201=20=E2=80=94=20span=20topology,=20resource=20identity,?= =?UTF-8?q?=20metrics=20port=20bind?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Fixes ### IMPORTANT 1 — Prometheus bind failure now fails startup Bind the metrics TcpListener synchronously in install() before tokio::spawn. serve_prometheus() now accepts a pre-bound TcpListener instead of a port number. A port conflict panics at startup (matching prior behaviour) rather than silently dropping the metrics endpoint from a detached task. ### IMPORTANT 2 — OTLP service.name defaults to buzz-relay Build a single shared Resource via service_resource() in telemetry.rs using ResourceBuilder::with_service_name(buzz-relay) followed by with_detector(EnvResourceDetector) so that OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES still win when set. Both SdkTracerProvider and SdkMeterProvider receive the same Resource so traces and metrics correlate under the same service identity in Datadog. ### IMPORTANT 3 — Span topology: WS flow now produces one connected trace Create explicit parent spans in handle_text_message() for EVENT (ws.event), REQ (ws.req), COUNT (ws.count), and AUTH (ws.auth) messages, each carrying conn_id. Spawned handler futures are wrapped with .instrument(span) so the tracing context is not dropped at the tokio::spawn boundary. handle_event() and handle_auth() now call Span::current().record() to populate the event_id and kind/conn_id fields declared in their #[instrument] attributes. ### NIT 4 — target_info series suppressed for byte-parity Add .without_target_info() to the Prometheus exporter builder so the new Resource (non-empty after fix 2) does not inject a target_info series that the old metrics-rs endpoint never emitted. ### NIT 5 — BUZZ_POOL_METRICS_INTERVAL_SECS=0 no longer panics Clamp interval_secs to >= 1. tokio::time::interval(Duration::ZERO) panics; a config typo of 0 would have silently killed the pool metrics task. ### CI — cargo fmt drift Run cargo fmt --all to fix rustfmt line-wrapping across the migrated crate::metrics::metrics()..add(...) call sites. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- crates/buzz-relay/src/api/media.rs | 11 +-- crates/buzz-relay/src/connection.rs | 54 +++++++++++---- crates/buzz-relay/src/handlers/auth.rs | 26 ++++--- crates/buzz-relay/src/handlers/event.rs | 24 +++++-- crates/buzz-relay/src/lib.rs | 4 +- crates/buzz-relay/src/main.rs | 15 ++-- crates/buzz-relay/src/metrics.rs | 91 +++++++++++++------------ crates/buzz-relay/src/state.rs | 20 ++++-- crates/buzz-relay/src/telemetry.rs | 30 +++++++- 9 files changed, 185 insertions(+), 90 deletions(-) diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 4e421b6ba4..1b8fdb6608 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -312,9 +312,10 @@ pub async fn upload_blob( } _ => "other", }; - crate::metrics::metrics() - .media_uploads_total - .add(1, &[opentelemetry::KeyValue::new("mime", mime_label.to_string())]); + crate::metrics::metrics().media_uploads_total.add( + 1, + &[opentelemetry::KeyValue::new("mime", mime_label.to_string())], + ); // Audit via bounded channel — same pattern as event audit. let desc = descriptor.clone(); @@ -334,7 +335,9 @@ pub async fn upload_blob( .await { tracing::error!("Media audit channel closed — entry lost: {e}"); - crate::metrics::metrics().audit_send_errors_total.add(1, &[]); + crate::metrics::metrics() + .audit_send_errors_total + .add(1, &[]); } Ok(Json(descriptor)) diff --git a/crates/buzz-relay/src/connection.rs b/crates/buzz-relay/src/connection.rs index cbf5117782..2faf612bb3 100644 --- a/crates/buzz-relay/src/connection.rs +++ b/crates/buzz-relay/src/connection.rs @@ -10,6 +10,7 @@ use axum::extract::ws::{Message as WsMessage, WebSocket}; use futures_util::{SinkExt, StreamExt}; use tokio::sync::{mpsc, Mutex, RwLock}; use tokio_util::sync::CancellationToken; +use tracing::Instrument as _; use tracing::{debug, info, trace, warn}; use uuid::Uuid; @@ -92,7 +93,9 @@ impl ConnectionState { let count = self.backpressure_count.fetch_add(1, Ordering::Relaxed) + 1; if count >= self.grace_limit { warn!(conn_id = %self.conn_id, count, "sustained backpressure — closing slow client"); - crate::metrics::metrics().ws_backpressure_disconnects_total.add(1, &[]); + crate::metrics::metrics() + .ws_backpressure_disconnects_total + .add(1, &[]); self.cancel.cancel(); } else { warn!(conn_id = %self.conn_id, count, grace = self.grace_limit, "send buffer full — grace {count}/{}", self.grace_limit); @@ -423,7 +426,11 @@ async fn handle_text_message(text: String, conn: Arc, state: Ar match msg { ClientMessage::Auth(event) => { - handlers::auth::handle_auth(event, Arc::clone(&conn), Arc::clone(&state)).await; + // Auth is synchronous in the WS loop — no span context is lost. + let span = tracing::info_span!("ws.auth", conn_id = %conn.conn_id); + handlers::auth::handle_auth(event, Arc::clone(&conn), Arc::clone(&state)) + .instrument(span) + .await; } ClientMessage::Event(event) => { let conn = Arc::clone(&conn); @@ -437,10 +444,21 @@ async fn handle_text_message(text: String, conn: Arc, state: Ar return; } }; - tokio::spawn(async move { - handlers::event::handle_event(event, conn, state).await; - drop(permit); - }); + // Capture the parent span BEFORE the spawn so it is propagated into + // the spawned future. A bare `tokio::spawn` drops tracing context. + let span = tracing::info_span!( + "ws.event", + conn_id = %conn.conn_id, + event_id = tracing::field::Empty, + kind = tracing::field::Empty, + ); + tokio::spawn( + async move { + handlers::event::handle_event(event, conn, state).await; + drop(permit); + } + .instrument(span), + ); } ClientMessage::Req { sub_id, filters } => { let conn = Arc::clone(&conn); @@ -454,10 +472,14 @@ async fn handle_text_message(text: String, conn: Arc, state: Ar return; } }; - tokio::spawn(async move { - handlers::req::handle_req(sub_id, filters, conn, state).await; - drop(permit); - }); + let span = tracing::info_span!("ws.req", conn_id = %conn.conn_id, sub_id = %sub_id); + tokio::spawn( + async move { + handlers::req::handle_req(sub_id, filters, conn, state).await; + drop(permit); + } + .instrument(span), + ); } ClientMessage::Count { sub_id, filters } => { let conn = Arc::clone(&conn); @@ -471,10 +493,14 @@ async fn handle_text_message(text: String, conn: Arc, state: Ar return; } }; - tokio::spawn(async move { - handlers::count::handle_count(sub_id, filters, conn, state).await; - drop(permit); - }); + let span = tracing::info_span!("ws.count", conn_id = %conn.conn_id, sub_id = %sub_id); + tokio::spawn( + async move { + handlers::count::handle_count(sub_id, filters, conn, state).await; + drop(permit); + } + .instrument(span), + ); } ClientMessage::Close(sub_id) => { handlers::close::handle_close(sub_id, Arc::clone(&conn), Arc::clone(&state)).await; diff --git a/crates/buzz-relay/src/handlers/auth.rs b/crates/buzz-relay/src/handlers/auth.rs index a6038d9f26..6aa5ddd5ef 100644 --- a/crates/buzz-relay/src/handlers/auth.rs +++ b/crates/buzz-relay/src/handlers/auth.rs @@ -66,6 +66,11 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: } }; + // Record the declared span fields now that we have the values. + tracing::Span::current() + .record("event_id", event_id_hex.as_str()) + .record("conn_id", conn_id.to_string().as_str()); + // Extract the NIP-OA auth tag before verification consumes the event. // The tag is integrity-protected by the event's Schnorr signature — if // tampered, NIP-42 verification will fail before we ever inspect it. @@ -105,9 +110,10 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: }; if !allowed { warn!(conn_id = %conn_id, pubkey = %pubkey.to_hex(), "pubkey not in allowlist"); - crate::metrics::metrics() - .auth_failures_total - .add(1, &[opentelemetry::KeyValue::new("reason", "allowlist_denied")]); + crate::metrics::metrics().auth_failures_total.add( + 1, + &[opentelemetry::KeyValue::new("reason", "allowlist_denied")], + ); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, @@ -130,9 +136,10 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: Ok(owner) => owner, Err(e) => { warn!(conn_id = %conn_id, pubkey = %pubkey.to_hex(), error = ?e, "not a relay member"); - crate::metrics::metrics() - .auth_failures_total - .add(1, &[opentelemetry::KeyValue::new("reason", "not_relay_member")]); + crate::metrics::metrics().auth_failures_total.add( + 1, + &[opentelemetry::KeyValue::new("reason", "not_relay_member")], + ); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, @@ -246,9 +253,10 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: } Err(e) => { warn!(conn_id = %conn_id, error = %e, "NIP-42 auth failed"); - crate::metrics::metrics() - .auth_failures_total - .add(1, &[opentelemetry::KeyValue::new("reason", "nip42_invalid")]); + crate::metrics::metrics().auth_failures_total.add( + 1, + &[opentelemetry::KeyValue::new("reason", "nip42_invalid")], + ); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index f0ee1044b3..4463b937da 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -363,7 +363,9 @@ pub(crate) async fn dispatch_persistent_event( }; if let Err(e) = state.audit_tx.send(audit_entry).await { error!(event_id = %event_id_hex, "Audit channel closed — entry lost: {e}"); - crate::metrics::metrics().audit_send_errors_total.add(1, &[]); + crate::metrics::metrics() + .audit_send_errors_total + .add(1, &[]); } // Skip workflow triggering for workflow-execution kinds and relay-signed workflow messages. @@ -415,10 +417,17 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc, state: Arc { if result.accepted { - crate::metrics::metrics() - .events_stored_total - .add(1, &[opentelemetry::KeyValue::new("kind", kind_str.to_string())]); + crate::metrics::metrics().events_stored_total.add( + 1, + &[opentelemetry::KeyValue::new("kind", kind_str.to_string())], + ); info!( event_id = %result.event_id, kind = kind_u32, diff --git a/crates/buzz-relay/src/lib.rs b/crates/buzz-relay/src/lib.rs index 48bbf8a7ff..088177e4de 100644 --- a/crates/buzz-relay/src/lib.rs +++ b/crates/buzz-relay/src/lib.rs @@ -33,10 +33,10 @@ pub mod router; pub mod state; /// Subscription registry with (channel, kind) fan-out index. pub mod subscription; -/// Row-zero host binding: resolve the request community from the connection host. -pub mod tenant; /// OpenTelemetry tracing initialisation (tracer provider + OTLP exporter). pub mod telemetry; +/// Row-zero host binding: resolve the request community from the connection host. +pub mod tenant; /// Webhook secret generation and constant-time comparison. pub mod webhook_secret; /// Workflow action sink — relay-side implementation of [`buzz_workflow::ActionSink`]. diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index c0d5def615..3af426243f 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -31,7 +31,12 @@ async fn main() -> anyhow::Result<()> { // JSON-only structured logs — simple, machine-parseable, CAKE-compatible. // If OTEL_EXPORTER_OTLP_ENDPOINT is set, also attach an OpenTelemetry tracing // layer that exports spans via OTLP gRPC alongside the JSON stdout logs. - let tracer_provider = telemetry::try_init_tracer(); + // + // Build a single shared Resource (service.name=buzz-relay by default, overridable + // via OTEL_SERVICE_NAME) used by both the trace and metric providers so that + // Datadog can correlate spans and metrics under the same service identity. + let resource = telemetry::service_resource(); + let tracer_provider = telemetry::try_init_tracer(resource.clone()); let otel_layer = tracer_provider.as_ref().map(|p| { use opentelemetry::trace::TracerProvider as _; tracing_opentelemetry::layer().with_tracer(p.tracer("buzz-relay")) @@ -58,7 +63,7 @@ async fn main() -> anyhow::Result<()> { "Config loaded" ); - let meter_provider = relay_metrics::install(config.metrics_port); + let meter_provider = relay_metrics::install(config.metrics_port, resource); info!( port = config.metrics_port, "Prometheus metrics exporter started" @@ -724,7 +729,8 @@ async fn main() -> anyhow::Result<()> { let interval_secs = std::env::var("BUZZ_POOL_METRICS_INTERVAL_SECS") .ok() .and_then(|v| v.parse::().ok()) - .unwrap_or(10); + .unwrap_or(10) + .max(1); // tokio::time::interval panics on Duration::ZERO tokio::spawn(async move { let m = relay_metrics::meter(); let db_pool_size = m.u64_gauge("buzz_db_pool_size").build(); @@ -735,8 +741,7 @@ async fn main() -> anyhow::Result<()> { let redis_pool_max = m.u64_gauge("buzz_redis_pool_max").build(); let redis_pool_waiting = m.u64_gauge("buzz_redis_pool_waiting").build(); - let mut interval = - tokio::time::interval(std::time::Duration::from_secs(interval_secs)); + let mut interval = tokio::time::interval(std::time::Duration::from_secs(interval_secs)); loop { interval.tick().await; let db_stats = pool_state.db.pool_stats(); diff --git a/crates/buzz-relay/src/metrics.rs b/crates/buzz-relay/src/metrics.rs index 0aa684625a..6715b60fd3 100644 --- a/crates/buzz-relay/src/metrics.rs +++ b/crates/buzz-relay/src/metrics.rs @@ -31,7 +31,9 @@ use opentelemetry::{ metrics::{Counter, Histogram, Meter, UpDownCounter}, KeyValue, }; -use opentelemetry_sdk::metrics::{Aggregation, Instrument, PeriodicReader, SdkMeterProvider, Stream}; +use opentelemetry_sdk::metrics::{ + Aggregation, Instrument, PeriodicReader, SdkMeterProvider, Stream, +}; use prometheus::Registry; // ─── Bucket boundaries ─────────────────────────────────────────────────────── @@ -42,8 +44,7 @@ pub const LATENCY_BUCKETS_MS: &[f64] = &[ ]; /// Seconds-scale buckets for internal processing histograms (event, search, audit). -pub const DURATION_BUCKETS_S: &[f64] = - &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0]; +pub const DURATION_BUCKETS_S: &[f64] = &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0]; /// Integer-count buckets for fan-out recipient histograms. pub const FANOUT_BUCKETS: &[f64] = &[0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0]; @@ -152,9 +153,7 @@ fn build_metrics() -> Metrics { fanout_recipients: m.f64_histogram("buzz_fanout_recipients").build(), multinode_fanout_total: m.u64_counter("buzz_multinode_fanout_total").build(), multinode_fanout_lag_total: m.u64_counter("buzz_multinode_fanout_lag_total").build(), - cache_invalidation_lag_total: m - .u64_counter("buzz_cache_invalidation_lag_total") - .build(), + cache_invalidation_lag_total: m.u64_counter("buzz_cache_invalidation_lag_total").build(), search_index_seconds: m.f64_histogram("buzz_search_index_seconds").build(), search_index_errors_total: m.u64_counter("buzz_search_index_errors_total").build(), @@ -167,16 +166,12 @@ fn build_metrics() -> Metrics { auth_failures_total: m.u64_counter("buzz_auth_failures_total").build(), media_uploads_total: m.u64_counter("buzz_media_uploads_total").build(), - media_upload_rejections_total: m - .u64_counter("buzz_media_upload_rejections_total") - .build(), + media_upload_rejections_total: m.u64_counter("buzz_media_upload_rejections_total").build(), workflow_runs_total: m.u64_counter("buzz_workflow_runs_total").build(), membership_cache_hits_total: m.u64_counter("buzz_membership_cache_hits_total").build(), - membership_cache_misses_total: m - .u64_counter("buzz_membership_cache_misses_total") - .build(), + membership_cache_misses_total: m.u64_counter("buzz_membership_cache_misses_total").build(), accessible_channels_cache_hits_total: m .u64_counter("buzz_accessible_channels_cache_hits_total") .build(), @@ -208,9 +203,17 @@ pub fn meter() -> Meter { /// /// # Panics /// Panics if called more than once or if the HTTP listener cannot bind to `port`. -pub fn install(port: u16) -> SdkMeterProvider { +pub fn install(port: u16, resource: opentelemetry_sdk::Resource) -> SdkMeterProvider { let registry = prometheus::Registry::new(); + // Bind synchronously so that a port conflict fails startup immediately, + // matching the prior `metrics-exporter-prometheus` behaviour. + let listener = { + let addr = std::net::SocketAddr::from(([0, 0, 0, 0], port)); + std::net::TcpListener::bind(addr) + .unwrap_or_else(|e| panic!("metrics listener failed to bind :{port}: {e}")) + }; + // Build the Prometheus exporter (pull-based: no periodic push needed). let prom_exporter = opentelemetry_prometheus::exporter() .with_registry(registry.clone()) @@ -220,10 +223,15 @@ pub fn install(port: u16) -> SdkMeterProvider { .without_counter_suffixes() // otel_scope_* labels add noise; the relay has a single scope. .without_scope_info() + // Suppress the `target_info` series added by OTEL for resource attributes. + // The old metrics-rs endpoint did not emit it; excluding it keeps byte-parity + // so the Datadog openmetrics annotation needs no changes. + .without_target_info() .build() .expect("Prometheus exporter must build exactly once"); let mut provider_builder = SdkMeterProvider::builder() + .with_resource(resource) .with_reader(prom_exporter) .with_view(explicit_bucket_view); @@ -256,8 +264,15 @@ pub fn install(port: u16) -> SdkMeterProvider { // for test isolation — the Prometheus endpoint test calls install() first). METRICS.get_or_init(build_metrics); - // Spawn the Prometheus HTTP listener on the metrics port. - tokio::spawn(serve_prometheus(port, registry)); + // Convert the synchronously-bound std listener and spawn the HTTP server. + // The bind already succeeded above; non-blocking mode is required before + // handing the socket to tokio. + listener + .set_nonblocking(true) + .expect("set metrics listener non-blocking"); + let async_listener = + tokio::net::TcpListener::from_std(listener).expect("convert std TcpListener to tokio"); + tokio::spawn(serve_prometheus(async_listener, registry)); provider } @@ -292,15 +307,17 @@ fn explicit_bucket_view(inst: &Instrument) -> Option { // ─── Prometheus HTTP endpoint ───────────────────────────────────────────────── -/// Serve the Prometheus `/metrics` endpoint on a bare TCP listener. +/// Serve the Prometheus `/metrics` endpoint on a pre-bound TCP listener. +/// +/// Accepts a listener that has already been bound synchronously in [`install`] +/// so that port-in-use errors surface at startup, not inside a detached task. /// /// This is intentionally minimal — no middleware, no auth. Port access controls /// are expected to be enforced at the network/mesh level (Istio excludes this /// port from the mesh by default in the Blox deployment). -async fn serve_prometheus(port: u16, registry: Registry) { +async fn serve_prometheus(listener: tokio::net::TcpListener, registry: Registry) { use axum::{routing::get, Router}; use prometheus::{Encoder, TextEncoder}; - use std::net::SocketAddr; let app = Router::new().route( "/metrics", @@ -314,21 +331,11 @@ async fn serve_prometheus(port: u16, registry: Registry) { .encode(&families, &mut buf) .expect("encode prometheus metrics"); let content_type = encoder.format_type().to_owned(); - ( - [( - axum::http::header::CONTENT_TYPE, - content_type, - )], - buf, - ) + ([(axum::http::header::CONTENT_TYPE, content_type)], buf) } }), ); - let addr = SocketAddr::from(([0, 0, 0, 0], port)); - let listener = tokio::net::TcpListener::bind(addr) - .await - .unwrap_or_else(|e| panic!("metrics listener failed to bind :{port}: {e}")); axum::serve(listener, app).await.ok(); } @@ -466,12 +473,11 @@ mod tests { .without_units() .without_counter_suffixes() .without_scope_info() + .without_target_info() .build() .expect("build test prometheus exporter"); - let provider = SdkMeterProvider::builder() - .with_reader(exporter) - .build(); + let provider = SdkMeterProvider::builder().with_reader(exporter).build(); // 2. Create a meter from this isolated provider (not the global one). let meter = opentelemetry::metrics::MeterProvider::meter(&provider, METER_SCOPE); @@ -488,18 +494,19 @@ mod tests { events_stored.add(1, &[KeyValue::new("kind", "42")]); auth_attempts.add(1, &[KeyValue::new("method", "nip42")]); - // 4. Bind port 0 → let OS pick a free port, then release so the - // listener can bind it. (Tiny TOCTOU gap acceptable in tests.) - let port = { - let sock = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral"); - sock.local_addr().expect("local_addr").port() - }; + // 4. Bind synchronously (matching install() behaviour) then convert. + let std_listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral"); + let port = std_listener.local_addr().expect("local_addr").port(); + // tokio requires non-blocking mode before from_std(). + std_listener.set_nonblocking(true).expect("set_nonblocking"); + let listener = + tokio::net::TcpListener::from_std(std_listener).expect("convert to tokio listener"); - // 5. Spawn the Prometheus HTTP server with the isolated registry. - tokio::spawn(serve_prometheus(port, registry)); + // 5. Spawn the Prometheus HTTP server with the pre-bound listener. + tokio::spawn(serve_prometheus(listener, registry)); - // Give the listener a moment to finish binding. - tokio::time::sleep(std::time::Duration::from_millis(50)).await; + // Yield to let the spawned Axum task start accepting connections. + tokio::task::yield_now().await; // 6. Fetch /metrics and verify the expected names are present. let url = format!("http://127.0.0.1:{port}/metrics"); diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index 963990aa8b..d5d3569732 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -166,7 +166,9 @@ impl ConnectionManager { let count = conn.backpressure_count.fetch_add(1, Ordering::Relaxed) + 1; if count >= conn.grace_limit { tracing::warn!(conn_id = %conn_id, count, "fan-out: sustained backpressure — cancelling slow client"); - crate::metrics::metrics().ws_backpressure_disconnects_total.add(1, &[]); + crate::metrics::metrics() + .ws_backpressure_disconnects_total + .add(1, &[]); conn.cancel.cancel(); } else { tracing::warn!(conn_id = %conn_id, count, grace = conn.grace_limit, "fan-out: send buffer full — grace {count}/{}", conn.grace_limit); @@ -466,10 +468,14 @@ impl AppState { ) -> Result { let key = (community_id, channel_id, pubkey.to_vec()); if let Some(cached) = self.membership_cache.get(&key) { - crate::metrics::metrics().membership_cache_hits_total.add(1, &[]); + crate::metrics::metrics() + .membership_cache_hits_total + .add(1, &[]); return Ok(cached); } - crate::metrics::metrics().membership_cache_misses_total.add(1, &[]); + crate::metrics::metrics() + .membership_cache_misses_total + .add(1, &[]); let result = self.db.is_member(community_id, channel_id, pubkey).await?; self.membership_cache.insert(key, result); Ok(result) @@ -598,10 +604,14 @@ impl AppState { ) -> Result, buzz_db::DbError> { let key = (community_id, pubkey.to_vec()); if let Some(cached) = self.accessible_channels_cache.get(&key) { - crate::metrics::metrics().accessible_channels_cache_hits_total.add(1, &[]); + crate::metrics::metrics() + .accessible_channels_cache_hits_total + .add(1, &[]); return Ok(cached); } - crate::metrics::metrics().accessible_channels_cache_misses_total.add(1, &[]); + crate::metrics::metrics() + .accessible_channels_cache_misses_total + .add(1, &[]); let result = self .db .get_accessible_channel_ids(community_id, pubkey) diff --git a/crates/buzz-relay/src/telemetry.rs b/crates/buzz-relay/src/telemetry.rs index 51bdbbd807..489b3c1cdd 100644 --- a/crates/buzz-relay/src/telemetry.rs +++ b/crates/buzz-relay/src/telemetry.rs @@ -23,14 +23,39 @@ //! - `OTEL_TRACES_SAMPLER` (default: `parentbased_always_on`) //! - `OTEL_TRACES_SAMPLER_ARG` -use opentelemetry_sdk::trace::SdkTracerProvider; +use opentelemetry_sdk::{resource::EnvResourceDetector, trace::SdkTracerProvider, Resource}; + +/// Build the OTEL [`Resource`] shared by the trace and metric providers. +/// +/// Strategy (priority order): +/// 1. `OTEL_SERVICE_NAME` env var (standard OTEL env, read by [`EnvResourceDetector`]) +/// 2. `service.name` in `OTEL_RESOURCE_ATTRIBUTES` env var (also read by env detector) +/// 3. Hard-coded fallback `buzz-relay` +/// +/// The env detector is run first; its attributes win on merge. The fallback +/// resource only provides `service.name` when neither env var sets it, so +/// user-supplied values are always respected. +/// +/// Both the tracer provider (traces) and the meter provider (metrics) receive +/// the same `Resource` instance so Datadog can correlate spans and metrics on +/// the same `service.name`. +pub fn service_resource() -> Resource { + // Start with buzz-relay as the service.name fallback, then overlay the env + // detector. The builder's `with_detector` call passes the detector output + // as the "other" in merge(), so env-supplied values (OTEL_SERVICE_NAME, + // OTEL_RESOURCE_ATTRIBUTES) always win over the fallback. + Resource::builder_empty() + .with_service_name("buzz-relay") + .with_detector(Box::new(EnvResourceDetector::new())) + .build() +} /// Build and install the OTEL tracer provider, returning it so the caller /// can register a shutdown hook. /// /// Returns `None` when `OTEL_EXPORTER_OTLP_ENDPOINT` is unset — in that /// case the tracing subscriber stack is unchanged. -pub fn try_init_tracer() -> Option { +pub fn try_init_tracer(resource: Resource) -> Option { if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_err() { return None; } @@ -41,6 +66,7 @@ pub fn try_init_tracer() -> Option { { Ok(exporter) => { let provider = SdkTracerProvider::builder() + .with_resource(resource) .with_batch_exporter(exporter) .build(); opentelemetry::global::set_tracer_provider(provider.clone()); From ab043cfb4a5d773ef270983e386e1658529880c3 Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Tue, 30 Jun 2026 13:18:49 -0400 Subject: [PATCH 3/6] fix(telemetry): honor OTEL_SERVICE_NAME explicitly in service_resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EnvResourceDetector reads OTEL_RESOURCE_ATTRIBUTES only, not OTEL_SERVICE_NAME (opentelemetry_sdk 0.32.1 resource/env.rs:23). SdkProvidedResourceDetector does read OTEL_SERVICE_NAME but always emits a service.name key, falling back to unknown_service: when unset — which would clobber the buzz-relay default. Read OTEL_SERVICE_NAME explicitly: non-empty value wins over the buzz-relay fallback; OTEL_RESOURCE_ATTRIBUTES (via EnvResourceDetector overlaid last) still wins over OTEL_SERVICE_NAME per OTEL spec. Correct the module and function doc comments that claimed the SDK detector handled OTEL_SERVICE_NAME automatically. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- crates/buzz-relay/src/telemetry.rs | 36 ++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/crates/buzz-relay/src/telemetry.rs b/crates/buzz-relay/src/telemetry.rs index 489b3c1cdd..69f524bb40 100644 --- a/crates/buzz-relay/src/telemetry.rs +++ b/crates/buzz-relay/src/telemetry.rs @@ -17,9 +17,9 @@ //! the JSON stdout logs continue to work exactly as before and no OTLP //! connection is attempted. //! -//! Standard OTEL env vars honoured automatically by the SDK: -//! - `OTEL_SERVICE_NAME` (default: `buzz-relay`) -//! - `OTEL_RESOURCE_ATTRIBUTES` +//! Standard OTEL env vars honoured: +//! - `OTEL_SERVICE_NAME` (default: `buzz-relay`; read explicitly — not via SDK detector) +//! - `OTEL_RESOURCE_ATTRIBUTES` (overlaid by [`EnvResourceDetector`]) //! - `OTEL_TRACES_SAMPLER` (default: `parentbased_always_on`) //! - `OTEL_TRACES_SAMPLER_ARG` @@ -28,24 +28,32 @@ use opentelemetry_sdk::{resource::EnvResourceDetector, trace::SdkTracerProvider, /// Build the OTEL [`Resource`] shared by the trace and metric providers. /// /// Strategy (priority order): -/// 1. `OTEL_SERVICE_NAME` env var (standard OTEL env, read by [`EnvResourceDetector`]) -/// 2. `service.name` in `OTEL_RESOURCE_ATTRIBUTES` env var (also read by env detector) -/// 3. Hard-coded fallback `buzz-relay` +/// 1. `service.name` in `OTEL_RESOURCE_ATTRIBUTES` — overlaid last by +/// [`EnvResourceDetector`], wins over everything below. +/// 2. `OTEL_SERVICE_NAME` — read explicitly (non-empty wins over the fallback). +/// 3. Hard-coded fallback `buzz-relay`. /// -/// The env detector is run first; its attributes win on merge. The fallback -/// resource only provides `service.name` when neither env var sets it, so -/// user-supplied values are always respected. +/// Note: [`EnvResourceDetector`] only reads `OTEL_RESOURCE_ATTRIBUTES`; it +/// does **not** read `OTEL_SERVICE_NAME`. `SdkProvidedResourceDetector` does +/// read `OTEL_SERVICE_NAME` but always emits a `service.name` key (falling +/// back to `unknown_service:` when unset), which would clobber our +/// `buzz-relay` default. We therefore read `OTEL_SERVICE_NAME` explicitly +/// so the fallback is fully under our control. /// /// Both the tracer provider (traces) and the meter provider (metrics) receive /// the same `Resource` instance so Datadog can correlate spans and metrics on /// the same `service.name`. pub fn service_resource() -> Resource { - // Start with buzz-relay as the service.name fallback, then overlay the env - // detector. The builder's `with_detector` call passes the detector output - // as the "other" in merge(), so env-supplied values (OTEL_SERVICE_NAME, - // OTEL_RESOURCE_ATTRIBUTES) always win over the fallback. + // Honor OTEL_SERVICE_NAME when set+non-empty; otherwise use buzz-relay. + // EnvResourceDetector overlays OTEL_RESOURCE_ATTRIBUTES last, so an + // explicit service.name there still wins over OTEL_SERVICE_NAME per spec. + let service_name = std::env::var("OTEL_SERVICE_NAME") + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "buzz-relay".to_string()); + Resource::builder_empty() - .with_service_name("buzz-relay") + .with_service_name(service_name) .with_detector(Box::new(EnvResourceDetector::new())) .build() } From df8736e5ddb843c53ea13a3a015b4dfdab610c9e Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Tue, 30 Jun 2026 14:29:59 -0400 Subject: [PATCH 4/6] refactor(relay): revert OTEL metrics to metrics-rs + Prometheus scrape, keep OTLP tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the OpenTelemetry metrics exporter (opentelemetry-prometheus, prometheus crate, OTLP metric push) in favour of the original metrics-rs facade (metrics::counter!/gauge!/histogram! macros) backed by metrics-exporter-prometheus. OTLP tracing (telemetry.rs, tracing-opentelemetry, OTEL trace spans in connection/event/auth handlers) is intentionally preserved — only the metrics path is reverted. Changes: - Cargo.toml: restore metrics + metrics-exporter-prometheus workspace deps; strip metrics/logs features from opentelemetry, opentelemetry_sdk, opentelemetry-otlp, tracing-opentelemetry; remove opentelemetry-prometheus and prometheus = "0.14" - metrics.rs: restore PrometheusBuilder-based setup verbatim from origin/main - All call sites: crate::metrics::metrics()..add/record -> metrics-rs macros (counter!/gauge!/histogram!) across connection, auth, event, count, state, subscription, bridge, media - main.rs: fanout_lag + cache_inval_lag background consumers converted to metrics::counter!; pool gauge section (net-new, kept per spec) converted from relay_metrics::meter() OTEL API to metrics::gauge!.set(); meter_provider shutdown removed; telemetry comment updated to trace-only - telemetry.rs: doc comment updated — Resource is for trace provider only - Cargo.lock: opentelemetry-prometheus and prometheus crates removed Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- Cargo.lock | 179 ++++++--- Cargo.toml | 11 +- crates/buzz-relay/Cargo.toml | 4 +- crates/buzz-relay/src/api/bridge.rs | 8 +- crates/buzz-relay/src/api/media.rs | 19 +- crates/buzz-relay/src/connection.rs | 12 +- crates/buzz-relay/src/handlers/auth.rs | 21 +- crates/buzz-relay/src/handlers/count.rs | 8 +- crates/buzz-relay/src/handlers/event.rs | 38 +- crates/buzz-relay/src/main.rs | 44 +-- crates/buzz-relay/src/metrics.rs | 497 ++---------------------- crates/buzz-relay/src/state.rs | 26 +- crates/buzz-relay/src/subscription.rs | 8 +- crates/buzz-relay/src/telemetry.rs | 7 +- 14 files changed, 237 insertions(+), 645 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5c1a112d3..e1ff60bfb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1053,14 +1053,14 @@ dependencies = [ "infer", "mesh-llm-host-runtime", "mesh-llm-sdk", + "metrics", + "metrics-exporter-prometheus", "moka", "nostr", "opentelemetry 0.32.0", "opentelemetry-otlp 0.32.0", - "opentelemetry-prometheus", "opentelemetry-semantic-conventions", "opentelemetry_sdk 0.32.1", - "prometheus", "rand 0.10.1", "redis", "reqwest 0.13.3", @@ -2399,6 +2399,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -2838,6 +2849,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + [[package]] name = "hashbrown" version = "0.14.5" @@ -3185,6 +3202,7 @@ dependencies = [ "hyper", "hyper-util", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -3855,6 +3873,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "left-right" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0c21e4c8ff95f487fb34e6f9182875f42c84cef966d29216bf115d9bba835a" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "libc" version = "0.2.186" @@ -4465,6 +4494,56 @@ dependencies = [ "tracing", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64", + "evmap", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "indexmap", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "rustls", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.4", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -5411,19 +5490,6 @@ dependencies = [ "tonic-types", ] -[[package]] -name = "opentelemetry-prometheus" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0359983e7f79cf33c9abd89e5d7ddf67c46c419d0148598022d70e70c01aba" -dependencies = [ - "once_cell", - "opentelemetry 0.32.0", - "opentelemetry_sdk 0.32.1", - "prometheus", - "tracing", -] - [[package]] name = "opentelemetry-proto" version = "0.31.0" @@ -5939,21 +6005,6 @@ dependencies = [ "windows 0.62.2", ] -[[package]] -name = "prometheus" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" -dependencies = [ - "cfg-if 1.0.4", - "fnv", - "lazy_static", - "memchr", - "parking_lot", - "protobuf", - "thiserror 2.0.18", -] - [[package]] name = "proptest" version = "1.11.0" @@ -6024,26 +6075,6 @@ dependencies = [ "prost", ] -[[package]] -name = "protobuf" -version = "3.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" -dependencies = [ - "once_cell", - "protobuf-support", - "thiserror 1.0.69", -] - -[[package]] -name = "protobuf-support" -version = "3.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" -dependencies = [ - "thiserror 1.0.69", -] - [[package]] name = "protoc-bin-vendored" version = "3.2.0" @@ -6114,6 +6145,21 @@ version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -6316,6 +6362,33 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rapidhash" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "redb" version = "3.1.3" @@ -7316,6 +7389,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "skippy-cache" version = "0.68.0" diff --git a/Cargo.toml b/Cargo.toml index fd5a0f7d6b..8f972ce7b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,12 +67,13 @@ cron = "0.16" # Observability tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } -tracing-opentelemetry = { version = "0.33", features = ["metrics"] } -opentelemetry = { version = "0.32", features = ["trace", "metrics", "logs"] } -opentelemetry_sdk = { version = "0.32", features = ["trace", "metrics", "rt-tokio"] } -opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "metrics", "grpc-tonic", "tls-ring"] } -opentelemetry-prometheus = { version = "0.32" } +tracing-opentelemetry = { version = "0.33" } +opentelemetry = { version = "0.32", features = ["trace"] } +opentelemetry_sdk = { version = "0.32", features = ["trace", "rt-tokio"] } +opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "grpc-tonic", "tls-ring"] } opentelemetry-semantic-conventions = { version = "0.32" } +metrics = "0.24" +metrics-exporter-prometheus = "0.18" # Error handling thiserror = "2" diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index 594863099c..3bde131ff5 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -40,9 +40,7 @@ tracing-opentelemetry = { workspace = true } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } opentelemetry-otlp = { workspace = true } -opentelemetry-prometheus = { workspace = true } opentelemetry-semantic-conventions = { workspace = true } -prometheus = "0.14" thiserror = { workspace = true } anyhow = { workspace = true } uuid = { workspace = true } @@ -68,6 +66,8 @@ rand = { workspace = true } hex = { workspace = true } url = { workspace = true } moka = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } [features] dev = ["buzz-auth/dev"] diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index da82ff5efc..3882363d80 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -744,9 +744,7 @@ pub async fn count_events( match state.db.query_events(&q).await { Ok(stored_events) => { if crate::handlers::req::count_fallback_exceeded(stored_events.len()) { - crate::metrics::metrics() - .count_fallback_rejections_total - .add(1, &[]); + metrics::counter!("buzz_count_fallback_rejections_total").increment(1); return Err(api_error( StatusCode::BAD_REQUEST, "count filter requires narrower constraints", @@ -803,9 +801,7 @@ pub async fn count_events( match state.db.query_events(&query).await { Ok(stored_events) => { if crate::handlers::req::count_fallback_exceeded(stored_events.len()) { - crate::metrics::metrics() - .count_fallback_rejections_total - .add(1, &[]); + metrics::counter!("buzz_count_fallback_rejections_total").increment(1); return Err(api_error( StatusCode::BAD_REQUEST, "count filter requires narrower constraints", diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 1b8fdb6608..bc7d9ed544 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -193,15 +193,13 @@ impl FromRequestParts> for AuthenticatedUpload { .map_err(|_| MediaError::RelayMembershipRequired)?; if upload_rate_limited(state, &auth_event.pubkey) { - crate::metrics::metrics() - .media_upload_rejections_total - .add(1, &[opentelemetry::KeyValue::new("reason", "rate_limit")]); + metrics::counter!("buzz_media_upload_rejections_total", "reason" => "rate_limit") + .increment(1); return Err(MediaError::UploadRateLimitExceeded); } let upload_permit = acquire_upload_permit(state, &auth_event.pubkey).inspect_err(|_| { - crate::metrics::metrics() - .media_upload_rejections_total - .add(1, &[opentelemetry::KeyValue::new("reason", "concurrency")]); + metrics::counter!("buzz_media_upload_rejections_total", "reason" => "concurrency") + .increment(1); })?; Ok(AuthenticatedUpload { @@ -312,10 +310,7 @@ pub async fn upload_blob( } _ => "other", }; - crate::metrics::metrics().media_uploads_total.add( - 1, - &[opentelemetry::KeyValue::new("mime", mime_label.to_string())], - ); + metrics::counter!("buzz_media_uploads_total", "mime" => mime_label.to_owned()).increment(1); // Audit via bounded channel — same pattern as event audit. let desc = descriptor.clone(); @@ -335,9 +330,7 @@ pub async fn upload_blob( .await { tracing::error!("Media audit channel closed — entry lost: {e}"); - crate::metrics::metrics() - .audit_send_errors_total - .add(1, &[]); + metrics::counter!("buzz_audit_send_errors_total").increment(1); } Ok(Json(descriptor)) diff --git a/crates/buzz-relay/src/connection.rs b/crates/buzz-relay/src/connection.rs index 2faf612bb3..a3e72414d9 100644 --- a/crates/buzz-relay/src/connection.rs +++ b/crates/buzz-relay/src/connection.rs @@ -93,9 +93,7 @@ impl ConnectionState { let count = self.backpressure_count.fetch_add(1, Ordering::Relaxed) + 1; if count >= self.grace_limit { warn!(conn_id = %self.conn_id, count, "sustained backpressure — closing slow client"); - crate::metrics::metrics() - .ws_backpressure_disconnects_total - .add(1, &[]); + metrics::counter!("buzz_ws_backpressure_disconnects_total").increment(1); self.cancel.cancel(); } else { warn!(conn_id = %self.conn_id, count, grace = self.grace_limit, "send buffer full — grace {count}/{}", self.grace_limit); @@ -156,7 +154,7 @@ pub async fn handle_connection( }); info!(conn_id = %conn_id, addr = %addr, "WebSocket connection established"); - crate::metrics::metrics().ws_connections_total.add(1, &[]); + metrics::counter!("buzz_ws_connections_total").increment(1); let challenge_msg = RelayMessage::auth_challenge(&challenge); if tx @@ -170,7 +168,7 @@ pub async fn handle_connection( // Gauge incremented AFTER challenge send succeeds — early disconnects // don't leak. Decremented in the cleanup path below. - crate::metrics::metrics().ws_connections_active.add(1, &[]); + metrics::gauge!("buzz_ws_connections_active").increment(1.0); // Register after challenge succeeds — avoids leaked entries on early disconnect. state.conn_manager.register( @@ -211,7 +209,7 @@ pub async fn handle_connection( timeout_secs = AUTH_TIMEOUT.as_secs(), "NIP-42 auth timeout — closing connection" ); - crate::metrics::metrics().ws_auth_timeouts_total.add(1, &[]); + metrics::counter!("buzz_ws_auth_timeouts_total").increment(1); auth_timeout_cancel.cancel(); } } @@ -251,7 +249,7 @@ pub async fn handle_connection( .await; } } - crate::metrics::metrics().ws_connections_active.add(-1, &[]); + metrics::gauge!("buzz_ws_connections_active").decrement(1.0); info!(conn_id = %conn_id, addr = %addr, "WebSocket connection closed"); drop(permit); diff --git a/crates/buzz-relay/src/handlers/auth.rs b/crates/buzz-relay/src/handlers/auth.rs index 6aa5ddd5ef..bc356979f5 100644 --- a/crates/buzz-relay/src/handlers/auth.rs +++ b/crates/buzz-relay/src/handlers/auth.rs @@ -80,9 +80,7 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: crate::api::bridge::nip42_expected_relay_url(&state.config.relay_url, &conn.tenant); let auth_svc = Arc::clone(&state.auth); - crate::metrics::metrics() - .auth_attempts_total - .add(1, &[opentelemetry::KeyValue::new("method", "nip42")]); + metrics::counter!("buzz_auth_attempts_total", "method" => "nip42").increment(1); // Pure NIP-42 verification — crypto only, no DB lookups. match auth_svc @@ -110,10 +108,8 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: }; if !allowed { warn!(conn_id = %conn_id, pubkey = %pubkey.to_hex(), "pubkey not in allowlist"); - crate::metrics::metrics().auth_failures_total.add( - 1, - &[opentelemetry::KeyValue::new("reason", "allowlist_denied")], - ); + metrics::counter!("buzz_auth_failures_total", "reason" => "allowlist_denied") + .increment(1); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, @@ -136,10 +132,8 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: Ok(owner) => owner, Err(e) => { warn!(conn_id = %conn_id, pubkey = %pubkey.to_hex(), error = ?e, "not a relay member"); - crate::metrics::metrics().auth_failures_total.add( - 1, - &[opentelemetry::KeyValue::new("reason", "not_relay_member")], - ); + metrics::counter!("buzz_auth_failures_total", "reason" => "not_relay_member") + .increment(1); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, @@ -253,10 +247,7 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc, state: } Err(e) => { warn!(conn_id = %conn_id, error = %e, "NIP-42 auth failed"); - crate::metrics::metrics().auth_failures_total.add( - 1, - &[opentelemetry::KeyValue::new("reason", "nip42_invalid")], - ); + metrics::counter!("buzz_auth_failures_total", "reason" => "nip42_invalid").increment(1); *conn.auth_state.write().await = AuthState::Failed; conn.send(RelayMessage::ok( &event_id_hex, diff --git a/crates/buzz-relay/src/handlers/count.rs b/crates/buzz-relay/src/handlers/count.rs index f92dfdfc63..7cb4882185 100644 --- a/crates/buzz-relay/src/handlers/count.rs +++ b/crates/buzz-relay/src/handlers/count.rs @@ -164,9 +164,7 @@ pub async fn handle_count( match state.db.query_events(&q).await { Ok(stored_events) => { if super::req::count_fallback_exceeded(stored_events.len()) { - crate::metrics::metrics() - .count_fallback_rejections_total - .add(1, &[]); + metrics::counter!("buzz_count_fallback_rejections_total").increment(1); conn.send(RelayMessage::closed( &sub_id, "restricted: count filter requires narrower constraints", @@ -229,9 +227,7 @@ pub async fn handle_count( match state.db.query_events(&query).await { Ok(stored_events) => { if super::req::count_fallback_exceeded(stored_events.len()) { - crate::metrics::metrics() - .count_fallback_rejections_total - .add(1, &[]); + metrics::counter!("buzz_count_fallback_rejections_total").increment(1); conn.send(RelayMessage::closed( &sub_id, "restricted: count filter requires narrower constraints", diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index 4463b937da..f655ae0572 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -27,9 +27,7 @@ use super::ingest::{IngestAuth, IngestError}; /// Increment the rejection counter with a bounded reason label. fn reject(reason: &'static str) { - crate::metrics::metrics() - .events_rejected_total - .add(1, &[opentelemetry::KeyValue::new("reason", reason)]); + metrics::counter!("buzz_events_rejected_total", "reason" => reason).increment(1); } /// Bound the `kind` label to prevent cardinality explosion from arbitrary Nostr kinds. @@ -159,9 +157,7 @@ pub(crate) async fn fan_out_event_to_local_subscribers( ) { let matches = state.sub_registry.fan_out_scoped(community_id, stored); let matches = filter_fanout_by_access(state, community_id, stored, matches).await; - crate::metrics::metrics() - .fanout_recipients - .record(matches.len() as f64, &[]); + metrics::histogram!("buzz_fanout_recipients").record(matches.len() as f64); if matches.is_empty() { return; } @@ -217,7 +213,7 @@ pub async fn fan_out_pubsub_event(state: &Arc, channel_event: buzz_pub let matches = state.sub_registry.fan_out_scoped(community_id, &stored); let matches = filter_fanout_by_access(state, community_id, &stored, matches).await; - crate::metrics::metrics().multinode_fanout_total.add(1, &[]); + metrics::counter!("buzz_multinode_fanout_total").increment(1); if matches.is_empty() { return; } @@ -281,9 +277,7 @@ pub(crate) async fn dispatch_persistent_event( .sub_registry .fan_out_scoped(tenant.community(), stored_event); let matches = filter_fanout_by_access(state, tenant.community(), stored_event, matches).await; - crate::metrics::metrics() - .fanout_recipients - .record(matches.len() as f64, &[]); + metrics::histogram!("buzz_fanout_recipients").record(matches.len() as f64); debug!( event_id = %event_id_hex, channel_id = ?stored_event.channel_id, @@ -363,9 +357,7 @@ pub(crate) async fn dispatch_persistent_event( }; if let Err(e) = state.audit_tx.send(audit_entry).await { error!(event_id = %event_id_hex, "Audit channel closed — entry lost: {e}"); - crate::metrics::metrics() - .audit_send_errors_total - .add(1, &[]); + metrics::counter!("buzz_audit_send_errors_total").increment(1); } // Skip workflow triggering for workflow-execution kinds and relay-signed workflow messages. @@ -397,9 +389,8 @@ pub(crate) async fn dispatch_persistent_event( { tracing::error!(event_id = ?workflow_event.event.id, "Workflow trigger failed: {e}"); } else { - crate::metrics::metrics() - .workflow_runs_total - .add(1, &[opentelemetry::KeyValue::new("trigger", trigger_kind)]); + metrics::counter!("buzz_workflow_runs_total", "trigger" => trigger_kind) + .increment(1); } }); } @@ -424,10 +415,7 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc kind_str.clone()).increment(1); let (conn_id, pubkey_bytes, auth_pubkey, scopes, channel_ids) = { let auth = conn.auth_state.read().await; @@ -526,10 +514,7 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc { if result.accepted { - crate::metrics::metrics().events_stored_total.add( - 1, - &[opentelemetry::KeyValue::new("kind", kind_str.to_string())], - ); + metrics::counter!("buzz_events_stored_total", "kind" => kind_str).increment(1); info!( event_id = %result.event_id, kind = kind_u32, @@ -537,9 +522,8 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc anyhow::Result<()> { // layer that exports spans via OTLP gRPC alongside the JSON stdout logs. // // Build a single shared Resource (service.name=buzz-relay by default, overridable - // via OTEL_SERVICE_NAME) used by both the trace and metric providers so that - // Datadog can correlate spans and metrics under the same service identity. + // via OTEL_SERVICE_NAME) for the trace provider so that Datadog can identify + // spans under the correct service identity. let resource = telemetry::service_resource(); let tracer_provider = telemetry::try_init_tracer(resource.clone()); let otel_layer = tracer_provider.as_ref().map(|p| { @@ -63,7 +63,7 @@ async fn main() -> anyhow::Result<()> { "Config loaded" ); - let meter_provider = relay_metrics::install(config.metrics_port, resource); + relay_metrics::install(config.metrics_port); info!( port = config.metrics_port, "Prometheus metrics exporter started" @@ -661,9 +661,6 @@ async fn main() -> anyhow::Result<()> { let state_for_sub = Arc::clone(&state); let mut rx = state_for_sub.pubsub.subscribe_local(); tokio::spawn(async move { - let fanout_lag = relay_metrics::meter() - .u64_counter("buzz_multinode_fanout_lag_total") - .build(); loop { match rx.recv().await { Ok(channel_event) => { @@ -674,7 +671,7 @@ async fn main() -> anyhow::Result<()> { .await; } Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - fanout_lag.add(n, &[]); + metrics::counter!("buzz_multinode_fanout_lag_total").increment(n); tracing::warn!("Multi-node fan-out lagged by {n} messages"); } Err(tokio::sync::broadcast::error::RecvError::Closed) => { @@ -694,9 +691,6 @@ async fn main() -> anyhow::Result<()> { let state_for_cache = Arc::clone(&state); let mut rx = state_for_cache.pubsub.subscribe_cache_invalidations(); tokio::spawn(async move { - let cache_inval_lag = relay_metrics::meter() - .u64_counter("buzz_cache_invalidation_lag_total") - .build(); loop { match rx.recv().await { Ok(scoped) => { @@ -708,7 +702,7 @@ async fn main() -> anyhow::Result<()> { .apply_cache_invalidation(scoped.community_id, scoped.invalidation); } Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - cache_inval_lag.add(n, &[]); + metrics::counter!("buzz_cache_invalidation_lag_total").increment(n); tracing::warn!("Cache-invalidation consumer lagged by {n} messages"); } Err(tokio::sync::broadcast::error::RecvError::Closed) => { @@ -732,29 +726,20 @@ async fn main() -> anyhow::Result<()> { .unwrap_or(10) .max(1); // tokio::time::interval panics on Duration::ZERO tokio::spawn(async move { - let m = relay_metrics::meter(); - let db_pool_size = m.u64_gauge("buzz_db_pool_size").build(); - let db_pool_idle = m.u64_gauge("buzz_db_pool_idle").build(); - let db_pool_active = m.u64_gauge("buzz_db_pool_active").build(); - let redis_pool_available = m.u64_gauge("buzz_redis_pool_available").build(); - let redis_pool_size = m.u64_gauge("buzz_redis_pool_size").build(); - let redis_pool_max = m.u64_gauge("buzz_redis_pool_max").build(); - let redis_pool_waiting = m.u64_gauge("buzz_redis_pool_waiting").build(); - let mut interval = tokio::time::interval(std::time::Duration::from_secs(interval_secs)); loop { interval.tick().await; let db_stats = pool_state.db.pool_stats(); let active = db_stats.size.saturating_sub(db_stats.idle); - db_pool_size.record(db_stats.size as u64, &[]); - db_pool_idle.record(db_stats.idle as u64, &[]); - db_pool_active.record(active as u64, &[]); + metrics::gauge!("buzz_db_pool_size").set(db_stats.size as f64); + metrics::gauge!("buzz_db_pool_idle").set(db_stats.idle as f64); + metrics::gauge!("buzz_db_pool_active").set(active as f64); let rs = pool_state.redis_pool.status(); - redis_pool_available.record(rs.available as u64, &[]); - redis_pool_size.record(rs.size as u64, &[]); - redis_pool_max.record(rs.max_size as u64, &[]); - redis_pool_waiting.record(rs.waiting as u64, &[]); + metrics::gauge!("buzz_redis_pool_available").set(rs.available as f64); + metrics::gauge!("buzz_redis_pool_size").set(rs.size as f64); + metrics::gauge!("buzz_redis_pool_max").set(rs.max_size as f64); + metrics::gauge!("buzz_redis_pool_waiting").set(rs.waiting as f64); } }); } @@ -768,10 +753,7 @@ async fn main() -> anyhow::Result<()> { .drain(std::time::Duration::from_secs(5)) .await; - // Flush pending OTEL spans and metrics before exit. - if let Err(e) = meter_provider.shutdown() { - tracing::warn!(error = %e, "OTEL meter provider shutdown error"); - } + // Flush pending OTEL spans before exit. if let Some(tp) = tracer_provider { if let Err(e) = tp.shutdown() { tracing::warn!(error = %e, "OTEL tracer provider shutdown error"); diff --git a/crates/buzz-relay/src/metrics.rs b/crates/buzz-relay/src/metrics.rs index 6715b60fd3..954e78434d 100644 --- a/crates/buzz-relay/src/metrics.rs +++ b/crates/buzz-relay/src/metrics.rs @@ -1,25 +1,19 @@ -//! OpenTelemetry metrics: provider setup, instrument handles, and HTTP middleware. +//! Prometheus metrics: recorder setup, upkeep task, and HTTP middleware. //! //! ```text -//! ┌──────────────────────────────────────────────────────────────────────┐ -//! │ OTEL Meter API via global Metrics struct (pre-built instrument │ -//! │ handles — one allocation at startup, zero per call-site) │ -//! │ ↓ │ -//! │ SdkMeterProvider │ -//! │ ├── PrometheusExporter → prometheus::Registry → HTTP :9102 │ -//! │ └── (if OTEL_EXPORTER_OTLP_ENDPOINT set) │ -//! │ PeriodicReader + OTLP MetricExporter → collector/DD agent │ -//! └──────────────────────────────────────────────────────────────────────┘ +//! ┌──────────────────────────────────────────────────────────┐ +//! │ metrics-rs facade (metrics::counter!, histogram!, etc.) │ +//! │ ↓ │ +//! │ PrometheusBuilder → HTTP listener on :9102 │ +//! │ ↓ │ +//! │ GET /metrics → Prometheus text format │ +//! └──────────────────────────────────────────────────────────┘ //! ``` //! -//! All metric names, types, and label sets are preserved from the prior -//! `metrics-rs` implementation so existing Prometheus scrapers and Datadog -//! dashboards need no changes. -//! -//! Custom histogram bucket boundaries are registered as OTEL Views before the -//! provider is built. +//! Framework metrics (`http_requests_total`, `http_request_latency_ms`) are +//! recorded by [`track_metrics`] middleware on the app router. Buzz-specific +//! metrics are recorded inline at their call sites. -use std::sync::OnceLock; use std::time::Instant; use axum::{ @@ -27,350 +21,49 @@ use axum::{ middleware::Next, response::Response, }; -use opentelemetry::{ - metrics::{Counter, Histogram, Meter, UpDownCounter}, - KeyValue, -}; -use opentelemetry_sdk::metrics::{ - Aggregation, Instrument, PeriodicReader, SdkMeterProvider, Stream, -}; -use prometheus::Registry; - -// ─── Bucket boundaries ─────────────────────────────────────────────────────── +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; /// HTTP latency buckets (milliseconds) — only for `http_request_latency_ms`. -pub const LATENCY_BUCKETS_MS: &[f64] = &[ +const LATENCY_BUCKETS_MS: [f64; 11] = [ 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0, 10000.0, ]; /// Seconds-scale buckets for internal processing histograms (event, search, audit). -pub const DURATION_BUCKETS_S: &[f64] = &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0]; +const DURATION_BUCKETS_S: [f64; 10] = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0]; /// Integer-count buckets for fan-out recipient histograms. -pub const FANOUT_BUCKETS: &[f64] = &[0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0]; - -/// Scope name used when obtaining meters. -pub const METER_SCOPE: &str = "buzz-relay"; - -// ─── Global instrument handles ──────────────────────────────────────────────── - -/// Pre-built OTEL instrument handles. One global instance; zero per-call-site -/// allocation. Initialized by [`install`]; panics if accessed before that. -#[allow(missing_docs)] // field names are self-documenting metric names -pub struct Metrics { - // HTTP framework - pub http_requests_total: Counter, - pub http_request_latency_ms: Histogram, - - // WebSocket connections - pub ws_connections_total: Counter, - pub ws_connections_active: UpDownCounter, - pub ws_backpressure_disconnects_total: Counter, - pub ws_auth_timeouts_total: Counter, - - // Subscriptions - pub subscriptions_active: UpDownCounter, - - // Events - pub events_received_total: Counter, - pub events_stored_total: Counter, - pub events_rejected_total: Counter, - pub event_processing_seconds: Histogram, - - // Fan-out - pub fanout_recipients: Histogram, - pub multinode_fanout_total: Counter, - pub multinode_fanout_lag_total: Counter, - pub cache_invalidation_lag_total: Counter, - - // Search - pub search_index_seconds: Histogram, - pub search_index_errors_total: Counter, +const FANOUT_BUCKETS: [f64; 9] = [0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0]; - // Audit - pub audit_log_seconds: Histogram, - pub audit_log_errors_total: Counter, - pub audit_send_errors_total: Counter, - - // Auth - pub auth_attempts_total: Counter, - pub auth_failures_total: Counter, - - // Media - pub media_uploads_total: Counter, - pub media_upload_rejections_total: Counter, - - // Workflows - pub workflow_runs_total: Counter, - - // Cache - pub membership_cache_hits_total: Counter, - pub membership_cache_misses_total: Counter, - pub accessible_channels_cache_hits_total: Counter, - pub accessible_channels_cache_misses_total: Counter, - - // Count fallback - pub count_fallback_rejections_total: Counter, -} - -static METRICS: OnceLock = OnceLock::new(); - -/// Access the global [`Metrics`] instance. +/// Install the global metrics recorder and spawn the Prometheus HTTP exporter. /// -/// If [`install`] has not yet been called (e.g. in unit tests that don't -/// start a full relay), returns a lazily-initialised set of no-op instruments -/// built from the global meter (which is a no-op provider by default). -/// This matches the prior `metrics-rs` behaviour where macros silently -/// did nothing without explicit initialisation. -pub fn metrics() -> &'static Metrics { - METRICS.get_or_init(build_metrics) -} - -/// Build the [`Metrics`] instrument handles from the current global meter. +/// `build()` returns the recorder + exporter future and internally spawns +/// the upkeep task, so no separate upkeep call is needed. /// -/// Called once — either from [`install`] (real provider set) or lazily from -/// [`metrics`] (falls back to OTEL's built-in no-op meter). -fn build_metrics() -> Metrics { - let m = opentelemetry::global::meter(METER_SCOPE); - Metrics { - http_requests_total: m.u64_counter("http_requests_total").build(), - http_request_latency_ms: m.f64_histogram("http_request_latency_ms").build(), - - ws_connections_total: m.u64_counter("buzz_ws_connections_total").build(), - ws_connections_active: m.i64_up_down_counter("buzz_ws_connections_active").build(), - ws_backpressure_disconnects_total: m - .u64_counter("buzz_ws_backpressure_disconnects_total") - .build(), - ws_auth_timeouts_total: m.u64_counter("buzz_ws_auth_timeouts_total").build(), - - subscriptions_active: m.i64_up_down_counter("buzz_subscriptions_active").build(), - - events_received_total: m.u64_counter("buzz_events_received_total").build(), - events_stored_total: m.u64_counter("buzz_events_stored_total").build(), - events_rejected_total: m.u64_counter("buzz_events_rejected_total").build(), - event_processing_seconds: m.f64_histogram("buzz_event_processing_seconds").build(), - - fanout_recipients: m.f64_histogram("buzz_fanout_recipients").build(), - multinode_fanout_total: m.u64_counter("buzz_multinode_fanout_total").build(), - multinode_fanout_lag_total: m.u64_counter("buzz_multinode_fanout_lag_total").build(), - cache_invalidation_lag_total: m.u64_counter("buzz_cache_invalidation_lag_total").build(), - - search_index_seconds: m.f64_histogram("buzz_search_index_seconds").build(), - search_index_errors_total: m.u64_counter("buzz_search_index_errors_total").build(), - - audit_log_seconds: m.f64_histogram("buzz_audit_log_seconds").build(), - audit_log_errors_total: m.u64_counter("buzz_audit_log_errors_total").build(), - audit_send_errors_total: m.u64_counter("buzz_audit_send_errors_total").build(), - - auth_attempts_total: m.u64_counter("buzz_auth_attempts_total").build(), - auth_failures_total: m.u64_counter("buzz_auth_failures_total").build(), - - media_uploads_total: m.u64_counter("buzz_media_uploads_total").build(), - media_upload_rejections_total: m.u64_counter("buzz_media_upload_rejections_total").build(), - - workflow_runs_total: m.u64_counter("buzz_workflow_runs_total").build(), - - membership_cache_hits_total: m.u64_counter("buzz_membership_cache_hits_total").build(), - membership_cache_misses_total: m.u64_counter("buzz_membership_cache_misses_total").build(), - accessible_channels_cache_hits_total: m - .u64_counter("buzz_accessible_channels_cache_hits_total") - .build(), - accessible_channels_cache_misses_total: m - .u64_counter("buzz_accessible_channels_cache_misses_total") - .build(), - - count_fallback_rejections_total: m - .u64_counter("buzz_count_fallback_rejections_total") - .build(), - } -} - -/// Returns a [`Meter`] scoped to the relay. Useful for one-off or dynamic -/// instruments (e.g. pool gauge task). -pub fn meter() -> Meter { - opentelemetry::global::meter(METER_SCOPE) -} - -// ─── Provider setup ─────────────────────────────────────────────────────────── - -/// Install the global OTEL meter provider and spawn the Prometheus HTTP exporter. -/// -/// Returns the [`SdkMeterProvider`] so the caller can shut it down gracefully -/// on SIGTERM. -/// -/// If `OTEL_EXPORTER_OTLP_ENDPOINT` is set, a second reader is attached that -/// pushes metrics via OTLP gRPC on the configured interval (default 60 s). -/// -/// # Panics -/// Panics if called more than once or if the HTTP listener cannot bind to `port`. -pub fn install(port: u16, resource: opentelemetry_sdk::Resource) -> SdkMeterProvider { - let registry = prometheus::Registry::new(); - - // Bind synchronously so that a port conflict fails startup immediately, - // matching the prior `metrics-exporter-prometheus` behaviour. - let listener = { - let addr = std::net::SocketAddr::from(([0, 0, 0, 0], port)); - std::net::TcpListener::bind(addr) - .unwrap_or_else(|e| panic!("metrics listener failed to bind :{port}: {e}")) - }; - - // Build the Prometheus exporter (pull-based: no periodic push needed). - let prom_exporter = opentelemetry_prometheus::exporter() - .with_registry(registry.clone()) - // Don't add unit suffixes — keep names identical to the old metrics-rs names. - .without_units() - // Don't add `_total` suffix — our counter names already end in `_total`. - .without_counter_suffixes() - // otel_scope_* labels add noise; the relay has a single scope. - .without_scope_info() - // Suppress the `target_info` series added by OTEL for resource attributes. - // The old metrics-rs endpoint did not emit it; excluding it keeps byte-parity - // so the Datadog openmetrics annotation needs no changes. - .without_target_info() +/// Must be called from within a Tokio runtime. +/// Panics if a recorder is already installed or the port is in use. +pub fn install(port: u16) { + let (recorder, exporter) = PrometheusBuilder::new() + .with_http_listener(([0, 0, 0, 0], port)) + // Per-metric buckets: ms for HTTP latency, seconds for internal processing. + .set_buckets_for_metric( + Matcher::Full("http_request_latency_ms".to_owned()), + &LATENCY_BUCKETS_MS, + ) + .expect("valid ms bucket boundaries") + .set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), &DURATION_BUCKETS_S) + .expect("valid seconds bucket boundaries") + .set_buckets_for_metric( + Matcher::Full("buzz_fanout_recipients".to_owned()), + &FANOUT_BUCKETS, + ) + .expect("valid fanout bucket boundaries") .build() - .expect("Prometheus exporter must build exactly once"); - - let mut provider_builder = SdkMeterProvider::builder() - .with_resource(resource) - .with_reader(prom_exporter) - .with_view(explicit_bucket_view); + .expect("metrics exporter must build exactly once"); - // Attach OTLP metric exporter only when the endpoint env var is set. - if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok() { - match opentelemetry_otlp::MetricExporter::builder() - .with_tonic() - .build() - { - Ok(exporter) => { - let periodic = PeriodicReader::builder(exporter).build(); - provider_builder = provider_builder.with_reader(periodic); - } - Err(e) => { - tracing::warn!(error = %e, "Failed to build OTLP metric exporter; OTLP metrics disabled"); - } - } - } - - let provider = provider_builder.build(); - - // Store globally so `global::meter(...)` works throughout the codebase. - // Must happen BEFORE build_metrics() so the real provider backs all handles. - opentelemetry::global::set_meter_provider(provider.clone()); - - // Build all instrument handles from the now-installed global meter. - // `get_or_init` is safe here: in production `install()` runs before any - // metric emission, so this is always the first init. If somehow called - // after a lazy noop init in tests, the noop handles are kept (acceptable - // for test isolation — the Prometheus endpoint test calls install() first). - METRICS.get_or_init(build_metrics); - - // Convert the synchronously-bound std listener and spawn the HTTP server. - // The bind already succeeded above; non-blocking mode is required before - // handing the socket to tokio. - listener - .set_nonblocking(true) - .expect("set metrics listener non-blocking"); - let async_listener = - tokio::net::TcpListener::from_std(listener).expect("convert std TcpListener to tokio"); - tokio::spawn(serve_prometheus(async_listener, registry)); - - provider -} - -// ─── View ───────────────────────────────────────────────────────────────────── - -/// OTEL View mapping named histograms to their explicit bucket boundaries. -/// -/// Any histogram whose name doesn't match uses the SDK default buckets. -fn explicit_bucket_view(inst: &Instrument) -> Option { - let boundaries: &[f64] = if inst.name() == "http_request_latency_ms" { - LATENCY_BUCKETS_MS - } else if inst.name() == "buzz_event_processing_seconds" - || inst.name() == "buzz_search_index_seconds" - || inst.name() == "buzz_audit_log_seconds" - { - DURATION_BUCKETS_S - } else if inst.name() == "buzz_fanout_recipients" { - FANOUT_BUCKETS - } else { - return None; // use SDK default - }; - - Stream::builder() - .with_aggregation(Aggregation::ExplicitBucketHistogram { - boundaries: boundaries.to_vec(), - record_min_max: false, - }) - .build() - .ok() -} - -// ─── Prometheus HTTP endpoint ───────────────────────────────────────────────── - -/// Serve the Prometheus `/metrics` endpoint on a pre-bound TCP listener. -/// -/// Accepts a listener that has already been bound synchronously in [`install`] -/// so that port-in-use errors surface at startup, not inside a detached task. -/// -/// This is intentionally minimal — no middleware, no auth. Port access controls -/// are expected to be enforced at the network/mesh level (Istio excludes this -/// port from the mesh by default in the Blox deployment). -async fn serve_prometheus(listener: tokio::net::TcpListener, registry: Registry) { - use axum::{routing::get, Router}; - use prometheus::{Encoder, TextEncoder}; - - let app = Router::new().route( - "/metrics", - get(move || { - let reg = registry.clone(); - async move { - let encoder = TextEncoder::new(); - let families = reg.gather(); - let mut buf = Vec::new(); - encoder - .encode(&families, &mut buf) - .expect("encode prometheus metrics"); - let content_type = encoder.format_type().to_owned(); - ([(axum::http::header::CONTENT_TYPE, content_type)], buf) - } - }), - ); - - axum::serve(listener, app).await.ok(); -} - -// ─── Cardinality control ────────────────────────────────────────────────────── - -/// Map arbitrary Nostr event kinds to a bounded label value. -/// -/// The allow-list matches the metrics observed in production; all other kinds -/// collapse to `"other"` to avoid cardinality explosion. -pub fn bounded_kind_label(kind: u16) -> &'static str { - match kind { - 0 => "0", // NIP-01 metadata - 1 => "1", // short text note - 3 => "3", // contact list - 4 => "4", // encrypted DM - 5 => "5", // deletion - 6 => "6", // repost - 7 => "7", // reaction - 40 => "40", // channel create - 41 => "41", // channel metadata - 42 => "42", // channel message - 43 => "43", // channel hide - 44 => "44", // channel mute - 1059 => "1059", // NIP-44 gift wrap - 1984 => "1984", // report - 9734 => "9734", // zap request - 9735 => "9735", // zap - 10000 => "10000", // mute list - 10001 => "10001", // pin list - _ => "other", - } + metrics::set_global_recorder(recorder).expect("global recorder must be set exactly once"); + tokio::spawn(exporter); } -// ─── HTTP metrics middleware ─────────────────────────────────────────────────── - /// Axum middleware that records CAKE framework HTTP metrics. /// /// Emits: @@ -424,115 +117,9 @@ pub async fn track_metrics(req: Request, next: Next) -> Response { let status = response.status().as_u16().to_string(); let latency_ms = start.elapsed().as_secs_f64() * 1000.0; - let labels = [ - KeyValue::new("code", status), - KeyValue::new("caller", caller), - KeyValue::new("action", action), - ]; - let m = metrics(); - m.http_requests_total.add(1, &labels); - m.http_request_latency_ms.record(latency_ms, &labels); + let labels = [("code", status), ("caller", caller), ("action", action)]; + metrics::counter!("http_requests_total", &labels).increment(1); + metrics::histogram!("http_request_latency_ms", &labels).record(latency_ms); response } - -// ─── Tests ──────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn bounded_kind_label_known_kinds_are_stable() { - assert_eq!(bounded_kind_label(1), "1"); - assert_eq!(bounded_kind_label(42), "42"); - assert_eq!(bounded_kind_label(9735), "9735"); - } - - #[test] - fn bounded_kind_label_unknown_collapses_to_other() { - assert_eq!(bounded_kind_label(12345), "other"); - assert_eq!(bounded_kind_label(0xFFFF), "other"); - } - - /// Verify that the Prometheus HTTP endpoint returns 200 and includes - /// known metric names after installation. - /// - /// This test builds its own isolated Prometheus registry and meter - /// provider (not the global one) so it is fully independent of any - /// other test's metric state. - #[tokio::test] - async fn prometheus_endpoint_serves_known_metric_names() { - use opentelemetry_sdk::metrics::SdkMeterProvider; - use prometheus::Registry; - - // 1. Build an isolated registry + OTEL provider. - let registry = Registry::new(); - let exporter = opentelemetry_prometheus::exporter() - .with_registry(registry.clone()) - .without_units() - .without_counter_suffixes() - .without_scope_info() - .without_target_info() - .build() - .expect("build test prometheus exporter"); - - let provider = SdkMeterProvider::builder().with_reader(exporter).build(); - - // 2. Create a meter from this isolated provider (not the global one). - let meter = opentelemetry::metrics::MeterProvider::meter(&provider, METER_SCOPE); - - // 3. Build and record values for a sample of the relay's named instruments. - // These must appear in the Prometheus output. - let ws_conn = meter.u64_counter("buzz_ws_connections_total").build(); - let events_recv = meter.u64_counter("buzz_events_received_total").build(); - let events_stored = meter.u64_counter("buzz_events_stored_total").build(); - let auth_attempts = meter.u64_counter("buzz_auth_attempts_total").build(); - - ws_conn.add(1, &[]); - events_recv.add(1, &[KeyValue::new("kind", "1")]); - events_stored.add(1, &[KeyValue::new("kind", "42")]); - auth_attempts.add(1, &[KeyValue::new("method", "nip42")]); - - // 4. Bind synchronously (matching install() behaviour) then convert. - let std_listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral"); - let port = std_listener.local_addr().expect("local_addr").port(); - // tokio requires non-blocking mode before from_std(). - std_listener.set_nonblocking(true).expect("set_nonblocking"); - let listener = - tokio::net::TcpListener::from_std(std_listener).expect("convert to tokio listener"); - - // 5. Spawn the Prometheus HTTP server with the pre-bound listener. - tokio::spawn(serve_prometheus(listener, registry)); - - // Yield to let the spawned Axum task start accepting connections. - tokio::task::yield_now().await; - - // 6. Fetch /metrics and verify the expected names are present. - let url = format!("http://127.0.0.1:{port}/metrics"); - let body = reqwest::get(&url) - .await - .expect("GET /metrics") - .error_for_status() - .expect("HTTP 200 from /metrics") - .text() - .await - .expect("read response body"); - - for expected in &[ - "buzz_ws_connections_total", - "buzz_events_received_total", - "buzz_events_stored_total", - "buzz_auth_attempts_total", - ] { - assert!( - body.contains(expected), - "/metrics body is missing '{expected}';\nbody:\n{body}", - ); - } - - // Keep the provider alive until the assertions complete so metrics - // aren't flushed/dropped before the HTTP response arrives. - drop(provider); - } -} diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index d5d3569732..51b98ddef1 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -166,9 +166,7 @@ impl ConnectionManager { let count = conn.backpressure_count.fetch_add(1, Ordering::Relaxed) + 1; if count >= conn.grace_limit { tracing::warn!(conn_id = %conn_id, count, "fan-out: sustained backpressure — cancelling slow client"); - crate::metrics::metrics() - .ws_backpressure_disconnects_total - .add(1, &[]); + metrics::counter!("buzz_ws_backpressure_disconnects_total").increment(1); conn.cancel.cancel(); } else { tracing::warn!(conn_id = %conn_id, count, grace = conn.grace_limit, "fan-out: send buffer full — grace {count}/{}", conn.grace_limit); @@ -468,14 +466,10 @@ impl AppState { ) -> Result { let key = (community_id, channel_id, pubkey.to_vec()); if let Some(cached) = self.membership_cache.get(&key) { - crate::metrics::metrics() - .membership_cache_hits_total - .add(1, &[]); + metrics::counter!("buzz_membership_cache_hits_total").increment(1); return Ok(cached); } - crate::metrics::metrics() - .membership_cache_misses_total - .add(1, &[]); + metrics::counter!("buzz_membership_cache_misses_total").increment(1); let result = self.db.is_member(community_id, channel_id, pubkey).await?; self.membership_cache.insert(key, result); Ok(result) @@ -604,14 +598,10 @@ impl AppState { ) -> Result, buzz_db::DbError> { let key = (community_id, pubkey.to_vec()); if let Some(cached) = self.accessible_channels_cache.get(&key) { - crate::metrics::metrics() - .accessible_channels_cache_hits_total - .add(1, &[]); + metrics::counter!("buzz_accessible_channels_cache_hits_total").increment(1); return Ok(cached); } - crate::metrics::metrics() - .accessible_channels_cache_misses_total - .add(1, &[]); + metrics::counter!("buzz_accessible_channels_cache_misses_total").increment(1); let result = self .db .get_accessible_channel_ids(community_id, pubkey) @@ -683,12 +673,10 @@ impl AuditShutdownHandle { async fn log_audit_entry(audit: &buzz_audit::AuditService, entry: buzz_audit::NewAuditEntry) { let t = std::time::Instant::now(); if let Err(e) = audit.log(entry).await { - crate::metrics::metrics().audit_log_errors_total.add(1, &[]); + metrics::counter!("buzz_audit_log_errors_total").increment(1); tracing::error!("Audit log failed: {e}"); } else { - crate::metrics::metrics() - .audit_log_seconds - .record(t.elapsed().as_secs_f64(), &[]); + metrics::histogram!("buzz_audit_log_seconds").record(t.elapsed().as_secs_f64()); } } diff --git a/crates/buzz-relay/src/subscription.rs b/crates/buzz-relay/src/subscription.rs index ff0ae32a53..419cf503cb 100644 --- a/crates/buzz-relay/src/subscription.rs +++ b/crates/buzz-relay/src/subscription.rs @@ -80,7 +80,7 @@ impl SubscriptionRegistry { .entry(conn_id) .or_default() .insert(sub_id.clone(), (filters.clone(), community_id, channel_id)); - crate::metrics::metrics().subscriptions_active.add(1, &[]); + metrics::gauge!("buzz_subscriptions_active").increment(1.0); if let Some(ch_id) = channel_id { match extract_kinds_from_filters(&filters) { @@ -167,7 +167,7 @@ impl SubscriptionRegistry { if let Some(mut conn_subs) = self.subs.get_mut(&conn_id) { if let Some((filters, community_id, channel_id)) = conn_subs.remove(sub_id) { self.remove_from_index(conn_id, sub_id, &filters, community_id, channel_id); - crate::metrics::metrics().subscriptions_active.add(-1, &[]); + metrics::gauge!("buzz_subscriptions_active").decrement(1.0); return Some(RemovedSubscription { community_id, channel_id, @@ -189,9 +189,7 @@ impl SubscriptionRegistry { channel_id: *channel_id, }); } - crate::metrics::metrics() - .subscriptions_active - .add(-(count as i64), &[]); + metrics::gauge!("buzz_subscriptions_active").decrement(count as f64); } removed } diff --git a/crates/buzz-relay/src/telemetry.rs b/crates/buzz-relay/src/telemetry.rs index 69f524bb40..7324e0fab4 100644 --- a/crates/buzz-relay/src/telemetry.rs +++ b/crates/buzz-relay/src/telemetry.rs @@ -25,7 +25,7 @@ use opentelemetry_sdk::{resource::EnvResourceDetector, trace::SdkTracerProvider, Resource}; -/// Build the OTEL [`Resource`] shared by the trace and metric providers. +/// Build the OTEL [`Resource`] used by the trace provider. /// /// Strategy (priority order): /// 1. `service.name` in `OTEL_RESOURCE_ATTRIBUTES` — overlaid last by @@ -40,9 +40,8 @@ use opentelemetry_sdk::{resource::EnvResourceDetector, trace::SdkTracerProvider, /// `buzz-relay` default. We therefore read `OTEL_SERVICE_NAME` explicitly /// so the fallback is fully under our control. /// -/// Both the tracer provider (traces) and the meter provider (metrics) receive -/// the same `Resource` instance so Datadog can correlate spans and metrics on -/// the same `service.name`. +/// The tracer provider receives this `Resource` so Datadog can identify +/// spans under the correct `service.name`. pub fn service_resource() -> Resource { // Honor OTEL_SERVICE_NAME when set+non-empty; otherwise use buzz-relay. // EnvResourceDetector overlays OTEL_RESOURCE_ATTRIBUTES last, so an From 47f188b94a39fcd3a05054b6511c03c2cad6b523 Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Wed, 1 Jul 2026 14:42:13 -0400 Subject: [PATCH 5/6] =?UTF-8?q?fix(relay):=20address=20Wren=20review=20pas?= =?UTF-8?q?s=20=E2=80=94=20pre-subscriber=20warn,=20unused=20dep,=20teleme?= =?UTF-8?q?try=20tests,=20db=20pool=20max?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding 1: change try_init_tracer to return TracerInit enum (Enabled/Disabled/ExporterBuildFailed) so the exporter-build failure warning is emitted by the caller after tracing_subscriber is installed, preventing the warn! from firing with no subscriber attached. Finding 2: remove opentelemetry-semantic-conventions from root Cargo.toml and crates/buzz-relay/Cargo.toml — the crate had zero references in buzz-relay/src; confirmed dropped from Cargo.lock after build. Finding 3: add #[cfg(test)] unit tests in telemetry.rs covering service_resource() (default, OTEL_SERVICE_NAME honored, empty string fallback) and try_init_tracer() (Disabled when endpoint unset, not-Disabled when endpoint set). ENV_LOCK Mutex serializes env-mutating tests to prevent cross-test races. Finding 4: store max_connections on Db struct (set from DbConfig in new(), read from pool.options() in from_pool()); add max: u32 to DbPoolStats; emit buzz_db_pool_max gauge alongside existing db pool gauges in main.rs. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- Cargo.lock | 7 -- Cargo.toml | 1 - crates/buzz-db/src/api_token.rs | 2 +- crates/buzz-db/src/lib.rs | 18 +++- crates/buzz-relay/Cargo.toml | 1 - crates/buzz-relay/src/main.rs | 21 +++-- crates/buzz-relay/src/telemetry.rs | 139 ++++++++++++++++++++++++++--- 7 files changed, 156 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1ff60bfb9..f7a2552a7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1059,7 +1059,6 @@ dependencies = [ "nostr", "opentelemetry 0.32.0", "opentelemetry-otlp 0.32.0", - "opentelemetry-semantic-conventions", "opentelemetry_sdk 0.32.1", "rand 0.10.1", "redis", @@ -5520,12 +5519,6 @@ dependencies = [ "tonic-prost", ] -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" - [[package]] name = "opentelemetry_sdk" version = "0.31.0" diff --git a/Cargo.toml b/Cargo.toml index 8f972ce7b7..bdfc8d367c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,7 +71,6 @@ tracing-opentelemetry = { version = "0.33" } opentelemetry = { version = "0.32", features = ["trace"] } opentelemetry_sdk = { version = "0.32", features = ["trace", "rt-tokio"] } opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "grpc-tonic", "tls-ring"] } -opentelemetry-semantic-conventions = { version = "0.32" } metrics = "0.24" metrics-exporter-prometheus = "0.18" diff --git a/crates/buzz-db/src/api_token.rs b/crates/buzz-db/src/api_token.rs index f105d681a6..50821743d2 100644 --- a/crates/buzz-db/src/api_token.rs +++ b/crates/buzz-db/src/api_token.rs @@ -350,7 +350,7 @@ mod tests { let pool = PgPool::connect(TEST_DB_URL) .await .expect("connect to test DB"); - Db { pool } + Db::from_pool(pool) } async fn make_community(pool: &PgPool) -> Uuid { diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 81881c70b1..01e29c20ca 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -129,6 +129,8 @@ pub async fn insert_mentions( #[derive(Clone, Debug)] pub struct Db { pub(crate) pool: PgPool, + /// Maximum connections configured for this pool (from [`DbConfig::max_connections`]). + pub(crate) max_connections: u32, } /// Snapshot of Postgres connection pool utilisation. @@ -138,6 +140,8 @@ pub struct DbPoolStats { pub size: u32, /// Connections available for immediate reuse. pub idle: u32, + /// Pool ceiling — the `max_connections` value set at construction. + pub max: u32, } /// Configuration for the Postgres connection pool. @@ -210,12 +214,18 @@ impl Db { .idle_timeout(Duration::from_secs(config.idle_timeout_secs)) .connect(&config.database_url) .await?; - Ok(Self { pool }) + Ok(Self { + pool, + max_connections: config.max_connections, + }) } /// Creates a `Db` from an existing `PgPool` (useful in tests). pub fn from_pool(pool: PgPool) -> Self { - Self { pool } + Self { + max_connections: pool.options().get_max_connections(), + pool, + } } /// Run pending database migrations. @@ -232,10 +242,12 @@ impl Db { /// /// `size` — total connections (idle + active) /// `idle` — connections available for immediate reuse + /// `max` — pool ceiling set at construction pub fn pool_stats(&self) -> DbPoolStats { DbPoolStats { size: self.pool.size(), idle: self.pool.num_idle() as u32, + max: self.max_connections, } } @@ -2537,7 +2549,7 @@ mod tests { let pool = PgPool::connect(TEST_DB_URL) .await .expect("connect to test DB"); - Db { pool } + Db::from_pool(pool) } async fn make_community(pool: &PgPool) -> Uuid { diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index 3bde131ff5..d4cc3664ac 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -40,7 +40,6 @@ tracing-opentelemetry = { workspace = true } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } opentelemetry-otlp = { workspace = true } -opentelemetry-semantic-conventions = { workspace = true } thiserror = { workspace = true } anyhow = { workspace = true } uuid = { workspace = true } diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index f385cc3972..6feb907f2f 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -36,11 +36,14 @@ async fn main() -> anyhow::Result<()> { // via OTEL_SERVICE_NAME) for the trace provider so that Datadog can identify // spans under the correct service identity. let resource = telemetry::service_resource(); - let tracer_provider = telemetry::try_init_tracer(resource.clone()); - let otel_layer = tracer_provider.as_ref().map(|p| { - use opentelemetry::trace::TracerProvider as _; - tracing_opentelemetry::layer().with_tracer(p.tracer("buzz-relay")) - }); + let tracer_init = telemetry::try_init_tracer(resource.clone()); + let otel_layer = match &tracer_init { + telemetry::TracerInit::Enabled(p) => { + use opentelemetry::trace::TracerProvider as _; + Some(tracing_opentelemetry::layer().with_tracer(p.tracer("buzz-relay"))) + } + _ => None, + }; tracing_subscriber::registry() .with(fmt::layer().json().flatten_event(true)) @@ -48,6 +51,11 @@ async fn main() -> anyhow::Result<()> { .with(otel_layer) .init(); + // Log any exporter-build failure now that the subscriber is installed. + if let telemetry::TracerInit::ExporterBuildFailed(ref e) = tracer_init { + warn!(error = %e, "Failed to build OTLP trace exporter; distributed tracing disabled"); + } + info!("Starting buzz-relay"); let config = Config::from_env().map_err(|e| { @@ -734,6 +742,7 @@ async fn main() -> anyhow::Result<()> { metrics::gauge!("buzz_db_pool_size").set(db_stats.size as f64); metrics::gauge!("buzz_db_pool_idle").set(db_stats.idle as f64); metrics::gauge!("buzz_db_pool_active").set(active as f64); + metrics::gauge!("buzz_db_pool_max").set(db_stats.max as f64); let rs = pool_state.redis_pool.status(); metrics::gauge!("buzz_redis_pool_available").set(rs.available as f64); @@ -754,7 +763,7 @@ async fn main() -> anyhow::Result<()> { .await; // Flush pending OTEL spans before exit. - if let Some(tp) = tracer_provider { + if let telemetry::TracerInit::Enabled(tp) = tracer_init { if let Err(e) = tp.shutdown() { tracing::warn!(error = %e, "OTEL tracer provider shutdown error"); } diff --git a/crates/buzz-relay/src/telemetry.rs b/crates/buzz-relay/src/telemetry.rs index 7324e0fab4..b9f3f7d170 100644 --- a/crates/buzz-relay/src/telemetry.rs +++ b/crates/buzz-relay/src/telemetry.rs @@ -57,14 +57,28 @@ pub fn service_resource() -> Resource { .build() } -/// Build and install the OTEL tracer provider, returning it so the caller -/// can register a shutdown hook. +/// Outcome of [`try_init_tracer`], distinguishing the two disabled states so +/// the caller can log appropriately after the tracing subscriber is installed. +pub enum TracerInit { + /// OTLP endpoint configured; provider is live and ready. + Enabled(SdkTracerProvider), + /// `OTEL_EXPORTER_OTLP_ENDPOINT` was unset — no-op, no connection. + Disabled, + /// Endpoint was set but the exporter failed to build. The inner error + /// string is suitable for a `tracing::warn!` call made by the caller + /// **after** `tracing_subscriber::registry()…init()`. + ExporterBuildFailed(String), +} + +/// Build and install the OTEL tracer provider, returning the outcome so the +/// caller can act after the tracing subscriber is ready. /// -/// Returns `None` when `OTEL_EXPORTER_OTLP_ENDPOINT` is unset — in that -/// case the tracing subscriber stack is unchanged. -pub fn try_init_tracer(resource: Resource) -> Option { +/// Deliberately does **not** call `tracing::warn!` internally — the subscriber +/// may not be installed yet at call time, which would silently drop the event. +/// Callers are responsible for logging [`TracerInit::ExporterBuildFailed`]. +pub fn try_init_tracer(resource: Resource) -> TracerInit { if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_err() { - return None; + return TracerInit::Disabled; } match opentelemetry_otlp::SpanExporter::builder() @@ -77,14 +91,111 @@ pub fn try_init_tracer(resource: Resource) -> Option { .with_batch_exporter(exporter) .build(); opentelemetry::global::set_tracer_provider(provider.clone()); - Some(provider) - } - Err(e) => { - tracing::warn!( - error = %e, - "Failed to build OTLP trace exporter; distributed tracing disabled" - ); - None + TracerInit::Enabled(provider) } + Err(e) => TracerInit::ExporterBuildFailed(e.to_string()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::KeyValue; + use std::sync::Mutex; + + // Env vars are process-global — serialize tests that mutate them to prevent + // cross-test races when the suite runs with multiple threads. + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + // Helper: read service.name from the Resource's schema_url-independent KV list. + fn service_name_from(resource: &Resource) -> Option { + resource + .iter() + .find(|(k, _)| k.as_str() == "service.name") + .map(|(_, v)| v.to_string()) + } + + #[test] + fn test_service_resource_default_when_env_unset() { + let _guard = ENV_LOCK.lock().unwrap(); + std::env::remove_var("OTEL_SERVICE_NAME"); + std::env::remove_var("OTEL_RESOURCE_ATTRIBUTES"); + + let r = service_resource(); + assert_eq!( + service_name_from(&r).as_deref(), + Some("buzz-relay"), + "expected buzz-relay fallback when OTEL_SERVICE_NAME is unset" + ); + } + + #[test] + fn test_service_resource_honors_otel_service_name() { + let _guard = ENV_LOCK.lock().unwrap(); + std::env::set_var("OTEL_SERVICE_NAME", "my-custom-relay"); + std::env::remove_var("OTEL_RESOURCE_ATTRIBUTES"); + + let r = service_resource(); + std::env::remove_var("OTEL_SERVICE_NAME"); + + assert_eq!( + service_name_from(&r).as_deref(), + Some("my-custom-relay"), + "expected OTEL_SERVICE_NAME to be honoured" + ); + } + + #[test] + fn test_service_resource_empty_string_falls_back_to_default() { + let _guard = ENV_LOCK.lock().unwrap(); + std::env::set_var("OTEL_SERVICE_NAME", ""); + std::env::remove_var("OTEL_RESOURCE_ATTRIBUTES"); + + let r = service_resource(); + std::env::remove_var("OTEL_SERVICE_NAME"); + + assert_eq!( + service_name_from(&r).as_deref(), + Some("buzz-relay"), + "expected buzz-relay fallback when OTEL_SERVICE_NAME is empty" + ); + } + + #[test] + fn test_try_init_tracer_disabled_when_endpoint_unset() { + let _guard = ENV_LOCK.lock().unwrap(); + std::env::remove_var("OTEL_EXPORTER_OTLP_ENDPOINT"); + + let resource = Resource::builder_empty() + .with_attribute(KeyValue::new("service.name", "test")) + .build(); + let result = try_init_tracer(resource); + + assert!( + matches!(result, TracerInit::Disabled), + "expected Disabled when OTEL_EXPORTER_OTLP_ENDPOINT is unset" + ); + } + + #[test] + fn test_try_init_tracer_exporter_build_failed_on_bad_endpoint() { + let _guard = ENV_LOCK.lock().unwrap(); + // Set endpoint to a value that causes tonic to reject during build + // (invalid URI scheme forces a build-time error). + std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", "not-a-valid-uri:///bad"); + + let resource = Resource::builder_empty() + .with_attribute(KeyValue::new("service.name", "test")) + .build(); + let result = try_init_tracer(resource); + std::env::remove_var("OTEL_EXPORTER_OTLP_ENDPOINT"); + + // Either ExporterBuildFailed (build rejected the URI) or Enabled + // (tonic accepted it for lazy connection) — both are valid SDK + // behaviours. What we assert is it is NOT Disabled. + assert!( + !matches!(result, TracerInit::Disabled), + "expected Enabled or ExporterBuildFailed when endpoint is set" + ); } } From 361e7801d4fa5306fad2e5a161988b506c96843b Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Wed, 1 Jul 2026 14:59:38 -0400 Subject: [PATCH 6/6] test(relay): pin ExporterBuildFailed mapping via classify_exporter_result helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract Err->TracerInit classification into a private classify_exporter_result fn so the test can synthesize an ExporterBuildError directly rather than relying on tonic's URI-validation behaviour at build time (which may lazily accept bad URIs). Rewrite test_try_init_tracer_exporter_build_failed_on_bad_endpoint as test_classify_exporter_result_maps_err_to_exporter_build_failed: constructs ExporterBuildError::InvalidUri directly, asserts matches!(result, TracerInit::ExporterBuildFailed(_)). No env vars needed — no lock required. Production behaviour unchanged: try_init_tracer delegates to the helper. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- crates/buzz-relay/src/telemetry.rs | 45 +++++++++++++++++++----------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/crates/buzz-relay/src/telemetry.rs b/crates/buzz-relay/src/telemetry.rs index b9f3f7d170..8dd305cd42 100644 --- a/crates/buzz-relay/src/telemetry.rs +++ b/crates/buzz-relay/src/telemetry.rs @@ -23,6 +23,7 @@ //! - `OTEL_TRACES_SAMPLER` (default: `parentbased_always_on`) //! - `OTEL_TRACES_SAMPLER_ARG` +use opentelemetry_otlp::ExporterBuildError; use opentelemetry_sdk::{resource::EnvResourceDetector, trace::SdkTracerProvider, Resource}; /// Build the OTEL [`Resource`] used by the trace provider. @@ -81,10 +82,21 @@ pub fn try_init_tracer(resource: Resource) -> TracerInit { return TracerInit::Disabled; } - match opentelemetry_otlp::SpanExporter::builder() + let result = opentelemetry_otlp::SpanExporter::builder() .with_tonic() - .build() - { + .build(); + classify_exporter_result(result, resource) +} + +/// Map a `Result` to a [`TracerInit`] variant. +/// +/// Extracted so the `Err → ExporterBuildFailed` classification can be unit-tested +/// deterministically without relying on SDK behaviour for specific URI inputs. +fn classify_exporter_result( + result: Result, + resource: Resource, +) -> TracerInit { + match result { Ok(exporter) => { let provider = SdkTracerProvider::builder() .with_resource(resource) @@ -177,25 +189,26 @@ mod tests { ); } + /// Pin the `Err(ExporterBuildError) → TracerInit::ExporterBuildFailed` mapping + /// deterministically by synthesising an error value directly, without relying on + /// SDK/tonic behaviour for a particular URI at build time. #[test] - fn test_try_init_tracer_exporter_build_failed_on_bad_endpoint() { - let _guard = ENV_LOCK.lock().unwrap(); - // Set endpoint to a value that causes tonic to reject during build - // (invalid URI scheme forces a build-time error). - std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", "not-a-valid-uri:///bad"); - + fn test_classify_exporter_result_maps_err_to_exporter_build_failed() { let resource = Resource::builder_empty() .with_attribute(KeyValue::new("service.name", "test")) .build(); - let result = try_init_tracer(resource); - std::env::remove_var("OTEL_EXPORTER_OTLP_ENDPOINT"); - // Either ExporterBuildFailed (build rejected the URI) or Enabled - // (tonic accepted it for lazy connection) — both are valid SDK - // behaviours. What we assert is it is NOT Disabled. + // Construct a concrete ExporterBuildError variant directly. + // InvalidUri is available under the grpc-tonic feature we already depend on. + let err = ExporterBuildError::InvalidUri( + "bad-scheme://host".to_string(), + "unsupported scheme".to_string(), + ); + let result = classify_exporter_result(Err(err), resource); + assert!( - !matches!(result, TracerInit::Disabled), - "expected Enabled or ExporterBuildFailed when endpoint is set" + matches!(result, TracerInit::ExporterBuildFailed(_)), + "expected ExporterBuildFailed when classify_exporter_result receives an Err" ); } }