From af81c0627a3c1dfb7650ab1be9083fd0ffb748d3 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 18:17:59 +0000 Subject: [PATCH 01/37] fix(node): honor disabled optional indexes Keep txindex and compact-filter index handles optional so disabled indexes are neither opened nor advertised, while RPC lookup paths preserve Core-compatible disabled-index behavior. Verification: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-rpc --lib --no-fail-fast - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-rpc --test handler_smoke --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --lib --no-default-features --features $FEATURES --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --test state_storage --no-default-features --features $FEATURES --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --test rpc_wiring --no-default-features --features $FEATURES --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --test sync_smoke --no-default-features --features $FEATURES --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features $FEATURES -- -D warnings - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --no-default-features --features $FEATURES --no-fail-fast - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast Op: correct Restores: spec: optional txindex/blockfilterindex disabled semantics --- crates/node/src/apply.rs | 73 +++++++++++++------------ crates/node/src/config.rs | 3 ++ crates/node/src/run.rs | 7 +-- crates/node/src/state.rs | 64 +++++++++++++--------- crates/node/src/sync.rs | 4 +- crates/node/tests/rpc_wiring.rs | 24 +++++++-- crates/node/tests/state_storage.rs | 36 +++++++++++++ crates/node/tests/sync_smoke.rs | 4 +- crates/rpc/src/context.rs | 75 +++++++++++++------------- crates/rpc/src/error.rs | 4 ++ crates/rpc/src/handlers/chain.rs | 28 ++++++---- crates/rpc/src/handlers/tx.rs | 2 +- crates/rpc/tests/handler_smoke.rs | 87 ++++++++++++++++++++++++++---- 13 files changed, 281 insertions(+), 130 deletions(-) diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index 4165a7d..306e062 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -65,10 +65,10 @@ pub struct ApplyHandles { pub utxo: Arc, /// Shared coinstats listener. pub coin_stats: Arc, - /// Shared best-effort confirmed transaction indexer. - pub tx_index: Arc>>, - /// Shared best-effort compact-filter indexer. - pub filter_index: Arc>, + /// Shared best-effort confirmed transaction indexer when txindex is enabled. + pub tx_index: Option>>>, + /// Shared best-effort compact-filter indexer when blockfilterindex is enabled. + pub filter_index: Option>>, /// Shared mempool. pub mempool: Arc>, /// Shared block records exposed to RPC handlers. @@ -92,8 +92,8 @@ impl ApplyHandles { block_tree: Arc>, utxo: Arc, coin_stats: Arc, - tx_index: Arc>>, - filter_index: Arc>, + tx_index: Option>>>, + filter_index: Option>>, mempool: Arc>, blocks: Arc>>, transactions: Arc>>, @@ -254,7 +254,11 @@ pub fn apply_block( metrics::histogram!("node.apply_block.bip68_seconds").record(bip68_dur.as_secs_f64()); bip68_result?; - let filter_bytes = compute_basic_filter(block, handles, block_hash, height)?; + let filter_bytes = if handles.filter_index.is_some() { + compute_basic_filter(block, handles, block_hash, height)? + } else { + None + }; let block_bytes = bitcoin::consensus::encode::serialize(block); @@ -310,40 +314,39 @@ pub fn apply_block( metrics::histogram!("node.apply_block.coin_stats_finish_seconds") .record(coin_stats_dur.as_secs_f64()); let tx_index_ingest_started = quanta::Instant::now(); - let tx_index_ingest_result = handles.tx_index.lock().ingest_block(&block_bytes, height); - match tx_index_ingest_result { - Ok(counts) => { - tracing::debug!( - height, - txids = counts.txids, - funding = counts.funding, - spending = counts.spending, - headers = counts.headers, - "tx_index ingested block" - ); - } - Err(error) => { - tracing::warn!( - height, - %error, - "tx_index failed to ingest block; best-effort path continues" - ); + if let Some(tx_index) = &handles.tx_index { + let tx_index_ingest_result = tx_index.lock().ingest_block(&block_bytes, height); + match tx_index_ingest_result { + Ok(counts) => { + tracing::debug!( + height, + txids = counts.txids, + funding = counts.funding, + spending = counts.spending, + headers = counts.headers, + "tx_index ingested block" + ); + } + Err(error) => { + tracing::warn!( + height, + %error, + "tx_index failed to ingest block; best-effort path continues" + ); + } } } let tx_index_ingest_dur = tx_index_ingest_started.elapsed(); metrics::histogram!("node.apply_block.tx_index_ingest_seconds") .record(tx_index_ingest_dur.as_secs_f64()); let filter_started = quanta::Instant::now(); - if let Some(filter_bytes) = filter_bytes { + if let (Some(filter_index), Some(filter_bytes)) = (&handles.filter_index, filter_bytes) { let prev_filter_header = handles .applied_tip .load_full() - .and_then(|tip| handles.filter_index.filter_header(tip.hash).ok().flatten()) + .and_then(|tip| filter_index.filter_header(tip.hash).ok().flatten()) .unwrap_or_default(); - match handles - .filter_index - .put_filter(block_hash, prev_filter_header, &filter_bytes) - { + match filter_index.put_filter(block_hash, prev_filter_header, &filter_bytes) { Ok(filter_header) => { tracing::debug!( height, @@ -2360,8 +2363,8 @@ mod consensus_rule_tests { Arc::new(bitcoin_rs_coinstats::CoinStatsListener::new( bitcoin_rs_coinstats::CoinStats::default(), )), - noop_tx_index(), - filter_index, + Some(noop_tx_index()), + Some(filter_index), Arc::new(RwLock::new(Mempool::new(MempoolLimits::default()))), Arc::new(RwLock::new(Vec::new())), Arc::new(RwLock::new(HashMap::::new())), @@ -2379,8 +2382,8 @@ mod consensus_rule_tests { Arc::new(bitcoin_rs_coinstats::CoinStatsListener::new( bitcoin_rs_coinstats::CoinStats::default(), )), - noop_tx_index(), - noop_filter_index(), + Some(noop_tx_index()), + Some(noop_filter_index()), Arc::new(RwLock::new(Mempool::new(MempoolLimits::default()))), Arc::new(RwLock::new(Vec::new())), Arc::new(RwLock::new(HashMap::::new())), diff --git a/crates/node/src/config.rs b/crates/node/src/config.rs index 9334289..dec7e0d 100644 --- a/crates/node/src/config.rs +++ b/crates/node/src/config.rs @@ -272,6 +272,9 @@ impl Config { if self.electrum_tls_cert.is_some() && self.electrum_bind.is_none() { bail!("electrum_tls_cert requires electrum_bind"); } + if self.electrum_bind.is_some() && !self.txindex { + bail!("electrum_bind requires txindex"); + } match (&self.g2_muhash_samples, self.g2_muhash_tip_height) { (Some(_), Some(0)) => bail!("g2_muhash_tip_height must be greater than zero"), (Some(_), None) => bail!("g2_muhash_samples requires g2_muhash_tip_height"), diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index 9f4c8cd..0e29ae8 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -67,9 +67,10 @@ fn spawn_electrum_listener( bitcoin_rs_primitives::Network::Signet => bitcoin::Network::Signet, bitcoin_rs_primitives::Network::Regtest => bitcoin::Network::Regtest, }; + let history_reader = state.electrum_history_reader()?; let index = state - .electrum_index_handle() - .with_history_reader(state.electrum_history_reader()) + .electrum_index_handle()? + .with_history_reader(history_reader) .with_network(network); let mempool = bitcoin_rs_electrum::MempoolHandle::from_arc(state.mempool()); let cfg = bitcoin_rs_electrum::ServerConfig::default(); @@ -378,7 +379,7 @@ pub fn run(mut config: Config) -> Result<()> { Some(state.p2p_outbound_sender()), Arc::clone(&banned), Arc::new(parking_lot::RwLock::new(Vec::new())), - Some(state.tx_index()), + state.tx_index(), ); if let Some(prune_service) = state.prune_service() { rpc_context = rpc_context.with_prune_service(prune_service); diff --git a/crates/node/src/state.rs b/crates/node/src/state.rs index 0e740bd..f6f3bd9 100644 --- a/crates/node/src/state.rs +++ b/crates/node/src/state.rs @@ -603,9 +603,9 @@ pub struct NodeState { storage: NodeStorage, utxo: Arc, coin_stats: Arc, - tx_index: TxIndexHandle, - tx_index_storage: Arc, - filter_index: FilterIndexHandle, + tx_index: Option, + tx_index_storage: Option>, + filter_index: Option, prune_service: Option>, zmq_publisher: Arc, active_zmq_notifications: Vec, @@ -653,9 +653,17 @@ impl NodeState { .context("open G2 MuHash sample writer")? .map(Arc::new); let storage = NodeStorage::open(&config)?; - let (tx_index, tx_index_storage) = open_tx_index(&config)?; - let tx_index_storage = Arc::new(tx_index_storage); - let filter_index = open_filter_index(&config)?; + let (tx_index, tx_index_storage) = if config.txindex { + let (tx_index, tx_index_storage) = open_tx_index(&config)?; + (Some(tx_index), Some(Arc::new(tx_index_storage))) + } else { + (None, None) + }; + let filter_index = if config.blockfilterindex { + Some(open_filter_index(&config)?) + } else { + None + }; let zmq_publications = config.zmq_publications(); let active_zmq_notifications: Vec<_> = zmq_publications .iter() @@ -707,8 +715,8 @@ impl NodeState { block_tree: Arc::clone(&block_tree), utxo: Arc::clone(&utxo), coin_stats: Arc::clone(&coin_stats), - tx_index: Arc::clone(&tx_index), - filter_index: Arc::clone(&filter_index), + tx_index: tx_index.as_ref().map(Arc::clone), + filter_index: filter_index.as_ref().map(Arc::clone), mempool: Arc::clone(&mempool), blocks: Arc::clone(&blocks), transactions: Arc::clone(&transactions), @@ -739,7 +747,7 @@ impl NodeState { utxo, coin_stats, tx_index, - tx_index_storage: Arc::clone(&tx_index_storage), + tx_index_storage, filter_index, prune_service, zmq_publisher, @@ -799,8 +807,8 @@ impl NodeState { /// Returns the shared block indexer handle. #[must_use] - pub fn tx_index(&self) -> Arc>> { - Arc::clone(&self.tx_index) + pub fn tx_index(&self) -> Option>>> { + self.tx_index.as_ref().map(Arc::clone) } /// Builds an Electrum `IndexHandle` backed by the live txindex store. @@ -808,25 +816,29 @@ impl NodeState { /// The handle observes the same `KvStore` the writer side ingests into via /// `apply_block`, so `blockchain.block.headers` returns real data once IBD /// is underway. - #[must_use] - pub fn electrum_index_handle(&self) -> bitcoin_rs_electrum::IndexHandle { - self.tx_index_storage.electrum_index_handle() + pub fn electrum_index_handle(&self) -> Result { + self.tx_index_storage + .as_ref() + .map(|storage| storage.electrum_index_handle()) + .context("electrum requires txindex") } /// Builds an Electrum-side history reader wired through the live txindex store /// and the in-memory block log. The handle can be attached to `IndexHandle` /// via `with_history_reader`. - #[must_use] pub fn electrum_history_reader( &self, - ) -> Arc { - self.tx_index_storage.electrum_history_reader(self.blocks()) + ) -> Result> { + self.tx_index_storage + .as_ref() + .map(|storage| storage.electrum_history_reader(self.blocks())) + .context("electrum requires txindex") } /// Returns the shared compact-filter index handle. #[must_use] - pub fn filter_index(&self) -> FilterIndexHandle { - Arc::clone(&self.filter_index) + pub fn filter_index(&self) -> Option { + self.filter_index.as_ref().map(Arc::clone) } /// Returns the manual pruning service when pruning is enabled. @@ -1013,8 +1025,8 @@ impl NodeState { block_tree: Arc::clone(&self.block_tree), utxo: Arc::clone(&self.utxo), coin_stats: Arc::clone(&self.coin_stats), - tx_index: Arc::clone(&self.tx_index), - filter_index: Arc::clone(&self.filter_index), + tx_index: self.tx_index.as_ref().map(Arc::clone), + filter_index: self.filter_index.as_ref().map(Arc::clone), mempool: Arc::clone(&self.mempool), blocks: Arc::clone(&self.blocks), transactions: Arc::clone(&self.transactions), @@ -1114,9 +1126,10 @@ mod tests { let mut config = crate::Config::default_for_network(crate::Network::Regtest); config.data_dir = dir.path().join("node"); config.p2p_listen.clear(); + config.txindex = true; let state = NodeState::open(config)?; - let a = state.tx_index(); - let b = state.tx_index(); + let a = state.tx_index().expect("txindex enabled"); + let b = state.tx_index().expect("txindex enabled"); assert!(Arc::ptr_eq(&a, &b), "tx_index handle stable across calls"); Ok(()) } @@ -1142,9 +1155,10 @@ mod tests { let mut config = crate::Config::default_for_network(crate::Network::Regtest); config.data_dir = dir.path().join("node"); config.p2p_listen.clear(); + config.blockfilterindex = true; let state = NodeState::open(config)?; - let a = state.filter_index(); - let b = state.filter_index(); + let a = state.filter_index().expect("blockfilterindex enabled"); + let b = state.filter_index().expect("blockfilterindex enabled"); assert!( Arc::ptr_eq(&a, &b), "filter_index handle stable across calls" diff --git a/crates/node/src/sync.rs b/crates/node/src/sync.rs index 55dd28e..643dd87 100644 --- a/crates/node/src/sync.rs +++ b/crates/node/src/sync.rs @@ -819,8 +819,8 @@ mod tests { Arc::new(bitcoin_rs_coinstats::CoinStatsListener::new( bitcoin_rs_coinstats::CoinStats::default(), )), - noop_tx_index(), - noop_filter_index(), + Some(noop_tx_index()), + Some(noop_filter_index()), Arc::new(RwLock::new(Mempool::new(MempoolLimits::default()))), Arc::new(RwLock::new(Vec::new())), Arc::new(RwLock::new(HashMap::::new())), diff --git a/crates/node/tests/rpc_wiring.rs b/crates/node/tests/rpc_wiring.rs index 2957b6c..80b732d 100644 --- a/crates/node/tests/rpc_wiring.rs +++ b/crates/node/tests/rpc_wiring.rs @@ -21,6 +21,8 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { let dir = tempdir()?; let mut config = Config::default(); config.data_dir = dir.path().join("node"); + config.txindex = true; + config.blockfilterindex = true; config.zmqpubhashblock = vec!["inproc://rpc-wiring-zmq-pubhashblock".to_owned()]; config.zmqpubhashblockhwm = Some(21); let state = NodeState::open(config)?; @@ -51,7 +53,7 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { Arc::clone(&transactions), Arc::clone(&utxo), Arc::clone(&coin_stats), - Arc::clone(&filter_index), + filter_index.clone(), Arc::clone(&network), Arc::clone(&mining_template_id), Arc::clone(&peers), @@ -61,7 +63,7 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { p2p_outbound, Arc::clone(&banned), Arc::clone(&added_nodes), - Some(Arc::clone(&tx_index)), + tx_index.clone(), ) .with_zmq_notifications(state.active_zmq_notifications()); @@ -91,10 +93,24 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { "coin_stats must share identity" ); assert!( - Arc::ptr_eq(&ctx.filter_index, &filter_index), + Arc::ptr_eq( + ctx.filter_index + .as_ref() + .expect("filter index must be wired"), + filter_index + .as_ref() + .expect("node filter index must be wired") + ), "filter_index must share identity" ); - assert!(ctx.indexer.is_some(), "indexer handle must be wired"); + let ctx_indexer = ctx.indexer.as_ref().expect("indexer handle must be wired"); + assert!( + Arc::ptr_eq( + ctx_indexer, + tx_index.as_ref().expect("node txindex must be wired") + ), + "indexer handle must share identity" + ); assert!( Arc::ptr_eq(&ctx.network, &network), "network must share identity" diff --git a/crates/node/tests/state_storage.rs b/crates/node/tests/state_storage.rs index bdd15a7..684987e 100644 --- a/crates/node/tests/state_storage.rs +++ b/crates/node/tests/state_storage.rs @@ -2,6 +2,7 @@ use anyhow::Result; use bitcoin_rs_node::{Config, Network, state::NodeState}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; #[test] fn opens_storage_backend() -> Result<()> { @@ -17,6 +18,41 @@ fn opens_storage_backend() -> Result<()> { Ok(()) } +#[test] +fn optional_indexes_are_not_opened_when_disabled() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut config = Config::default_for_network(Network::Regtest); + config.data_dir = temp.path().join("node"); + config.p2p_listen.clear(); + config.txindex = false; + config.blockfilterindex = false; + + let state = NodeState::open(config)?; + + assert!(state.tx_index().is_none()); + assert!(state.filter_index().is_none()); + assert!(!state.data_dir().join("txindex").exists()); + assert!(!state.data_dir().join("filters").exists()); + Ok(()) +} + +#[test] +fn electrum_bind_requires_txindex() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut config = Config::default_for_network(Network::Regtest); + config.data_dir = temp.path().join("node"); + config.p2p_listen.clear(); + config.electrum_bind = Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)); + config.txindex = false; + + let error = config + .validate() + .expect_err("electrum without txindex unexpectedly validated"); + + assert!(error.to_string().contains("electrum_bind requires txindex")); + Ok(()) +} + fn assert_backend_opens(backend: &str) -> Result<()> { let temp = tempfile::tempdir()?; let mut config = Config::default_for_network(Network::Regtest); diff --git a/crates/node/tests/sync_smoke.rs b/crates/node/tests/sync_smoke.rs index 0517934..285b4ad 100644 --- a/crates/node/tests/sync_smoke.rs +++ b/crates/node/tests/sync_smoke.rs @@ -469,8 +469,8 @@ fn apply_handles_with_coin_stats_and_utxo( block_tree, Arc::clone(&utxo), Arc::clone(&coin_stats), - noop_tx_index(), - noop_filter_index(), + Some(noop_tx_index()), + Some(noop_filter_index()), Arc::new(RwLock::new(Mempool::new(MempoolLimits::default()))), Arc::new(RwLock::new(Vec::new())), Arc::new(RwLock::new(HashMap::::new())), diff --git a/crates/rpc/src/context.rs b/crates/rpc/src/context.rs index 714482e..ad79d58 100644 --- a/crates/rpc/src/context.rs +++ b/crates/rpc/src/context.rs @@ -151,39 +151,6 @@ pub trait PruneService: Send + Sync { /// Reports whether pruning is enabled and the highest completed prune height. fn status(&self) -> PruneStatus; } -#[derive(Debug, Default)] -struct NoopFilterIndex; - -impl bitcoin_rs_filters::FilterIndexLike for NoopFilterIndex { - fn put_filter( - &self, - _block_hash: bitcoin_rs_primitives::Hash256, - _prev_header: bitcoin_rs_primitives::Hash256, - _filter_bytes: &[u8], - ) -> Result { - Ok(bitcoin_rs_primitives::Hash256::default()) - } - - fn filter_header( - &self, - _block_hash: bitcoin_rs_primitives::Hash256, - ) -> Result, bitcoin_rs_filters::FilterIndexError> { - Ok(None) - } - - fn filter( - &self, - _block_hash: bitcoin_rs_primitives::Hash256, - ) -> Result>, bitcoin_rs_filters::FilterIndexError> { - Ok(None) - } -} - -fn noop_filter_index() -> Arc> { - let filter_index: Box = Box::new(NoopFilterIndex); - Arc::new(filter_index) -} - /// Shared state consumed by JSON-RPC handlers. pub struct Context { /// Best-chain tip snapshot published by chain validation. @@ -200,8 +167,8 @@ pub struct Context { pub utxo: Arc, /// Incremental UTXO-set statistics. pub coin_stats: Arc, - /// BIP157/158 compact-filter index used by filter RPCs. - pub filter_index: Arc>, + /// BIP157/158 compact-filter index used by filter RPCs when enabled. + pub filter_index: Option>>, /// Optional storage pruning mutator. pub prune_service: Option>, /// Optional shared confirmed-block indexer used to resolve prevout values for fee statistics. @@ -278,7 +245,7 @@ impl Context { transactions: Arc::new(RwLock::new(HashMap::new())), utxo: Arc::new(utxo), coin_stats, - filter_index: noop_filter_index(), + filter_index: None, indexer: None, prune_service: None, network: Arc::new(RwLock::new(NetworkState::default())), @@ -311,7 +278,7 @@ impl Context { transactions: Arc>>, utxo: Arc, coin_stats: Arc, - filter_index: Arc>, + filter_index: Option>>, network: Arc>, mining_template_id: Arc>, peers: Arc>>, @@ -585,6 +552,31 @@ fn target_to_f64(target: bitcoin::pow::Target) -> f64 { mod tests { use super::*; + struct NoopFilterIndex; + + impl bitcoin_rs_filters::FilterIndexLike for NoopFilterIndex { + fn put_filter( + &self, + _block_hash: bitcoin_rs_primitives::Hash256, + prev_header: bitcoin_rs_primitives::Hash256, + _filter_bytes: &[u8], + ) -> Result { + Ok(prev_header) + } + + fn filter_header( + &self, + _block_hash: bitcoin_rs_primitives::Hash256, + ) -> Result, bitcoin_rs_filters::FilterIndexError> + { + Ok(None) + } + } + + fn noop_filter_index() -> Arc> { + Arc::new(Box::new(NoopFilterIndex)) + } + #[test] #[allow(clippy::arc_with_non_send_sync)] fn from_handles_shares_tip_handles_with_caller() { @@ -608,7 +600,7 @@ mod tests { Arc::new(RwLock::new(HashMap::new())), Arc::clone(&utxo), Arc::clone(&coin_stats), - Arc::clone(&filter_index), + Some(Arc::clone(&filter_index)), Arc::new(RwLock::new(NetworkState::default())), Arc::new(ArcSwap::from_pointee(CompactString::new("0"))), Arc::new(RwLock::new(Vec::new())), @@ -637,7 +629,12 @@ mod tests { "coin_stats must be shared with caller" ); assert!( - Arc::ptr_eq(&ctx.filter_index, &filter_index), + Arc::ptr_eq( + ctx.filter_index + .as_ref() + .expect("filter_index must be wired"), + &filter_index + ), "filter_index must be shared with caller" ); assert!( diff --git a/crates/rpc/src/error.rs b/crates/rpc/src/error.rs index 2fa99ec..6072354 100644 --- a/crates/rpc/src/error.rs +++ b/crates/rpc/src/error.rs @@ -28,6 +28,9 @@ pub enum RpcError { /// A method is intentionally disabled by policy. #[error("{0}")] MethodDisabled(&'static str), + /// A requested optional index is disabled. + #[error("{0}")] + IndexDisabled(&'static str), /// Internal server failure. #[error("internal error: {0}")] Internal(String), @@ -67,6 +70,7 @@ impl RpcError { Self::InvalidParams(_) => Self::INVALID_PARAMS, Self::InvalidType(_) => Self::CORE_INVALID_TYPE, Self::NotFound(_) => Self::CORE_NOT_FOUND, + Self::IndexDisabled(_) => -1, Self::MethodDisabled(_) | Self::Internal(_) => Self::INTERNAL_ERROR, } } diff --git a/crates/rpc/src/handlers/chain.rs b/crates/rpc/src/handlers/chain.rs index a986b35..0a9f0a2 100644 --- a/crates/rpc/src/handlers/chain.rs +++ b/crates/rpc/src/handlers/chain.rs @@ -652,13 +652,14 @@ pub(crate) fn gettxoutsetinfo(ctx: &Arc, params: &Value) -> Result, params: &Value) -> Result { let hash = required_str(params, 0, "block hash is required")?; let hash = parse_hash(hash)?; - let filter_bytes = ctx - .filter_index + let filter_index = ctx.filter_index.as_ref().ok_or(RpcError::IndexDisabled( + "Index is not enabled for filtertype basic", + ))?; + let filter_bytes = filter_index .filter(hash) .map_err(|error| RpcError::Internal(error.to_string()))? .ok_or(RpcError::NotFound("block filter not found"))?; - let header = ctx - .filter_index + let header = filter_index .filter_header(hash) .map_err(|error| RpcError::Internal(error.to_string()))? .ok_or(RpcError::NotFound("block filter header not found"))?; @@ -691,13 +692,20 @@ pub(crate) fn getindexinfo(ctx: &Arc, params: &Value) -> Result Ok(json!({ - "txindex": entry(), - "basicblockfilterindex": entry(), - })), - Some("txindex") => Ok(json!({ "txindex": entry() })), - Some("basicblockfilterindex") => Ok(json!({ "basicblockfilterindex": entry() })), + None => Ok(Value::from(indexes)), + Some("txindex") if ctx.indexer.is_some() => Ok(json!({ "txindex": entry() })), + Some("basic block filter index") if ctx.filter_index.is_some() => { + Ok(json!({ "basic block filter index": entry() })) + } Some(_) => Ok(json!({})), } } diff --git a/crates/rpc/src/handlers/tx.rs b/crates/rpc/src/handlers/tx.rs index 0ec8504..0590180 100644 --- a/crates/rpc/src/handlers/tx.rs +++ b/crates/rpc/src/handlers/tx.rs @@ -42,7 +42,7 @@ pub(crate) fn getrawtransaction(ctx: &Arc, params: &Value) -> Result Result<(), Box> filter: vec![0xab, 0xcd], header, }); - ctx.filter_index = Arc::new(filter_index); + ctx.filter_index = Some(Arc::new(filter_index)); let handler = Handler::new(Arc::new(ctx)); let block_hash_hex = block_hash.to_string_be(); @@ -274,6 +274,23 @@ fn getblockfilter_reads_filter_index() -> Result<(), Box> Ok(()) } +#[test] +fn getblockfilter_reports_disabled_basic_filter_index() { + let handler = Handler::new(Arc::new(Context::new())); + let block_hash = Hash256::from_le_bytes(&[9_u8; 32]); + let block_hash_hex = block_hash.to_string_be(); + + let error = handler + .dispatch("getblockfilter", &json!([block_hash_hex.as_str()])) + .expect_err("disabled block filter index unexpectedly succeeded"); + + assert_eq!(error.code(), -1); + assert_eq!( + error.to_string(), + "Index is not enabled for filtertype basic" + ); +} + #[test] fn getblockfilter_returns_not_found_for_missing_filter_row() -> Result<(), Box> { @@ -285,7 +302,7 @@ fn getblockfilter_returns_not_found_for_missing_filter_row() filter: vec![0xab, 0xcd], header, }); - ctx.filter_index = Arc::new(filter_index); + ctx.filter_index = Some(Arc::new(filter_index)); let handler = Handler::new(Arc::new(ctx)); let missing_hash = Hash256::from_le_bytes(&[7_u8; 32]); let missing_hash_hex = missing_hash.to_string_be(); @@ -301,9 +318,27 @@ fn getblockfilter_returns_not_found_for_missing_filter_row() } #[test] -fn getindexinfo_returns_both_indexes() -> Result<(), Box> { - let ctx = Arc::new(Context::new()); - let handler = Handler::new(Arc::clone(&ctx)); +fn getindexinfo_omits_disabled_indexes() -> Result<(), Box> { + let handler = Handler::new(Arc::new(Context::new())); + + let result = handler.dispatch("getindexinfo", &json!([]))?; + + assert_eq!(result.as_object().map(sonic_rs::Object::len), Some(0)); + Ok(()) +} + +#[test] +fn getindexinfo_returns_enabled_core_index_names() -> Result<(), Box> { + let mut ctx = Context::new(); + ctx.indexer = Some(Arc::new(Mutex::new(Box::new(FakeIndexer { + values: HashMap::new(), + })))); + ctx.filter_index = Some(Arc::new(Box::new(StaticFilterIndex { + block_hash: Hash256::from_le_bytes(&[9_u8; 32]), + filter: vec![0xab, 0xcd], + header: Hash256::from_le_bytes(&[8_u8; 32]), + }))); + let handler = Handler::new(Arc::new(ctx)); let result = handler.dispatch("getindexinfo", &json!([]))?; @@ -312,14 +347,48 @@ fn getindexinfo_returns_both_indexes() -> Result<(), Box> assert_eq!(txindex.get("synced").as_bool(), Some(false)); assert_eq!(txindex.get("best_block_height").as_u64(), Some(0)); - let filter_index = result.get("basicblockfilterindex"); + let filter_index = result.get("basic block filter index"); assert!( filter_index.is_some(), - "basicblockfilterindex entry missing: {result:?}" + "basic block filter index entry missing: {result:?}" ); assert_eq!(filter_index.get("synced").as_bool(), Some(false)); assert_eq!(filter_index.get("best_block_height").as_u64(), Some(0)); + assert!(result.get("basicblockfilterindex").is_none()); + + Ok(()) +} + +#[test] +fn getrawtransaction_does_not_use_confirmed_map_when_txindex_disabled() +-> Result<(), Box> { + let ctx = Context::new(); + let tx = tx(42, ScriptBuf::from_bytes(vec![0x51])); + let txid = ctx.add_transaction(tx); + let handler = Handler::new(Arc::new(ctx)); + + let error = handler + .dispatch("getrawtransaction", &json!([txid.to_string()])) + .expect_err("confirmed map lookup unexpectedly succeeded with txindex disabled"); + + assert_eq!(error.code(), RpcError::CORE_NOT_FOUND); + Ok(()) +} + +#[test] +fn getrawtransaction_uses_confirmed_map_when_txindex_enabled() +-> Result<(), Box> { + let mut ctx = Context::new(); + ctx.indexer = Some(Arc::new(Mutex::new(Box::new(FakeIndexer { + values: HashMap::new(), + })))); + let tx = tx(43, ScriptBuf::from_bytes(vec![0x51])); + let txid = ctx.add_transaction(tx); + let handler = Handler::new(Arc::new(ctx)); + + let result = handler.dispatch("getrawtransaction", &json!([txid.to_string()]))?; + assert!(result.as_str().is_some()); Ok(()) } @@ -662,11 +731,11 @@ impl Fixture { }; let block_hash_bytes = block.block_hash(); let block_hash = Hash256::from_le_bytes(block_hash_bytes.as_byte_array()); - ctx.filter_index = Arc::new(Box::new(StaticFilterIndex { + ctx.filter_index = Some(Arc::new(Box::new(StaticFilterIndex { block_hash, filter: vec![0x00], header: Hash256::from_le_bytes(&[0x08; 32]), - })); + }))); ctx.set_chain_tip(TipSnapshot { tip_id: NodeId::new(0), height: 7, From 8df8b4dc4b1feb2a4f919c94909b827960ccc4d3 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 18:34:38 +0000 Subject: [PATCH 02/37] test(node): gate consensus-backend tests Node's default feature set does not enable bitcoinconsensus, so non-taproot script verification tests must not run in that configuration. Keep the coverage in the full-feature gate. Verification: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --lib --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --lib --no-default-features --features $FEATURES --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features $FEATURES -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast Op: correct Restores: spec: backend-dependent node tests only run when bitcoinconsensus is enabled --- crates/node/src/apply.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index 306e062..ee9612b 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -1094,6 +1094,7 @@ mod consensus_rule_tests { } #[test] + #[cfg(feature = "bitcoinconsensus")] fn verify_block_transactions_accepts_same_block_spend() -> Result<(), Box> { let base_prevout = bitcoin::OutPoint { @@ -1209,6 +1210,7 @@ mod consensus_rule_tests { } #[test] + #[cfg(feature = "bitcoinconsensus")] fn verify_block_transactions_defers_same_block_coinbase_spend_to_maturity() { let mut coinbase = coinbase_transaction(0x65); coinbase.output[0].script_pubkey = op_true_script(); @@ -1860,6 +1862,7 @@ mod consensus_rule_tests { } #[test] + #[cfg(feature = "bitcoinconsensus")] fn apply_block_persists_non_empty_filter_for_valid_same_block_spend() -> Result<(), Box> { let genesis = bitcoin::blockdata::constants::genesis_block(bitcoin::Network::Regtest); From 48bec2a92b641139b71edecb6a763117dde0ca86 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 18:52:51 +0000 Subject: [PATCH 03/37] test(node): add bounded apply profile harness Add an ignored synthetic replay that exercises the real BlockSync/apply_block path with index hooks disabled and noop hook branches. This gives a bounded local source of apply_block profile rows without claiming mainnet or storage-index performance. Verification: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - RUST_LOG=bitcoin_rs_node::apply=info cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --test sync_smoke --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus bounded_apply_profile_replay -- --ignored --nocapture --test-threads=1 - RUST_LOG=bitcoin_rs_node::apply=info /home/alpha/dev/bitcoin-rs/bitcoin-rs/target/release/deps/sync_smoke-03e9c594e970bfab bounded_apply_profile_replay --ignored --nocapture --test-threads=1 - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --test sync_smoke --no-default-features --features $FEATURES --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features $FEATURES -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast Op: extend --- crates/node/tests/sync_smoke.rs | 152 +++++++++++++++++++++++++------- 1 file changed, 122 insertions(+), 30 deletions(-) diff --git a/crates/node/tests/sync_smoke.rs b/crates/node/tests/sync_smoke.rs index 285b4ad..70a73f2 100644 --- a/crates/node/tests/sync_smoke.rs +++ b/crates/node/tests/sync_smoke.rs @@ -1,6 +1,7 @@ //! Block sync smoke tests. use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; +use std::time::{Duration, Instant}; use arc_swap::ArcSwapOption; use bitcoin::hashes::Hash as _; @@ -261,7 +262,90 @@ fn tick_buffers_out_of_order_blocks_until_parent_arrives() -> Result<(), Box Result<(), Box> { let fixture = non_coinbase_spend_chain()?; + let outcome = replay_non_coinbase_spend_chain(&fixture, true)?; + assert_eq!(outcome.applied_height, 102); + assert_eq!( + outcome.applied_hash, + bitcoin_rs_primitives::Hash256::from_le_bytes( + fixture + .blocks + .last() + .ok_or_else(|| std::io::Error::other("missing final block"))? + .block_hash() + .as_byte_array(), + ) + ); + assert!( + outcome + .utxo + .get(&primitive_outpoint(fixture.mature_coinbase_outpoint)) + .is_none(), + "mature coinbase prevout must be removed by the height-101 spend", + ); + assert!( + outcome + .utxo + .get(&primitive_outpoint(fixture.funding_outpoint)) + .is_none(), + "funding prevout must be removed by the height-102 spend", + ); + assert!( + outcome + .utxo + .get(&primitive_outpoint(fixture.spend_outpoint)) + .is_some(), + "height-102 spend output must remain live", + ); + + let block_refs: Vec<&bitcoin::Block> = fixture.blocks.iter().collect(); + assert_eq!( + outcome.coin_stats.snapshot(), + expected_coin_stats(&block_refs)? + ); + Ok(()) +} + +#[test] +#[ignore = "bounded local profiling harness; run explicitly with RUST_LOG=bitcoin_rs_node::apply=info"] +fn bounded_apply_profile_replay() -> Result<(), Box> { + let _subscriber_already_set = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .is_err(); + let fixture = non_coinbase_spend_chain()?; + + // This profiles apply path branch overhead, not storage-backed index cost. + for use_noop_index_hooks in [false, true] { + let label = if use_noop_index_hooks { + "index_hooks=noop" + } else { + "index_hooks=disabled" + }; + let outcome = replay_non_coinbase_spend_chain(&fixture, use_noop_index_hooks)?; + println!( + "bounded_apply_profile_replay {label} elapsed_ms={} applied_height={} blocks={}", + elapsed_ms(outcome.elapsed), + outcome.applied_height, + fixture.blocks.len(), + ); + } + + Ok(()) +} + +struct ReplayOutcome { + elapsed: Duration, + applied_height: u32, + applied_hash: Hash256, + coin_stats: Arc, + utxo: Arc, +} + +fn replay_non_coinbase_spend_chain( + fixture: &SpendChainFixture, + use_noop_index_hooks: bool, +) -> Result> { let block_tree = Arc::new(RwLock::new(BlockTree::new())); let chain_tip = block_tree.read().tip_handle(); let applied_tip: Arc> = Arc::new(ArcSwapOption::empty()); @@ -271,12 +355,16 @@ fn tick_applies_non_coinbase_spend_and_updates_utxo_and_coinstats() let inbound_headers_rx = Arc::new(Mutex::new(inbound_headers_rx_raw)); let (inbound_blocks_tx, inbound_blocks_rx_raw) = unbounded::(); let inbound_blocks_rx = Arc::new(Mutex::new(inbound_blocks_rx_raw)); - let (handles, coin_stats, utxo) = apply_handles_with_coin_stats_and_utxo( + let (mut handles, coin_stats, utxo) = apply_handles_with_coin_stats_and_utxo( Network::Regtest, Arc::clone(&chain_tip), Arc::clone(&applied_tip), Arc::clone(&block_tree), ); + if !use_noop_index_hooks { + handles.tx_index = None; + handles.filter_index = None; + } let sync = BlockSync::new( handles, Arc::clone(&peers), @@ -290,42 +378,46 @@ fn tick_applies_non_coinbase_spend_and_updates_utxo_and_coinstats() inbound_blocks_tx.send(block.clone())?; } + let started = Instant::now(); sync.tick(); + let elapsed = started.elapsed(); let applied = applied_tip .load_full() .ok_or_else(|| std::io::Error::other("missing applied tip"))?; - assert_eq!(applied.height, 102); - assert_eq!( - applied.hash, - bitcoin_rs_primitives::Hash256::from_le_bytes( - fixture - .blocks - .last() - .ok_or_else(|| std::io::Error::other("missing final block"))? - .block_hash() - .as_byte_array(), - ) - ); - assert!( - utxo.get(&primitive_outpoint(fixture.mature_coinbase_outpoint)) - .is_none(), - "mature coinbase prevout must be removed by the height-101 spend", - ); - assert!( - utxo.get(&primitive_outpoint(fixture.funding_outpoint)) - .is_none(), - "funding prevout must be removed by the height-102 spend", - ); - assert!( - utxo.get(&primitive_outpoint(fixture.spend_outpoint)) - .is_some(), - "height-102 spend output must remain live", + let expected_height = u32::try_from( + fixture + .blocks + .len() + .checked_sub(1) + .ok_or_else(|| std::io::Error::other("empty replay fixture"))?, + )?; + let expected_hash = bitcoin_rs_primitives::Hash256::from_le_bytes( + fixture + .blocks + .last() + .ok_or_else(|| std::io::Error::other("missing final block"))? + .block_hash() + .as_byte_array(), ); + if applied.height != expected_height || applied.hash != expected_hash { + return Err(std::io::Error::other(format!( + "replay stopped before final block: applied height/hash {}/{:?}, expected {}/{:?}", + applied.height, applied.hash, expected_height, expected_hash, + )) + .into()); + } + Ok(ReplayOutcome { + elapsed, + applied_height: applied.height, + applied_hash: applied.hash, + coin_stats, + utxo, + }) +} - let block_refs: Vec<&bitcoin::Block> = fixture.blocks.iter().collect(); - assert_eq!(coin_stats.snapshot(), expected_coin_stats(&block_refs)?); - Ok(()) +fn elapsed_ms(duration: Duration) -> u128 { + duration.as_millis() } struct SpendChainFixture { From 6f53be418f93ee0133d7f7daf0f5e1859e33b81d Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 19:04:26 +0000 Subject: [PATCH 04/37] test(node): measure optional index storage cost Add an ignored Redb-backed direct index harness for bounded local measurement of txindex ingest and filter-index persistence cost over the synthetic sync-smoke chain. Verification: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --no-default-features --features redb,bitcoinconsensus --test sync_smoke optional_index_redb_direct_cost -- --ignored --nocapture --test-threads=1 - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --test sync_smoke --no-default-features --features $FEATURES --no-fail-fast - FEATURES=rocksdb,fjall,redb,mdbx,bitcoinconsensus cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features $FEATURES -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast Op: extend --- crates/node/tests/sync_smoke.rs | 80 +++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/node/tests/sync_smoke.rs b/crates/node/tests/sync_smoke.rs index 70a73f2..b5d4e20 100644 --- a/crates/node/tests/sync_smoke.rs +++ b/crates/node/tests/sync_smoke.rs @@ -334,6 +334,77 @@ fn bounded_apply_profile_replay() -> Result<(), Box> { Ok(()) } +#[cfg(feature = "redb")] +#[test] +#[ignore = "bounded local storage-backed optional-index cost harness; run explicitly"] +fn optional_index_redb_direct_cost() -> Result<(), Box> { + let temp = tempfile::tempdir()?; + let tx_store = bitcoin_rs_storage::RedbStore::open(temp.path().join("txindex"))?; + let filter_store = bitcoin_rs_storage::RedbStore::open(temp.path().join("filters"))?; + let mut tx_index = bitcoin_rs_index::Indexer::new(Arc::new(tx_store)); + let filter_index = bitcoin_rs_filters::FilterIndex::new(filter_store); + let fixture = non_coinbase_spend_chain()?; + + let mut txids = 0_usize; + let mut funding = 0_usize; + let mut spending = 0_usize; + let mut filter_bytes_len = 0_usize; + let mut txindex_us = 0_u128; + let mut filterindex_us = 0_u128; + let mut prev_header = Hash256::default(); + let mut final_block_hash = None; + + for (height, block) in fixture.blocks.iter().enumerate() { + let height = u32::try_from(height)?; + let block_bytes = bitcoin::consensus::serialize(block); + + let started = Instant::now(); + let counts = tx_index.ingest_block(&block_bytes, height)?; + txindex_us = txindex_us.saturating_add(started.elapsed().as_micros()); + txids = txids.saturating_add(counts.txids); + funding = funding.saturating_add(counts.funding); + spending = spending.saturating_add(counts.spending); + + let filter_bytes = deterministic_filter_bytes(block); + filter_bytes_len = filter_bytes_len.saturating_add(filter_bytes.len()); + let block_hash = Hash256::from_le_bytes(block.block_hash().as_byte_array()); + let started = Instant::now(); + prev_header = filter_index.put_filter(block_hash, prev_header, &filter_bytes)?; + filterindex_us = filterindex_us.saturating_add(started.elapsed().as_micros()); + final_block_hash = Some(block_hash); + } + + if txids == 0 || funding == 0 || spending == 0 || filter_bytes_len == 0 { + return Err(std::io::Error::other(format!( + "optional index direct cost no-op: txids={txids} funding={funding} spending={spending} filter_bytes={filter_bytes_len}", + )) + .into()); + } + let final_block_hash = final_block_hash + .ok_or_else(|| std::io::Error::other("missing final block hash after direct indexing"))?; + let final_header = filter_index + .filter_header(final_block_hash)? + .ok_or_else(|| { + std::io::Error::other("missing final filter header after direct indexing") + })?; + assert_eq!(final_header, prev_header); + + let total_us = txindex_us.saturating_add(filterindex_us); + println!( + "optional_index_redb_direct_cost blocks={} txids={} funding={} spending={} filter_bytes={} txindex_us={} filterindex_us={} total_us={}", + fixture.blocks.len(), + txids, + funding, + spending, + filter_bytes_len, + txindex_us, + filterindex_us, + total_us, + ); + + Ok(()) +} + struct ReplayOutcome { elapsed: Duration, applied_height: u32, @@ -420,6 +491,15 @@ fn elapsed_ms(duration: Duration) -> u128 { duration.as_millis() } +#[cfg(feature = "redb")] +fn deterministic_filter_bytes(block: &bitcoin::Block) -> Vec { + let mut filter_bytes = Vec::with_capacity(block.txdata.len().saturating_mul(32)); + for tx in &block.txdata { + filter_bytes.extend_from_slice(tx.compute_txid().as_byte_array()); + } + filter_bytes +} + struct SpendChainFixture { blocks: Vec, mature_coinbase_outpoint: BitcoinOutPoint, From 82ef0dabd09c8bb21590eac5468e4ca1dad7d516 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 19:34:04 +0000 Subject: [PATCH 05/37] perf(coinstats): add listener commit benchmark Add a Criterion harness in the coinstats crate so the existing coinstats -> utxo dependency owns the measurement without introducing a utxo -> coinstats cycle. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --benches --no-default-features -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --no-fail-fast - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats -- --sample-size 10 Synthetic benchmark results: - no_listener: [3.7867 ms 3.9379 ms 4.1257 ms] - coinstats_listener: [659.61 ms 664.15 ms 668.28 ms] Caveat: this is synthetic UtxoSet::commit_block evidence, not end-to-end node runtime evidence. Op: extend --- Cargo.lock | 1 + crates/coinstats/Cargo.toml | 5 + .../benches/utxo_commit_coinstats.rs | 112 ++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 crates/coinstats/benches/utxo_commit_coinstats.rs diff --git a/Cargo.lock b/Cargo.lock index 13213fc..39589c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,6 +287,7 @@ dependencies = [ "bitcoin-rs-storage", "bitcoin-rs-utxo", "bytemuck", + "criterion", "parking_lot", "proptest", "ruint", diff --git a/crates/coinstats/Cargo.toml b/crates/coinstats/Cargo.toml index e858979..0c57d20 100644 --- a/crates/coinstats/Cargo.toml +++ b/crates/coinstats/Cargo.toml @@ -32,6 +32,11 @@ redb = ["bitcoin-rs-storage/redb"] mdbx = ["bitcoin-rs-storage/mdbx"] [dev-dependencies] +criterion.workspace = true proptest.workspace = true serde_json.workspace = true tempfile = "3" + +[[bench]] +name = "utxo_commit_coinstats" +harness = false diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs new file mode 100644 index 0000000..d55cf1e --- /dev/null +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -0,0 +1,112 @@ +//! Synthetic UTXO commit benchmark with and without the coinstats listener. +// PERF: Criterion emits public harness items whose docs are irrelevant to the benchmark report. +#![allow(missing_docs)] + +use std::hint::black_box; + +use bitcoin::{Amount, ScriptBuf}; +use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener}; +use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; +use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoSet}; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; + +const PRELOAD_HEIGHT: u32 = 1; +const ADD_HEIGHT: u32 = 2; +const OP_COUNT: u64 = 10_000; +const CASE_SEED: u64 = 0x00ab_cdef; +const COMMIT_BLOCK_SEED: u64 = 0x0012_3456; + +const fn next_u64(state: &mut u64) -> u64 { + *state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + *state +} + +fn txid(seed: u64) -> Hash256 { + let mut bytes = [0_u8; 32]; + bytes[..8].copy_from_slice(&seed.to_le_bytes()); + bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); + bytes[16..24].copy_from_slice(&seed.wrapping_mul(0x9e37_79b9_7f4a_7c15).to_le_bytes()); + bytes[24..32].copy_from_slice(&seed.wrapping_add(0xd1b5_4a32_d192_ed03).to_le_bytes()); + Hash256::from_le_bytes(&bytes) +} + +fn txout(seed: u64) -> TxOut { + let mut script = Vec::with_capacity(34); + script.extend_from_slice(&[0x00, 0x20]); + script.extend_from_slice(&txid(seed).to_le_bytes()); + TxOut { + value: Amount::from_sat(5_000 + seed), + script_pubkey: ScriptBuf::from_bytes(script), + } +} + +fn synthetic_case(seed: u64, with_listener: bool) -> (UtxoSet, BlockChanges) { + let mut set = UtxoSet::new(); + if with_listener { + set.set_listener(Box::new(CoinStatsListener::new(CoinStats::new()))); + } + + let mut preload = BlockChanges::default(); + let mut changes = BlockChanges::default(); + let mut rng = seed; + + for _ in 0_u64..OP_COUNT { + let spend_seed = next_u64(&mut rng); + let outpoint = OutPoint::new(txid(spend_seed), 0); + preload.add(UtxoAdd::new( + outpoint, + txout(spend_seed), + false, + PRELOAD_HEIGHT, + )); + changes.remove(outpoint); + } + + if let Err(error) = set.commit_block(&preload, &txid(seed)) { + panic!("synthetic preload failed: {error}"); + } + + for i in 0_u64..OP_COUNT { + let add_seed = next_u64(&mut rng).wrapping_add(i); + let outpoint = OutPoint::new(txid(add_seed), 0); + changes.add(UtxoAdd::new(outpoint, txout(add_seed), false, ADD_HEIGHT)); + } + + (set, changes) +} + +fn utxo_commit_coinstats(c: &mut Criterion) { + let mut group = c.benchmark_group("utxo_commit_coinstats"); + let block_hash = txid(COMMIT_BLOCK_SEED); + + group.bench_function("no_listener", |b| { + b.iter_batched( + || synthetic_case(CASE_SEED, false), + |(set, changes)| { + if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { + panic!("synthetic commit failed: {error}"); + } + }, + BatchSize::SmallInput, + ); + }); + + group.bench_function("coinstats_listener", |b| { + b.iter_batched( + || synthetic_case(CASE_SEED, true), + |(set, changes)| { + if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { + panic!("synthetic commit failed: {error}"); + } + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +criterion_group!(benches, utxo_commit_coinstats); +criterion_main!(benches); From 5a36a0701121315cc73adba8bd8db3eea888525e Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 19:44:57 +0000 Subject: [PATCH 06/37] perf(coinstats): separate listener hook overhead Add a no-op UTXO listener case to the coinstats commit benchmark. This falsifies the cheap explanation that the synthetic delta is merely listener dispatch or shard hook overhead. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --benches --no-default-features -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --no-fail-fast - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats -- --sample-size 10 Synthetic benchmark results: - no_listener: [3.8390 ms 3.9874 ms 4.0988 ms] - noop_listener: [4.5625 ms 4.6284 ms 4.7106 ms] - coinstats_listener: [648.83 ms 654.98 ms 660.95 ms] Caveat: this separates synthetic hook overhead from CoinStats/MuHash work; it is not an end-to-end node runtime claim. Op: extend --- .../benches/utxo_commit_coinstats.rs | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs index d55cf1e..b2d4aa8 100644 --- a/crates/coinstats/benches/utxo_commit_coinstats.rs +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -7,7 +7,7 @@ use std::hint::black_box; use bitcoin::{Amount, ScriptBuf}; use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener}; use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; -use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoSet}; +use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoChangeListener, UtxoSet}; use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; const PRELOAD_HEIGHT: u32 = 1; @@ -16,6 +16,21 @@ const OP_COUNT: u64 = 10_000; const CASE_SEED: u64 = 0x00ab_cdef; const COMMIT_BLOCK_SEED: u64 = 0x0012_3456; +#[derive(Clone, Copy)] +enum ListenerKind { + None, + Noop, + CoinStats, +} + +struct NoopListener; + +impl UtxoChangeListener for NoopListener { + fn on_insert(&self, _op: &OutPoint, _txout: &TxOut, _height: u32, _coinbase: bool) {} + + fn on_remove(&self, _op: &OutPoint, _txout: &TxOut, _height: u32) {} +} + const fn next_u64(state: &mut u64) -> u64 { *state = state .wrapping_mul(6_364_136_223_846_793_005) @@ -42,10 +57,14 @@ fn txout(seed: u64) -> TxOut { } } -fn synthetic_case(seed: u64, with_listener: bool) -> (UtxoSet, BlockChanges) { +fn synthetic_case(seed: u64, listener_kind: ListenerKind) -> (UtxoSet, BlockChanges) { let mut set = UtxoSet::new(); - if with_listener { - set.set_listener(Box::new(CoinStatsListener::new(CoinStats::new()))); + match listener_kind { + ListenerKind::None => {} + ListenerKind::Noop => set.set_listener(Box::new(NoopListener)), + ListenerKind::CoinStats => { + set.set_listener(Box::new(CoinStatsListener::new(CoinStats::new()))); + } } let mut preload = BlockChanges::default(); @@ -83,7 +102,19 @@ fn utxo_commit_coinstats(c: &mut Criterion) { group.bench_function("no_listener", |b| { b.iter_batched( - || synthetic_case(CASE_SEED, false), + || synthetic_case(CASE_SEED, ListenerKind::None), + |(set, changes)| { + if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { + panic!("synthetic commit failed: {error}"); + } + }, + BatchSize::SmallInput, + ); + }); + + group.bench_function("noop_listener", |b| { + b.iter_batched( + || synthetic_case(CASE_SEED, ListenerKind::Noop), |(set, changes)| { if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { panic!("synthetic commit failed: {error}"); @@ -95,7 +126,7 @@ fn utxo_commit_coinstats(c: &mut Criterion) { group.bench_function("coinstats_listener", |b| { b.iter_batched( - || synthetic_case(CASE_SEED, true), + || synthetic_case(CASE_SEED, ListenerKind::CoinStats), |(set, changes)| { if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { panic!("synthetic commit failed: {error}"); From efa0ba31b1609855044c689f1a62d354017b3671 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 20:05:11 +0000 Subject: [PATCH 07/37] perf(node): add coinstats replay cost harness Add an ignored sync_smoke harness that replays the same non-coinbase spend fixture with the CoinStats listener attached to UTXO mutations and detached from UTXO mutations. The detached mode still wires CoinStats into ApplyHandles; it only skips the UTXO listener hook for measurement. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --features bitcoinconsensus,redb --test sync_smoke -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --features bitcoinconsensus,redb --test sync_smoke --no-fail-fast - RUST_LOG=bitcoin_rs_node::sync=debug,bitcoin_rs_node::apply=info cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --features bitcoinconsensus,redb --test sync_smoke bounded_apply_profile_replay_coinstats_listener_cost -- --ignored --nocapture Observed direct harness output in a feature-compatible test binary: - attached: elapsed_ms=31 applied_height=102 blocks=103 - detached: elapsed_ms=14 applied_height=102 blocks=103 - height 101/102 utxo_commit_us attached: 133/159 - height 101/102 utxo_commit_us detached: 65/61 Caveat: detached mode is a measurement control, not a production configuration or proof of end-to-end mainnet speed. Op: extend --- crates/node/tests/sync_smoke.rs | 73 +++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/crates/node/tests/sync_smoke.rs b/crates/node/tests/sync_smoke.rs index b5d4e20..e01e4eb 100644 --- a/crates/node/tests/sync_smoke.rs +++ b/crates/node/tests/sync_smoke.rs @@ -334,6 +334,37 @@ fn bounded_apply_profile_replay() -> Result<(), Box> { Ok(()) } +#[test] +#[ignore = "bounded local coinstats listener cost harness; run explicitly with RUST_LOG=bitcoin_rs_node::apply=info"] +fn bounded_apply_profile_replay_coinstats_listener_cost() -> Result<(), Box> +{ + let _subscriber_already_set = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .is_err(); + let fixture = non_coinbase_spend_chain()?; + + for coin_stats_listener in [ + CoinStatsListenerMode::Attached, + CoinStatsListenerMode::Detached, + ] { + let outcome = replay_non_coinbase_spend_chain_with_coin_stats_listener( + &fixture, + false, + coin_stats_listener, + )?; + println!( + "bounded_apply_profile_replay_coinstats_listener_cost coin_stats_listener={} elapsed_ms={} applied_height={} blocks={}", + coin_stats_listener.label(), + elapsed_ms(outcome.elapsed), + outcome.applied_height, + fixture.blocks.len(), + ); + } + + Ok(()) +} + #[cfg(feature = "redb")] #[test] #[ignore = "bounded local storage-backed optional-index cost harness; run explicitly"] @@ -413,9 +444,36 @@ struct ReplayOutcome { utxo: Arc, } +#[derive(Clone, Copy)] +enum CoinStatsListenerMode { + Attached, + Detached, +} + +impl CoinStatsListenerMode { + const fn label(self) -> &'static str { + match self { + Self::Attached => "attached", + Self::Detached => "detached", + } + } +} + fn replay_non_coinbase_spend_chain( fixture: &SpendChainFixture, use_noop_index_hooks: bool, +) -> Result> { + replay_non_coinbase_spend_chain_with_coin_stats_listener( + fixture, + use_noop_index_hooks, + CoinStatsListenerMode::Attached, + ) +} + +fn replay_non_coinbase_spend_chain_with_coin_stats_listener( + fixture: &SpendChainFixture, + use_noop_index_hooks: bool, + coin_stats_listener: CoinStatsListenerMode, ) -> Result> { let block_tree = Arc::new(RwLock::new(BlockTree::new())); let chain_tip = block_tree.read().tip_handle(); @@ -431,6 +489,7 @@ fn replay_non_coinbase_spend_chain( Arc::clone(&chain_tip), Arc::clone(&applied_tip), Arc::clone(&block_tree), + coin_stats_listener, ); if !use_noop_index_hooks { handles.tx_index = None; @@ -618,8 +677,13 @@ fn apply_handles_with_coin_stats( applied_tip: Arc>, block_tree: Arc>, ) -> (ApplyHandles, Arc) { - let (handles, coin_stats, _utxo) = - apply_handles_with_coin_stats_and_utxo(network, chain_tip, applied_tip, block_tree); + let (handles, coin_stats, _utxo) = apply_handles_with_coin_stats_and_utxo( + network, + chain_tip, + applied_tip, + block_tree, + CoinStatsListenerMode::Attached, + ); (handles, coin_stats) } @@ -629,10 +693,13 @@ fn apply_handles_with_coin_stats_and_utxo( chain_tip: Arc>, applied_tip: Arc>, block_tree: Arc>, + coin_stats_listener: CoinStatsListenerMode, ) -> (ApplyHandles, Arc, Arc) { let coin_stats = Arc::new(CoinStatsListener::new(CoinStats::default())); let mut utxo = UtxoSet::new(); - utxo.set_listener(Box::new((*coin_stats).clone())); + if matches!(coin_stats_listener, CoinStatsListenerMode::Attached) { + utxo.set_listener(Box::new((*coin_stats).clone())); + } let utxo = Arc::new(utxo); let handles = ApplyHandles::new( network, From 248bffcde6c4b1fc5faac25d3f03c8718efc4983 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 20:35:53 +0000 Subject: [PATCH 08/37] perf(coinstats): add accounting-only listener benchmark Add a benchmark-only listener variant that runs through the existing UTXO commit path while measuring lock plus simple stats accounting without coin encoding or MuHash updates. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --benches --no-run - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats Observed Criterion medians: - no_listener: 3.5933 ms - noop_listener: 4.2716 ms - accounting_listener: 31.108 ms - coinstats_listener: 635.70 ms Caveat: accounting_listener is a benchmark control, not production behavior or proof of mainnet speed. Op: extend --- .../benches/utxo_commit_coinstats.rs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs index b2d4aa8..fe99ea2 100644 --- a/crates/coinstats/benches/utxo_commit_coinstats.rs +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -9,6 +9,7 @@ use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener}; use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoChangeListener, UtxoSet}; use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use parking_lot::RwLock; const PRELOAD_HEIGHT: u32 = 1; const ADD_HEIGHT: u32 = 2; @@ -20,6 +21,7 @@ const COMMIT_BLOCK_SEED: u64 = 0x0012_3456; enum ListenerKind { None, Noop, + Accounting, CoinStats, } @@ -31,6 +33,50 @@ impl UtxoChangeListener for NoopListener { fn on_remove(&self, _op: &OutPoint, _txout: &TxOut, _height: u32) {} } +#[derive(Default)] +struct AccountingStats { + total_amount: u64, + bogo_size: u64, + utxo_count: u64, +} + +struct AccountingListener { + stats: RwLock, +} + +impl AccountingListener { + fn new() -> Self { + Self { + stats: RwLock::new(AccountingStats::default()), + } + } +} + +impl UtxoChangeListener for AccountingListener { + fn on_insert(&self, _op: &OutPoint, txout: &TxOut, _height: u32, _coinbase: bool) { + let mut stats = self.stats.write(); + stats.total_amount = stats.total_amount.saturating_add(txout.value.to_sat()); + stats.bogo_size = stats.bogo_size.saturating_add(simple_bogo_size(txout)); + stats.utxo_count = stats.utxo_count.saturating_add(1); + } + + fn on_remove(&self, _op: &OutPoint, txout: &TxOut, _height: u32) { + let mut stats = self.stats.write(); + stats.total_amount = stats.total_amount.saturating_sub(txout.value.to_sat()); + stats.bogo_size = stats.bogo_size.saturating_sub(simple_bogo_size(txout)); + stats.utxo_count = stats.utxo_count.saturating_sub(1); + } +} + +fn simple_bogo_size(txout: &TxOut) -> u64 { + let script_len = u64::try_from(txout.script_pubkey.len()).unwrap_or(u64::MAX); + 36_u64 + .saturating_add(4) + .saturating_add(8) + .saturating_add(2) + .saturating_add(script_len) +} + const fn next_u64(state: &mut u64) -> u64 { *state = state .wrapping_mul(6_364_136_223_846_793_005) @@ -62,6 +108,7 @@ fn synthetic_case(seed: u64, listener_kind: ListenerKind) -> (UtxoSet, BlockChan match listener_kind { ListenerKind::None => {} ListenerKind::Noop => set.set_listener(Box::new(NoopListener)), + ListenerKind::Accounting => set.set_listener(Box::new(AccountingListener::new())), ListenerKind::CoinStats => { set.set_listener(Box::new(CoinStatsListener::new(CoinStats::new()))); } @@ -124,6 +171,18 @@ fn utxo_commit_coinstats(c: &mut Criterion) { ); }); + group.bench_function("accounting_listener", |b| { + b.iter_batched( + || synthetic_case(CASE_SEED, ListenerKind::Accounting), + |(set, changes)| { + if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { + panic!("synthetic commit failed: {error}"); + } + }, + BatchSize::SmallInput, + ); + }); + group.bench_function("coinstats_listener", |b| { b.iter_batched( || synthetic_case(CASE_SEED, ListenerKind::CoinStats), From cf24d8699fc52905557fa67aba560a3fabd80a22 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Mon, 25 May 2026 20:58:48 +0000 Subject: [PATCH 09/37] perf(coinstats): add direct mutation benchmark Add a benchmark-only direct CoinStats insert/remove case beside the UTXO commit-path listener cases. The direct case starts from the same preloaded stats state and measures the mutation loop without UTXO shard/listener dispatch. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --benches --no-run - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats Observed Criterion medians: - no_listener: 3.5302 ms - noop_listener: 4.1248 ms - accounting_listener: 52.541 ms - coinstats_listener: 648.01 ms - direct_coinstats_insert_remove: 145.65 ms Caveat: direct_coinstats_insert_remove is diagnostic and excludes UTXO shard/listener dispatch; it is not production behavior or proof of mainnet speed. Op: extend --- .../benches/utxo_commit_coinstats.rs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs index fe99ea2..e7c6bc8 100644 --- a/crates/coinstats/benches/utxo_commit_coinstats.rs +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -44,6 +44,12 @@ struct AccountingListener { stats: RwLock, } +struct DirectCase { + stats: CoinStats, + spends: Vec, + adds: Vec, +} + impl AccountingListener { fn new() -> Self { Self { @@ -143,6 +149,33 @@ fn synthetic_case(seed: u64, listener_kind: ListenerKind) -> (UtxoSet, BlockChan (set, changes) } +fn synthetic_direct_case(seed: u64) -> DirectCase { + let mut stats = CoinStats::new(); + let mut spends = Vec::with_capacity(usize::try_from(OP_COUNT).unwrap_or(usize::MAX)); + let mut adds = Vec::with_capacity(usize::try_from(OP_COUNT).unwrap_or(usize::MAX)); + let mut rng = seed; + + for _ in 0_u64..OP_COUNT { + let spend_seed = next_u64(&mut rng); + let outpoint = OutPoint::new(txid(spend_seed), 0); + let spend = UtxoAdd::new(outpoint, txout(spend_seed), false, PRELOAD_HEIGHT); + stats.insert_utxo(&spend.outpoint, &spend.txout, spend.height, spend.coinbase); + spends.push(spend); + } + + for i in 0_u64..OP_COUNT { + let add_seed = next_u64(&mut rng).wrapping_add(i); + let outpoint = OutPoint::new(txid(add_seed), 0); + adds.push(UtxoAdd::new(outpoint, txout(add_seed), false, ADD_HEIGHT)); + } + + DirectCase { + stats, + spends, + adds, + } +} + fn utxo_commit_coinstats(c: &mut Criterion) { let mut group = c.benchmark_group("utxo_commit_coinstats"); let block_hash = txid(COMMIT_BLOCK_SEED); @@ -195,6 +228,27 @@ fn utxo_commit_coinstats(c: &mut Criterion) { ); }); + group.bench_function("direct_coinstats_insert_remove", |b| { + b.iter_batched( + || synthetic_direct_case(CASE_SEED), + |case| { + let DirectCase { + mut stats, + spends, + adds, + } = case; + for spend in &spends { + stats.remove_utxo(&spend.outpoint, &spend.txout, spend.height, spend.coinbase); + } + for add in &adds { + stats.insert_utxo(&add.outpoint, &add.txout, add.height, add.coinbase); + } + black_box(stats); + }, + BatchSize::SmallInput, + ); + }); + group.finish(); } From 2a70537d8b9c9a8d260e99b1db61efa1aa2becef Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 00:25:42 +0000 Subject: [PATCH 10/37] perf(coinstats): split direct mutation benchmark Add benchmark-only diagnostic splits for CoinStats direct mutation attribution. The new cases mirror the private coin preimage encoder locally and measure encoding/allocation separately from MuHash3072 insert/remove over pre-encoded bytes, while keeping the combined direct mutation baseline and commit-path listener cases. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --benches --no-run - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats Observed Criterion medians: - no_listener: 3.7033 ms - noop_listener: 3.9494 ms - accounting_listener: 50.415 ms - coinstats_listener: 629.41 ms - direct_coinstats_insert_remove: 143.73 ms - direct_coinstats_encode_only: 1.7696 ms - direct_coinstats_muhash_preencoded: 145.54 ms Caveat: split direct benchmarks are diagnostic controls and mirror private encoding; they are not production behavior or proof of mainnet speed. Op: extend --- .../benches/utxo_commit_coinstats.rs | 164 +++++++++++++----- 1 file changed, 122 insertions(+), 42 deletions(-) diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs index e7c6bc8..8092d09 100644 --- a/crates/coinstats/benches/utxo_commit_coinstats.rs +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -4,12 +4,15 @@ use std::hint::black_box; -use bitcoin::{Amount, ScriptBuf}; -use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener}; +use bitcoin::{Amount, ScriptBuf, consensus::Encodable}; +use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener, MuHash3072}; use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoChangeListener, UtxoSet}; -use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use criterion::{ + BatchSize, BenchmarkGroup, Criterion, criterion_group, criterion_main, measurement::WallTime, +}; use parking_lot::RwLock; +use zerocopy::IntoBytes; const PRELOAD_HEIGHT: u32 = 1; const ADD_HEIGHT: u32 = 2; @@ -50,6 +53,11 @@ struct DirectCase { adds: Vec, } +struct PreEncodedCase { + spend_bytes: Vec>, + add_bytes: Vec>, +} + impl AccountingListener { fn new() -> Self { Self { @@ -83,6 +91,17 @@ fn simple_bogo_size(txout: &TxOut) -> u64 { .saturating_add(script_len) } +fn bench_coin_hash_bytes(op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) -> Vec { + let mut out = Vec::with_capacity(36 + 4 + txout.script_pubkey.len() + 16); + out.extend_from_slice(op.as_bytes()); + let coinbase_bit = u32::from(coinbase); + out.extend_from_slice(&((height << 1) | coinbase_bit).to_le_bytes()); + if txout.consensus_encode(&mut out).is_err() { + unreachable!("vec-backed consensus encoder is infallible"); + } + out +} + const fn next_u64(state: &mut u64) -> u64 { *state = state .wrapping_mul(6_364_136_223_846_793_005) @@ -176,78 +195,139 @@ fn synthetic_direct_case(seed: u64) -> DirectCase { } } -fn utxo_commit_coinstats(c: &mut Criterion) { - let mut group = c.benchmark_group("utxo_commit_coinstats"); - let block_hash = txid(COMMIT_BLOCK_SEED); +fn synthetic_preencoded_case(seed: u64) -> PreEncodedCase { + let direct = synthetic_direct_case(seed); + let spend_bytes = direct + .spends + .iter() + .map(|spend| { + bench_coin_hash_bytes(&spend.outpoint, &spend.txout, spend.height, spend.coinbase) + }) + .collect(); + let add_bytes = direct + .adds + .iter() + .map(|add| bench_coin_hash_bytes(&add.outpoint, &add.txout, add.height, add.coinbase)) + .collect(); + + PreEncodedCase { + spend_bytes, + add_bytes, + } +} - group.bench_function("no_listener", |b| { +fn bench_commit_case( + group: &mut BenchmarkGroup<'_, WallTime>, + name: &'static str, + listener_kind: ListenerKind, + block_hash: &Hash256, +) { + group.bench_function(name, |b| { b.iter_batched( - || synthetic_case(CASE_SEED, ListenerKind::None), + || synthetic_case(CASE_SEED, listener_kind), |(set, changes)| { - if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { + if let Err(error) = set.commit_block(black_box(&changes), black_box(block_hash)) { panic!("synthetic commit failed: {error}"); } }, BatchSize::SmallInput, ); }); +} - group.bench_function("noop_listener", |b| { +fn bench_direct_coinstats(group: &mut BenchmarkGroup<'_, WallTime>) { + group.bench_function("direct_coinstats_insert_remove", |b| { b.iter_batched( - || synthetic_case(CASE_SEED, ListenerKind::Noop), - |(set, changes)| { - if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { - panic!("synthetic commit failed: {error}"); + || synthetic_direct_case(CASE_SEED), + |case| { + let DirectCase { + mut stats, + spends, + adds, + } = case; + for spend in &spends { + stats.remove_utxo(&spend.outpoint, &spend.txout, spend.height, spend.coinbase); } - }, - BatchSize::SmallInput, - ); - }); - - group.bench_function("accounting_listener", |b| { - b.iter_batched( - || synthetic_case(CASE_SEED, ListenerKind::Accounting), - |(set, changes)| { - if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { - panic!("synthetic commit failed: {error}"); + for add in &adds { + stats.insert_utxo(&add.outpoint, &add.txout, add.height, add.coinbase); } + black_box(stats); }, BatchSize::SmallInput, ); }); +} - group.bench_function("coinstats_listener", |b| { +fn bench_direct_encode_only(group: &mut BenchmarkGroup<'_, WallTime>) { + group.bench_function("direct_coinstats_encode_only", |b| { b.iter_batched( - || synthetic_case(CASE_SEED, ListenerKind::CoinStats), - |(set, changes)| { - if let Err(error) = set.commit_block(black_box(&changes), black_box(&block_hash)) { - panic!("synthetic commit failed: {error}"); + || synthetic_direct_case(CASE_SEED), + |case| { + let DirectCase { spends, adds, .. } = case; + let mut encoded = Vec::with_capacity(spends.len().saturating_add(adds.len())); + for spend in &spends { + encoded.push(bench_coin_hash_bytes( + &spend.outpoint, + &spend.txout, + spend.height, + spend.coinbase, + )); + } + for add in &adds { + encoded.push(bench_coin_hash_bytes( + &add.outpoint, + &add.txout, + add.height, + add.coinbase, + )); } + black_box(encoded); }, BatchSize::SmallInput, ); }); +} - group.bench_function("direct_coinstats_insert_remove", |b| { +fn bench_direct_muhash_preencoded(group: &mut BenchmarkGroup<'_, WallTime>) { + group.bench_function("direct_coinstats_muhash_preencoded", |b| { b.iter_batched( - || synthetic_direct_case(CASE_SEED), + || synthetic_preencoded_case(CASE_SEED), |case| { - let DirectCase { - mut stats, - spends, - adds, - } = case; - for spend in &spends { - stats.remove_utxo(&spend.outpoint, &spend.txout, spend.height, spend.coinbase); + let mut muhash = MuHash3072::new(); + for bytes in &case.spend_bytes { + muhash.remove(bytes); } - for add in &adds { - stats.insert_utxo(&add.outpoint, &add.txout, add.height, add.coinbase); + for bytes in &case.add_bytes { + muhash.insert(bytes); } - black_box(stats); + black_box(muhash); }, BatchSize::SmallInput, ); }); +} + +fn utxo_commit_coinstats(c: &mut Criterion) { + let mut group = c.benchmark_group("utxo_commit_coinstats"); + let block_hash = txid(COMMIT_BLOCK_SEED); + + bench_commit_case(&mut group, "no_listener", ListenerKind::None, &block_hash); + bench_commit_case(&mut group, "noop_listener", ListenerKind::Noop, &block_hash); + bench_commit_case( + &mut group, + "accounting_listener", + ListenerKind::Accounting, + &block_hash, + ); + bench_commit_case( + &mut group, + "coinstats_listener", + ListenerKind::CoinStats, + &block_hash, + ); + bench_direct_coinstats(&mut group); + bench_direct_encode_only(&mut group); + bench_direct_muhash_preencoded(&mut group); group.finish(); } From 27a28d90076fca3089d8386672507992af888070 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 00:38:48 +0000 Subject: [PATCH 11/37] perf(coinstats): benchmark sharded listener control Add a benchmark-only sharded CoinStats listener control to test whether the production listener's single write lock serializes UTXO shard workers. The sharded control routes callbacks by the same UtxoKey shard mapping as the UTXO set and keeps the existing production listener unchanged. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --benches --no-run - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats Observed Criterion medians: - no_listener: 3.7830 ms - noop_listener: 3.8697 ms - accounting_listener: 50.068 ms - sharded_coinstats_listener: 14.963 ms - coinstats_listener: 650.82 ms - direct_coinstats_insert_remove: 149.71 ms - direct_coinstats_encode_only: 1.9434 ms - direct_coinstats_muhash_preencoded: 147.11 ms Caveat: sharded_coinstats_listener is a benchmark control that does not combine shard snapshots; it is not production behavior or proof of mainnet speed. Op: extend --- .../benches/utxo_commit_coinstats.rs | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs index 8092d09..1f772ff 100644 --- a/crates/coinstats/benches/utxo_commit_coinstats.rs +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -7,7 +7,7 @@ use std::hint::black_box; use bitcoin::{Amount, ScriptBuf, consensus::Encodable}; use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener, MuHash3072}; use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; -use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoChangeListener, UtxoSet}; +use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoChangeListener, UtxoKey, UtxoSet}; use criterion::{ BatchSize, BenchmarkGroup, Criterion, criterion_group, criterion_main, measurement::WallTime, }; @@ -25,6 +25,7 @@ enum ListenerKind { None, Noop, Accounting, + ShardedCoinStats, CoinStats, } @@ -47,6 +48,10 @@ struct AccountingListener { stats: RwLock, } +struct ShardedCoinStatsListener { + shards: Vec>, +} + struct DirectCase { stats: CoinStats, spends: Vec, @@ -66,6 +71,20 @@ impl AccountingListener { } } +impl ShardedCoinStatsListener { + fn new() -> Self { + let shards = (0..UtxoKey::SHARD_COUNT) + .map(|_| RwLock::new(CoinStats::new())) + .collect(); + Self { shards } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.shards[index] + } +} + impl UtxoChangeListener for AccountingListener { fn on_insert(&self, _op: &OutPoint, txout: &TxOut, _height: u32, _coinbase: bool) { let mut stats = self.stats.write(); @@ -82,6 +101,24 @@ impl UtxoChangeListener for AccountingListener { } } +impl UtxoChangeListener for ShardedCoinStatsListener { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.shard(op) + .write() + .insert_utxo(op, txout, height, coinbase); + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { + self.shard(op).write().remove_utxo(op, txout, height, false); + } + + fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.shard(op) + .write() + .remove_utxo(op, txout, height, coinbase); + } +} + fn simple_bogo_size(txout: &TxOut) -> u64 { let script_len = u64::try_from(txout.script_pubkey.len()).unwrap_or(u64::MAX); 36_u64 @@ -134,6 +171,9 @@ fn synthetic_case(seed: u64, listener_kind: ListenerKind) -> (UtxoSet, BlockChan ListenerKind::None => {} ListenerKind::Noop => set.set_listener(Box::new(NoopListener)), ListenerKind::Accounting => set.set_listener(Box::new(AccountingListener::new())), + ListenerKind::ShardedCoinStats => { + set.set_listener(Box::new(ShardedCoinStatsListener::new())); + } ListenerKind::CoinStats => { set.set_listener(Box::new(CoinStatsListener::new(CoinStats::new()))); } @@ -319,6 +359,12 @@ fn utxo_commit_coinstats(c: &mut Criterion) { ListenerKind::Accounting, &block_hash, ); + bench_commit_case( + &mut group, + "sharded_coinstats_listener", + ListenerKind::ShardedCoinStats, + &block_hash, + ); bench_commit_case( &mut group, "coinstats_listener", From 1d179a58274d93e87043dfeb71e733a7e0f09a01 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 02:46:57 +0000 Subject: [PATCH 12/37] test(coinstats): characterize listener snapshot visibility Document the current CoinStatsListener visibility boundary while UtxoSet::commit_block is in flight. The test pauses after the first forwarded listener insert, observes listener-only state, then checks post-commit and post-finish_block snapshots. Evidence: - cargo test -p bitcoin-rs-coinstats --test snapshot_with_muhash snapshot_can_observe_mid_block_listener_state_before_finish_block - cargo test -p bitcoin-rs-coinstats --test snapshot_with_muhash - cargo fmt --all -- --check - cargo clippy -p bitcoin-rs-coinstats --all-targets -- -D warnings Op: extend --- .../coinstats/tests/snapshot_with_muhash.rs | 123 +++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/crates/coinstats/tests/snapshot_with_muhash.rs b/crates/coinstats/tests/snapshot_with_muhash.rs index 93b02fe..8b648ca 100644 --- a/crates/coinstats/tests/snapshot_with_muhash.rs +++ b/crates/coinstats/tests/snapshot_with_muhash.rs @@ -1,8 +1,18 @@ //! Snapshot trailer integration tests for coinstats. +use std::{ + sync::{ + Arc, Barrier, + atomic::{AtomicBool, Ordering}, + }, + thread, +}; + use bitcoin::{Amount, ScriptBuf}; use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener}; use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; -use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoSet, write_snapshot}; +use bitcoin_rs_utxo::{ + BlockChanges, UtxoAdd, UtxoChangeListener, UtxoKey, UtxoSet, write_snapshot, +}; #[test] fn snapshot_trailer_uses_listener_muhash() -> Result<(), Box> { @@ -108,6 +118,117 @@ fn listener_tracks_duplicate_txid_overwrite() -> Result<(), Box Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + let gate = Arc::new(AtomicBool::new(false)); + let barrier = Arc::new(Barrier::new(2)); + set.set_listener(Box::new(FirstInsertGate { + inner: listener.clone(), + gate: Arc::clone(&gate), + barrier: Arc::clone(&barrier), + })); + + let first_txid = txid(1); + let second_txid = txid(257); + assert_eq!( + UtxoKey::from_txid(&first_txid).shard(), + UtxoKey::from_txid(&second_txid).shard() + ); + + let first_outpoint = OutPoint::new(first_txid, 0); + let second_outpoint = OutPoint::new(second_txid, 0); + let first_txout = txout(1); + let second_txout = txout(257); + let height = 7; + let tx_delta = 2; + + let mut changes = BlockChanges::default(); + changes.add(UtxoAdd::new( + first_outpoint, + first_txout.clone(), + false, + height, + )); + changes.add(UtxoAdd::new( + second_outpoint, + second_txout.clone(), + true, + height, + )); + + let block_hash = txid(1_000); + thread::scope(|scope| { + let commit = scope.spawn(|| set.commit_block(&changes, &block_hash)); + + barrier.wait(); + let mid_block = listener.snapshot(); + let mut expected_mid_block = CoinStats::new(); + expected_mid_block.insert_utxo(&first_outpoint, &first_txout, height, false); + assert_eq!(mid_block, expected_mid_block); + assert_eq!(mid_block.height, 0); + assert_eq!(mid_block.tx_count, 0); + + barrier.wait(); + match commit.join() { + Ok(result) => result, + Err(payload) => std::panic::resume_unwind(payload), + } + })?; + + let post_commit = listener.snapshot(); + let mut expected_post_commit = CoinStats::new(); + expected_post_commit.insert_utxo(&first_outpoint, &first_txout, height, false); + expected_post_commit.insert_utxo(&second_outpoint, &second_txout, height, true); + assert_eq!(post_commit, expected_post_commit); + assert_eq!(post_commit.height, 0); + assert_eq!(post_commit.tx_count, 0); + + listener.finish_block(height, tx_delta); + let final_snapshot = listener.snapshot(); + let mut expected_final = expected_post_commit; + expected_final.finish_block(height, tx_delta); + assert_eq!(final_snapshot, expected_final); + assert_eq!(final_snapshot.total_amount, post_commit.total_amount); + assert_eq!(final_snapshot.bogo_size, post_commit.bogo_size); + assert_eq!(final_snapshot.utxo_count, post_commit.utxo_count); + Ok(()) +} + +struct FirstInsertGate { + inner: CoinStatsListener, + gate: Arc, + barrier: Arc, +} + +impl UtxoChangeListener for FirstInsertGate { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.inner.on_insert(op, txout, height, coinbase); + if self + .gate + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + self.barrier.wait(); + self.barrier.wait(); + } + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { + self.inner.on_remove(op, txout, height); + } + + fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.inner.on_remove_coin(op, txout, height, coinbase); + } + + fn muhash3072(&self) -> Option<[u8; 384]> { + self.inner.muhash3072() + } +} + fn txout(index: u32) -> TxOut { TxOut { value: Amount::from_sat(50_000 + u64::from(index)), From e4911f22d188fd7c249a6af8e1afa1736076dd22 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 05:29:10 +0000 Subject: [PATCH 13/37] perf(node): count coinstats replay listener callbacks Extend the ignored replay cost harness with a test-local listener wrapper that forwards to the real CoinStatsListener and counts callback kinds. This keeps production code untouched while making attached/detached replay evidence less blind. Evidence: - cargo test -p bitcoin-rs-node --test sync_smoke --no-default-features --features bitcoinconsensus,redb bounded_apply_profile_replay_coinstats_listener_cost -- --ignored --nocapture - fresh sync_smoke binary: attached elapsed_ms=28 listener_total_calls=106; detached elapsed_ms=15 listener_total_calls=0; test result ok - cargo fmt --all -- --check - cargo clippy -p bitcoin-rs-node --test sync_smoke --no-default-features --features bitcoinconsensus,redb -- -D warnings Op: extend --- crates/node/tests/sync_smoke.rs | 95 ++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 8 deletions(-) diff --git a/crates/node/tests/sync_smoke.rs b/crates/node/tests/sync_smoke.rs index e01e4eb..82a9bea 100644 --- a/crates/node/tests/sync_smoke.rs +++ b/crates/node/tests/sync_smoke.rs @@ -1,6 +1,9 @@ //! Block sync smoke tests. use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::Arc; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; use std::time::{Duration, Instant}; use arc_swap::ArcSwapOption; @@ -18,7 +21,7 @@ use bitcoin_rs_mempool::{Mempool, MempoolLimits}; use bitcoin_rs_node::{BlockSync, Config, Network, apply::ApplyHandles, state::NodeState}; use bitcoin_rs_p2p::{Message, PeerInfo}; use bitcoin_rs_primitives::{Hash256, OutPoint}; -use bitcoin_rs_utxo::UtxoSet; +use bitcoin_rs_utxo::{UtxoChangeListener, UtxoSet}; use crossbeam_channel::unbounded; use hashbrown::HashMap; use parking_lot::{Mutex, RwLock}; @@ -353,12 +356,17 @@ fn bounded_apply_profile_replay_coinstats_listener_cost() -> Result<(), Box, + listener_calls: ListenerCallCountSnapshot, utxo: Arc, } +#[derive(Default)] +struct ListenerCallCounters { + insert_calls: AtomicU64, + remove_calls: AtomicU64, + remove_coin_calls: AtomicU64, +} + +impl ListenerCallCounters { + fn snapshot(&self) -> ListenerCallCountSnapshot { + ListenerCallCountSnapshot { + insert_calls: self.insert_calls.load(Ordering::Relaxed), + remove_calls: self.remove_calls.load(Ordering::Relaxed), + remove_coin_calls: self.remove_coin_calls.load(Ordering::Relaxed), + } + } +} + +#[derive(Clone, Copy, Default)] +struct ListenerCallCountSnapshot { + insert_calls: u64, + remove_calls: u64, + remove_coin_calls: u64, +} + +impl ListenerCallCountSnapshot { + const fn total_calls(self) -> u64 { + self.insert_calls + .saturating_add(self.remove_calls) + .saturating_add(self.remove_coin_calls) + } +} + +struct CountingCoinStatsListener { + inner: CoinStatsListener, + counters: Arc, +} + +impl UtxoChangeListener for CountingCoinStatsListener { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.counters.insert_calls.fetch_add(1, Ordering::Relaxed); + self.inner.on_insert(op, txout, height, coinbase); + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { + self.counters.remove_calls.fetch_add(1, Ordering::Relaxed); + self.inner.on_remove(op, txout, height); + } + + fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.counters + .remove_coin_calls + .fetch_add(1, Ordering::Relaxed); + self.inner.on_remove_coin(op, txout, height, coinbase); + } + + fn muhash3072(&self) -> Option<[u8; 384]> { + self.inner.muhash3072() + } +} + #[derive(Clone, Copy)] enum CoinStatsListenerMode { Attached, @@ -484,7 +553,7 @@ fn replay_non_coinbase_spend_chain_with_coin_stats_listener( let inbound_headers_rx = Arc::new(Mutex::new(inbound_headers_rx_raw)); let (inbound_blocks_tx, inbound_blocks_rx_raw) = unbounded::(); let inbound_blocks_rx = Arc::new(Mutex::new(inbound_blocks_rx_raw)); - let (mut handles, coin_stats, utxo) = apply_handles_with_coin_stats_and_utxo( + let (mut handles, coin_stats, listener_calls, utxo) = apply_handles_with_coin_stats_and_utxo( Network::Regtest, Arc::clone(&chain_tip), Arc::clone(&applied_tip), @@ -542,6 +611,7 @@ fn replay_non_coinbase_spend_chain_with_coin_stats_listener( applied_height: applied.height, applied_hash: applied.hash, coin_stats, + listener_calls: listener_calls.snapshot(), utxo, }) } @@ -677,7 +747,7 @@ fn apply_handles_with_coin_stats( applied_tip: Arc>, block_tree: Arc>, ) -> (ApplyHandles, Arc) { - let (handles, coin_stats, _utxo) = apply_handles_with_coin_stats_and_utxo( + let (handles, coin_stats, _listener_calls, _utxo) = apply_handles_with_coin_stats_and_utxo( network, chain_tip, applied_tip, @@ -694,11 +764,20 @@ fn apply_handles_with_coin_stats_and_utxo( applied_tip: Arc>, block_tree: Arc>, coin_stats_listener: CoinStatsListenerMode, -) -> (ApplyHandles, Arc, Arc) { +) -> ( + ApplyHandles, + Arc, + Arc, + Arc, +) { let coin_stats = Arc::new(CoinStatsListener::new(CoinStats::default())); + let listener_calls = Arc::new(ListenerCallCounters::default()); let mut utxo = UtxoSet::new(); if matches!(coin_stats_listener, CoinStatsListenerMode::Attached) { - utxo.set_listener(Box::new((*coin_stats).clone())); + utxo.set_listener(Box::new(CountingCoinStatsListener { + inner: (*coin_stats).clone(), + counters: Arc::clone(&listener_calls), + })); } let utxo = Arc::new(utxo); let handles = ApplyHandles::new( @@ -715,7 +794,7 @@ fn apply_handles_with_coin_stats_and_utxo( Arc::new(RwLock::new(HashMap::::new())), Arc::new(bitcoin_rs_node::NoOpZmqPublisher), ); - (handles, coin_stats, utxo) + (handles, coin_stats, listener_calls, utxo) } struct NoopIndexer; From 74ea2ed43bb79fd5a494d71b7f5ddfd2385fa546 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 08:07:22 +0000 Subject: [PATCH 14/37] perf(coinstats): shard listener accounting Replace the single CoinStats listener write lock with private per-UTXO-shard signed deltas and a linearization barrier for snapshots and block folds. This preserves the public listener surface while removing the global callback serialization seen in the UTXO commit benchmark. Evidence: - cargo fmt --all -- --check - cargo clippy -p bitcoin-rs-coinstats --all-targets -- -D warnings - cargo test -p bitcoin-rs-coinstats - cargo test --workspace --no-fail-fast - cargo bench -p bitcoin-rs-coinstats --bench utxo_commit_coinstats -- utxo_commit_coinstats Op: compress --- crates/coinstats/src/stats.rs | 119 ++++++++++- .../coinstats/tests/snapshot_with_muhash.rs | 200 +++++++++++++++++- 2 files changed, 309 insertions(+), 10 deletions(-) diff --git a/crates/coinstats/src/stats.rs b/crates/coinstats/src/stats.rs index cc085f2..bcb2507 100644 --- a/crates/coinstats/src/stats.rs +++ b/crates/coinstats/src/stats.rs @@ -3,7 +3,7 @@ use core::convert::Infallible; use bitcoin::consensus::Encodable; use bitcoin_rs_primitives::{OutPoint, TxOut}; -use bitcoin_rs_utxo::UtxoChangeListener; +use bitcoin_rs_utxo::{UtxoChangeListener, UtxoKey}; use parking_lot::RwLock; use zerocopy::IntoBytes; @@ -126,7 +126,22 @@ pub enum CoinStatsDecodeError { /// UTXO listener that maintains [`CoinStats`]. #[derive(Clone, Debug)] pub struct CoinStatsListener { - stats: Arc>, + inner: Arc, +} + +#[derive(Debug)] +struct CoinStatsListenerInner { + linearize: RwLock<()>, + base: RwLock, + deltas: [RwLock; UtxoKey::SHARD_COUNT], +} + +#[derive(Clone, Debug, Default)] +struct CoinStatsDelta { + muhash: MuHash3072, + total_amount: i128, + bogo_size: i128, + utxo_count: i128, } impl CoinStatsListener { @@ -134,40 +149,126 @@ impl CoinStatsListener { #[must_use] pub fn new(stats: CoinStats) -> Self { Self { - stats: Arc::new(RwLock::new(stats)), + inner: Arc::new(CoinStatsListenerInner::new(stats)), } } /// Returns a point-in-time copy of the current stats. #[must_use] pub fn snapshot(&self) -> CoinStats { - self.stats.read().clone() + let _linearized = self.inner.linearize.write(); + self.inner.materialize() } /// Applies a per-block delta to the wrapped stats. pub fn finish_block(&self, height: u32, tx_delta: u64) { - self.stats.write().finish_block(height, tx_delta); + let _linearized = self.inner.linearize.write(); + self.inner.fold_deltas(height, tx_delta); + } +} + +impl CoinStatsListenerInner { + fn new(base: CoinStats) -> Self { + Self { + linearize: RwLock::new(()), + base: RwLock::new(base), + deltas: core::array::from_fn(|_| RwLock::new(CoinStatsDelta::default())), + } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.deltas[index] + } + + fn materialize(&self) -> CoinStats { + let mut stats = self.base.read().clone(); + for delta in &self.deltas { + delta.read().apply_to(&mut stats); + } + stats + } + + fn fold_deltas(&self, height: u32, tx_delta: u64) { + let mut base = self.base.write(); + for delta in &self.deltas { + let mut delta = delta.write(); + delta.apply_to(&mut base); + *delta = CoinStatsDelta::default(); + } + base.finish_block(height, tx_delta); + } +} + +impl CoinStatsDelta { + fn insert_utxo(&mut self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let encoded = coin_hash_bytes(op, txout, height, coinbase); + self.muhash.insert(&encoded); + self.total_amount = self + .total_amount + .saturating_add(i128::from(txout.value.to_sat())); + self.bogo_size = self.bogo_size.saturating_add(i128::from(bogo_size(txout))); + self.utxo_count = self.utxo_count.saturating_add(1); + } + + fn remove_utxo(&mut self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let encoded = coin_hash_bytes(op, txout, height, coinbase); + self.muhash.remove(&encoded); + self.total_amount = self + .total_amount + .saturating_sub(i128::from(txout.value.to_sat())); + self.bogo_size = self.bogo_size.saturating_sub(i128::from(bogo_size(txout))); + self.utxo_count = self.utxo_count.saturating_sub(1); + } + + fn apply_to(&self, stats: &mut CoinStats) { + stats.muhash.combine(&self.muhash); + apply_signed_delta(&mut stats.total_amount, self.total_amount); + apply_signed_delta(&mut stats.bogo_size, self.bogo_size); + apply_signed_delta(&mut stats.utxo_count, self.utxo_count); } } impl UtxoChangeListener for CoinStatsListener { fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { - self.stats.write().insert_utxo(op, txout, height, coinbase); + let _linearized = self.inner.linearize.read(); + self.inner + .shard(op) + .write() + .insert_utxo(op, txout, height, coinbase); } fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { - self.stats.write().remove_utxo(op, txout, height, false); + let _linearized = self.inner.linearize.read(); + self.inner + .shard(op) + .write() + .remove_utxo(op, txout, height, false); } fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { - self.stats.write().remove_utxo(op, txout, height, coinbase); + let _linearized = self.inner.linearize.read(); + self.inner + .shard(op) + .write() + .remove_utxo(op, txout, height, coinbase); } fn muhash3072(&self) -> Option<[u8; 384]> { - Some(self.stats.read().muhash.finalize()) + Some(self.snapshot().muhash.finalize()) } } +fn apply_signed_delta(value: &mut u64, delta: i128) { + if let Ok(increment) = u64::try_from(delta) { + *value = value.saturating_add(increment); + return; + } + + let decrement = u64::try_from(delta.saturating_neg()).unwrap_or(u64::MAX); + *value = value.saturating_sub(decrement); +} + fn coin_hash_bytes(op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) -> Vec { let mut out = Vec::with_capacity(OUTPOINT_BYTES + 4 + txout.script_pubkey.len() + 16); out.extend_from_slice(op.as_bytes()); diff --git a/crates/coinstats/tests/snapshot_with_muhash.rs b/crates/coinstats/tests/snapshot_with_muhash.rs index 8b648ca..4f7bfb7 100644 --- a/crates/coinstats/tests/snapshot_with_muhash.rs +++ b/crates/coinstats/tests/snapshot_with_muhash.rs @@ -11,7 +11,8 @@ use bitcoin::{Amount, ScriptBuf}; use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener}; use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; use bitcoin_rs_utxo::{ - BlockChanges, UtxoAdd, UtxoChangeListener, UtxoKey, UtxoSet, write_snapshot, + BlockChanges, UndoBatch, UtxoAdd, UtxoChangeListener, UtxoError, UtxoKey, UtxoSet, + write_snapshot, }; #[test] @@ -87,6 +88,203 @@ fn snapshot_trailer_tracks_listener_after_removal() -> Result<(), Box Result<(), Box> { + let outpoint = OutPoint::new(txid(20), 0); + let txout = txout(20); + let height = 17; + let mut set = UtxoSet::new(); + let mut preload = BlockChanges::default(); + preload.add(UtxoAdd::new(outpoint, txout.clone(), false, height)); + set.commit_block(&preload, &txid(200))?; + + let mut base = CoinStats::new(); + base.insert_utxo(&outpoint, &txout, height, false); + let listener = CoinStatsListener::new(base.clone()); + set.set_listener(Box::new(listener.clone())); + + let mut removes = BlockChanges::default(); + removes.remove(outpoint); + set.commit_block(&removes, &txid(201))?; + + let mut expected = base; + expected.remove_utxo(&outpoint, &txout, height, false); + let snapshot = listener.snapshot(); + assert_eq!(snapshot, expected); + assert_eq!(snapshot.height, 0); + assert_eq!(snapshot.tx_count, 0); + Ok(()) +} + +#[test] +fn finish_block_folds_deltas_once_before_next_block() -> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let first_outpoint = OutPoint::new(txid(40), 0); + let second_outpoint = OutPoint::new(txid(41), 0); + let first_txout = txout(40); + let second_txout = txout(41); + + let mut first = BlockChanges::default(); + first.add(UtxoAdd::new(first_outpoint, first_txout.clone(), false, 11)); + set.commit_block(&first, &txid(300))?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&first_outpoint, &first_txout, 11, false); + assert_eq!(listener.snapshot(), expected); + + listener.finish_block(11, 1); + expected.finish_block(11, 1); + assert_eq!(listener.snapshot(), expected); + assert_eq!(listener.snapshot(), expected); + + let mut second = BlockChanges::default(); + second.add(UtxoAdd::new( + second_outpoint, + second_txout.clone(), + true, + 12, + )); + set.commit_block(&second, &txid(301))?; + + expected.insert_utxo(&second_outpoint, &second_txout, 12, true); + assert_eq!(listener.snapshot(), expected); + + listener.finish_block(12, 1); + expected.finish_block(12, 1); + assert_eq!(listener.snapshot(), expected); + Ok(()) +} + +#[test] +fn listener_muhash_matches_snapshot_after_remove_and_readd() +-> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let outpoint = OutPoint::new(txid(60), 0); + let original = txout(60); + let replacement = txout(61); + + let mut add = BlockChanges::default(); + add.add(UtxoAdd::new(outpoint, original.clone(), false, 21)); + set.commit_block(&add, &txid(400))?; + + let mut remove = BlockChanges::default(); + remove.remove(outpoint); + set.commit_block(&remove, &txid(401))?; + + let mut readd = BlockChanges::default(); + readd.add(UtxoAdd::new(outpoint, replacement.clone(), true, 22)); + set.commit_block(&readd, &txid(402))?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&outpoint, &original, 21, false); + expected.remove_utxo(&outpoint, &original, 21, false); + expected.insert_utxo(&outpoint, &replacement, 22, true); + + let snapshot = listener.snapshot(); + let trailer = UtxoChangeListener::muhash3072(&listener).unwrap_or([0_u8; 384]); + assert_eq!(snapshot, expected); + assert_eq!(trailer, snapshot.muhash.finalize()); + assert_ne!(trailer, [0_u8; 384]); + Ok(()) +} + +#[test] +fn failed_block_prevalidation_leaves_listener_clean_for_retry() +-> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let valid_outpoint = OutPoint::new(txid(70), 0); + let valid_txout = txout(70); + let invalid_outpoint = OutPoint::new(txid(71), 0); + let invalid_txout = TxOut { + value: Amount::from_sat(71_000), + script_pubkey: ScriptBuf::from_bytes(vec![0x51; usize::from(u16::MAX) + 1]), + }; + + let mut failed = BlockChanges::default(); + failed.add(UtxoAdd::new(valid_outpoint, valid_txout.clone(), false, 31)); + failed.add(UtxoAdd::new(invalid_outpoint, invalid_txout, false, 31)); + let error = match set.commit_block(&failed, &txid(500)) { + Ok(()) => return Err("oversized script block unexpectedly committed".into()), + Err(error) => error, + }; + + assert!(matches!(error, UtxoError::ScriptTooLarge { .. })); + assert_eq!(set.get(&valid_outpoint), None); + assert_eq!(listener.snapshot(), CoinStats::new()); + + let mut retry = BlockChanges::default(); + retry.add(UtxoAdd::new(valid_outpoint, valid_txout.clone(), false, 31)); + set.commit_block(&retry, &txid(501))?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&valid_outpoint, &valid_txout, 31, false); + assert_eq!(set.get(&valid_outpoint), Some(valid_txout)); + assert_eq!(listener.snapshot(), expected); + Ok(()) +} + +#[test] +fn undo_block_reverses_unfinished_listener_deltas() -> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let spent_outpoint = OutPoint::new(txid(80), 0); + let spent_txout = txout(80); + let created_outpoint = OutPoint::new(txid(81), 0); + let created_txout = txout(81); + + let mut preload = BlockChanges::default(); + preload.add(UtxoAdd::new(spent_outpoint, spent_txout.clone(), false, 40)); + set.commit_block(&preload, &txid(600))?; + + let mut connected = BlockChanges::default(); + connected.remove(spent_outpoint); + connected.add(UtxoAdd::new( + created_outpoint, + created_txout.clone(), + true, + 41, + )); + set.commit_block(&connected, &txid(601))?; + + let mut after_connect = CoinStats::new(); + after_connect.insert_utxo(&spent_outpoint, &spent_txout, 40, false); + after_connect.remove_utxo(&spent_outpoint, &spent_txout, 40, false); + after_connect.insert_utxo(&created_outpoint, &created_txout, 41, true); + assert_eq!(listener.snapshot(), after_connect); + + let mut undo = UndoBatch::default(); + undo.restore(UtxoAdd::new(spent_outpoint, spent_txout.clone(), false, 40)); + undo.remove(created_outpoint); + set.undo_block(&undo)?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&spent_outpoint, &spent_txout, 40, false); + expected.remove_utxo(&spent_outpoint, &spent_txout, 40, false); + expected.insert_utxo(&created_outpoint, &created_txout, 41, true); + expected.insert_utxo(&spent_outpoint, &spent_txout, 40, false); + expected.remove_utxo(&created_outpoint, &created_txout, 41, true); + let mut canonical = CoinStats::new(); + canonical.insert_utxo(&spent_outpoint, &spent_txout, 40, false); + let snapshot = listener.snapshot(); + assert_eq!(set.get(&spent_outpoint), Some(spent_txout)); + assert_eq!(set.get(&created_outpoint), None); + assert_eq!(snapshot, expected); + assert_eq!(snapshot.muhash.finalize(), canonical.muhash.finalize()); + Ok(()) +} + #[test] fn listener_tracks_duplicate_txid_overwrite() -> Result<(), Box> { let listener = CoinStatsListener::new(CoinStats::new()); From bf7158f4f9f6f4e55c574a6dc9886bbb17803f7b Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 11:04:00 +0000 Subject: [PATCH 15/37] fix(node): restore clippy gate Replace test expect calls with explicit error propagation and keep the bitcoinconsensus-only test helper feature-gated so default clippy and feature builds both compile.\n\nEvidence:\n- cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check\n- cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --all-targets -- -D warnings\n- cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --no-default-features --features bitcoinconsensus,rocksdb,redb --no-fail-fast\n\nOp: correct\nRestores: spec:bitcoin-rs-node clippy -D warnings all-targets gate --- crates/node/src/apply.rs | 2 ++ crates/node/src/state.rs | 16 ++++++++++++---- crates/node/tests/rpc_wiring.rs | 29 ++++++++++++++++------------- crates/node/tests/state_storage.rs | 7 ++++--- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index ee9612b..ff9f765 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -2027,6 +2027,7 @@ mod consensus_rule_tests { } } + #[cfg(feature = "bitcoinconsensus")] fn block_with_prev_hash_and_transactions( prev_blockhash: bitcoin::BlockHash, txdata: Vec, @@ -2048,6 +2049,7 @@ mod consensus_rule_tests { block } + #[cfg(feature = "bitcoinconsensus")] fn mined_block_with_prev_hash_and_transactions( prev_blockhash: bitcoin::BlockHash, txdata: Vec, diff --git a/crates/node/src/state.rs b/crates/node/src/state.rs index f6f3bd9..135ce69 100644 --- a/crates/node/src/state.rs +++ b/crates/node/src/state.rs @@ -1128,8 +1128,12 @@ mod tests { config.p2p_listen.clear(); config.txindex = true; let state = NodeState::open(config)?; - let a = state.tx_index().expect("txindex enabled"); - let b = state.tx_index().expect("txindex enabled"); + let a = state + .tx_index() + .ok_or_else(|| anyhow::anyhow!("txindex enabled"))?; + let b = state + .tx_index() + .ok_or_else(|| anyhow::anyhow!("txindex enabled"))?; assert!(Arc::ptr_eq(&a, &b), "tx_index handle stable across calls"); Ok(()) } @@ -1157,8 +1161,12 @@ mod tests { config.p2p_listen.clear(); config.blockfilterindex = true; let state = NodeState::open(config)?; - let a = state.filter_index().expect("blockfilterindex enabled"); - let b = state.filter_index().expect("blockfilterindex enabled"); + let a = state + .filter_index() + .ok_or_else(|| anyhow::anyhow!("blockfilterindex enabled"))?; + let b = state + .filter_index() + .ok_or_else(|| anyhow::anyhow!("blockfilterindex enabled"))?; assert!( Arc::ptr_eq(&a, &b), "filter_index handle stable across calls" diff --git a/crates/node/tests/rpc_wiring.rs b/crates/node/tests/rpc_wiring.rs index 80b732d..5a9654d 100644 --- a/crates/node/tests/rpc_wiring.rs +++ b/crates/node/tests/rpc_wiring.rs @@ -92,23 +92,26 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { Arc::ptr_eq(&ctx.coin_stats, &coin_stats), "coin_stats must share identity" ); + let ctx_filter_index = ctx + .filter_index + .as_ref() + .ok_or_else(|| anyhow::anyhow!("filter index must be wired"))?; + let node_filter_index = filter_index + .as_ref() + .ok_or_else(|| anyhow::anyhow!("node filter index must be wired"))?; assert!( - Arc::ptr_eq( - ctx.filter_index - .as_ref() - .expect("filter index must be wired"), - filter_index - .as_ref() - .expect("node filter index must be wired") - ), + Arc::ptr_eq(ctx_filter_index, node_filter_index), "filter_index must share identity" ); - let ctx_indexer = ctx.indexer.as_ref().expect("indexer handle must be wired"); + let ctx_indexer = ctx + .indexer + .as_ref() + .ok_or_else(|| anyhow::anyhow!("indexer handle must be wired"))?; + let node_tx_index = tx_index + .as_ref() + .ok_or_else(|| anyhow::anyhow!("node txindex must be wired"))?; assert!( - Arc::ptr_eq( - ctx_indexer, - tx_index.as_ref().expect("node txindex must be wired") - ), + Arc::ptr_eq(ctx_indexer, node_tx_index), "indexer handle must share identity" ); assert!( diff --git a/crates/node/tests/state_storage.rs b/crates/node/tests/state_storage.rs index 684987e..b1378ff 100644 --- a/crates/node/tests/state_storage.rs +++ b/crates/node/tests/state_storage.rs @@ -45,9 +45,10 @@ fn electrum_bind_requires_txindex() -> Result<()> { config.electrum_bind = Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)); config.txindex = false; - let error = config - .validate() - .expect_err("electrum without txindex unexpectedly validated"); + let error = match config.validate() { + Ok(()) => anyhow::bail!("electrum without txindex unexpectedly validated"), + Err(error) => error, + }; assert!(error.to_string().contains("electrum_bind requires txindex")); Ok(()) From 3abb9813a0362d3833d39c7fa3f2b95704767d39 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 11:04:40 +0000 Subject: [PATCH 16/37] perf(node): fill block sync pending budget Use the existing BlockSync pending-budget window as the getdata issuance cap instead of keeping a second per-tick batch cap. The first tick now fills the pending window from the next applied height, and a second tick with the window full sends only getheaders.\n\nEvidence:\n- cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node tick_fills_pending_budget_from_next_applied_height_when_gap_exceeds_budget\n- cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node second_tick_does_not_re_request_already_pending_blocks\n- cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node sync::tests\n- cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check\n- cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --all-targets -- -D warnings\n- cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --no-default-features --features bitcoinconsensus,rocksdb,redb --no-fail-fast\n\nOp: extend --- crates/node/src/sync.rs | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/crates/node/src/sync.rs b/crates/node/src/sync.rs index 643dd87..ad7f104 100644 --- a/crates/node/src/sync.rs +++ b/crates/node/src/sync.rs @@ -25,8 +25,6 @@ use parking_lot::{Mutex, RwLock}; const LOCATOR_MAX_ENTRIES: usize = 32; /// Wire protocol version we advertise on outbound `getheaders`. const PROTOCOL_VERSION: u32 = 70_016; -/// Maximum number of block inventory entries we request per tick. -const GETDATA_BATCH_SIZE: usize = 16; /// Time after which a pending getdata is considered stuck and re-requestable. const PENDING_TIMEOUT: Duration = Duration::from_mins(1); /// Maximum number of in-flight getdata requests we'll track per `BlockSync`. @@ -250,14 +248,13 @@ impl BlockSync { ); return; } - let batch_cap = remaining_budget.min(GETDATA_BATCH_SIZE); - let mut hashes: Vec = Vec::with_capacity(batch_cap); + let mut hashes: Vec = Vec::with_capacity(remaining_budget); let tree = self.handles.block_tree.read(); let Some(mut height) = applied_height.checked_add(1) else { return; }; - while hashes.len() < batch_cap && height <= chain_tip.height { + while hashes.len() < remaining_budget && height <= chain_tip.height { let Some(node_id) = tree.node_at_height_from(chain_tip.tip_id, height) else { break; }; @@ -504,20 +501,20 @@ mod tests { } #[test] - fn tick_sends_getdata_from_next_applied_height_when_gap_exceeds_batch() + fn tick_fills_pending_budget_from_next_applied_height_when_gap_exceeds_budget() -> Result<(), Box> { let mut tree = BlockTree::new(); let genesis = genesis_header(); let genesis_id = tree.insert_node(None, genesis, NodeStatus::HeaderValid)?; let mut tip_id = genesis_id; let mut expected = Vec::new(); - let batch_size = u32::try_from(super::GETDATA_BATCH_SIZE)?; + let pending_budget = u32::try_from(super::PENDING_BUDGET)?; - for height in 1_u32..=batch_size + 4 { + for height in 1_u32..=pending_budget + 4 { let parent_hash = BlockHash::from_byte_array(tree.node(tip_id)?.hash.to_le_bytes()); let header = test_header(parent_hash, height); tip_id = tree.insert_node(Some(tip_id), header, NodeStatus::HeaderValid)?; - if height <= batch_size { + if height <= pending_budget { expected.push(BlockHash::from_byte_array( tree.node(tip_id)?.hash.to_le_bytes(), )); @@ -565,7 +562,30 @@ mod tests { }) .collect::, _>>()?; assert_eq!(requested, expected); - Ok(()) + + let second = rx.try_recv()?; + if !matches!(second, NetworkMessage::GetHeaders(_)) { + return Err(std::io::Error::other("expected first tick getheaders").into()); + } + + sync.tick(); + + let third = rx.try_recv()?; + if !matches!(third, NetworkMessage::GetHeaders(_)) { + return Err(std::io::Error::other("expected second tick getheaders only").into()); + } + match rx.try_recv() { + Ok(NetworkMessage::GetData(_)) => { + Err(std::io::Error::other("second tick requested beyond pending budget").into()) + } + Ok(_) => { + Err(std::io::Error::other("unexpected extra message after second tick").into()) + } + Err(crossbeam_channel::TryRecvError::Empty) => Ok(()), + Err(crossbeam_channel::TryRecvError::Disconnected) => { + Err(std::io::Error::other("outbound channel disconnected").into()) + } + } } #[test] From 6f160ac2afff90fbc7f36d29f2819617adb09b9a Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 13:08:51 +0000 Subject: [PATCH 17/37] perf(coinstats): skip clean shard folds Track dirty CoinStats listener shards so finish_block and snapshot skip clean shard deltas instead of combining identity MuHash accumulators every block. Existing linearize locking remains the synchronization contract; dirty bits are hints under that lock. Evidence: bounded apply replay coin_stats_us fell from 29,710us to 2,613us for index_hooks=disabled and from 14,252us to 2,513us for index_hooks=noop in /tmp/sync-smoke-bounded-apply-profile-logs*. Op: compress --- crates/coinstats/src/stats.rs | 62 ++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/crates/coinstats/src/stats.rs b/crates/coinstats/src/stats.rs index bcb2507..f52dec7 100644 --- a/crates/coinstats/src/stats.rs +++ b/crates/coinstats/src/stats.rs @@ -1,5 +1,8 @@ use alloc::sync::Arc; -use core::convert::Infallible; +use core::{ + convert::Infallible, + sync::atomic::{AtomicBool, Ordering}, +}; use bitcoin::consensus::Encodable; use bitcoin_rs_primitives::{OutPoint, TxOut}; @@ -134,6 +137,9 @@ struct CoinStatsListenerInner { linearize: RwLock<()>, base: RwLock, deltas: [RwLock; UtxoKey::SHARD_COUNT], + // Dirty bits are hints guarded by `linearize`: callbacks set the bit before + // dropping the read guard; snapshot/fold read it under the write guard. + dirty: [AtomicBool; UtxoKey::SHARD_COUNT], } #[derive(Clone, Debug, Default)] @@ -173,31 +179,52 @@ impl CoinStatsListenerInner { linearize: RwLock::new(()), base: RwLock::new(base), deltas: core::array::from_fn(|_| RwLock::new(CoinStatsDelta::default())), + dirty: core::array::from_fn(|_| AtomicBool::new(false)), } } - fn shard(&self, op: &OutPoint) -> &RwLock { - let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); - &self.deltas[index] + fn shard_index(op: &OutPoint) -> usize { + usize::from(UtxoKey::from_txid(&op.txid).shard()) } fn materialize(&self) -> CoinStats { let mut stats = self.base.read().clone(); - for delta in &self.deltas { - delta.read().apply_to(&mut stats); + for (delta, dirty) in self.deltas.iter().zip(&self.dirty) { + if dirty.load(Ordering::Relaxed) { + delta.read().apply_to(&mut stats); + } } stats } fn fold_deltas(&self, height: u32, tx_delta: u64) { let mut base = self.base.write(); - for delta in &self.deltas { - let mut delta = delta.write(); - delta.apply_to(&mut base); - *delta = CoinStatsDelta::default(); + for (delta, dirty) in self.deltas.iter().zip(&self.dirty) { + if dirty.load(Ordering::Relaxed) { + let mut delta = delta.write(); + dirty.store(false, Ordering::Relaxed); + delta.apply_to(&mut base); + *delta = CoinStatsDelta::default(); + } } base.finish_block(height, tx_delta); } + + fn insert_utxo(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let index = Self::shard_index(op); + self.deltas[index] + .write() + .insert_utxo(op, txout, height, coinbase); + self.dirty[index].store(true, Ordering::Relaxed); + } + + fn remove_utxo(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let index = Self::shard_index(op); + self.deltas[index] + .write() + .remove_utxo(op, txout, height, coinbase); + self.dirty[index].store(true, Ordering::Relaxed); + } } impl CoinStatsDelta { @@ -232,26 +259,17 @@ impl CoinStatsDelta { impl UtxoChangeListener for CoinStatsListener { fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { let _linearized = self.inner.linearize.read(); - self.inner - .shard(op) - .write() - .insert_utxo(op, txout, height, coinbase); + self.inner.insert_utxo(op, txout, height, coinbase); } fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { let _linearized = self.inner.linearize.read(); - self.inner - .shard(op) - .write() - .remove_utxo(op, txout, height, false); + self.inner.remove_utxo(op, txout, height, false); } fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { let _linearized = self.inner.linearize.read(); - self.inner - .shard(op) - .write() - .remove_utxo(op, txout, height, coinbase); + self.inner.remove_utxo(op, txout, height, coinbase); } fn muhash3072(&self) -> Option<[u8; 384]> { From fd27b74206f9ac19c886b9da1018fe4d76c794ae Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 14:32:27 +0000 Subject: [PATCH 18/37] perf(utxo): skip clean shard dispatch Track dirty UTXO shards while bucketing a block commit, use fixed shard bucket arrays, and execute the single-shard case directly under the stable-view lock. This removes the all-shard scan and Rayon dispatch overhead for the common tiny-block path without changing the public UTXO contract. Evidence: bounded apply replay non-genesis utxo_commit_us fell from 5,750us to 1,462us for index_hooks=disabled and from 5,321us to 1,850us for index_hooks=noop in /tmp/sync-smoke-bounded-apply-profile-logs-dirty-*. Op: compress --- crates/utxo/src/set.rs | 44 +++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/crates/utxo/src/set.rs b/crates/utxo/src/set.rs index beabb98..f96e0f3 100644 --- a/crates/utxo/src/set.rs +++ b/crates/utxo/src/set.rs @@ -427,15 +427,21 @@ impl UtxoSet { ) -> Result<(), UtxoError> { let mut adds_by_shard = empty_add_buckets(); let mut removes_by_shard = empty_remove_buckets(); + let mut dirty = [false; UtxoKey::SHARD_COUNT]; + let mut dirty_shards = Vec::new(); for add in adds { validate_add(add)?; let key = UtxoKey::from_txid(&add.outpoint.txid); - adds_by_shard[usize::from(key.shard())].push((key, add.outpoint.txid, add.payload())); + let shard_idx = usize::from(key.shard()); + mark_dirty(&mut dirty, &mut dirty_shards, shard_idx); + adds_by_shard[shard_idx].push((key, add.outpoint.txid, add.payload())); } for remove in removes { let key = UtxoKey::from_txid(&remove.txid); - removes_by_shard[usize::from(key.shard())].push(SpendPayload { + let shard_idx = usize::from(key.shard()); + mark_dirty(&mut dirty, &mut dirty_shards, shard_idx); + removes_by_shard[shard_idx].push(SpendPayload { op: remove, key, vout: remove.vout, @@ -444,16 +450,21 @@ impl UtxoSet { } let _stable_commit = self.stable_view_lock.write(); + let listener = self.listener.as_deref(); + + if let [shard_idx] = dirty_shards.as_slice() { + return self.shards[*shard_idx].commit_batch( + &adds_by_shard[*shard_idx], + &removes_by_shard[*shard_idx], + listener, + ); + } let errors = Mutex::new(Vec::new()); - let listener = self.listener.as_deref(); rayon::scope(|scope| { - for shard_idx in 0..UtxoKey::SHARD_COUNT { + for shard_idx in dirty_shards.iter().copied() { let shard_adds = &adds_by_shard[shard_idx]; let shard_removes = &removes_by_shard[shard_idx]; - if shard_adds.is_empty() && shard_removes.is_empty() { - continue; - } let shard = &self.shards[shard_idx]; let errors = &errors; scope.spawn(move |_| { @@ -486,12 +497,23 @@ fn validate_add(add: &UtxoAdd) -> Result<(), UtxoError> { Ok(()) } -fn empty_add_buckets<'a>() -> Vec)>> { - (0..UtxoKey::SHARD_COUNT).map(|_| Vec::new()).collect() +fn empty_add_buckets<'a>() -> [Vec<(UtxoKey, Hash256, BuildPayload<'a>)>; UtxoKey::SHARD_COUNT] { + core::array::from_fn(|_| Vec::new()) } -fn empty_remove_buckets<'a>() -> Vec>> { - (0..UtxoKey::SHARD_COUNT).map(|_| Vec::new()).collect() +fn empty_remove_buckets<'a>() -> [Vec>; UtxoKey::SHARD_COUNT] { + core::array::from_fn(|_| Vec::new()) +} + +fn mark_dirty( + dirty: &mut [bool; UtxoKey::SHARD_COUNT], + dirty_shards: &mut Vec, + shard_idx: usize, +) { + if !dirty[shard_idx] { + dirty[shard_idx] = true; + dirty_shards.push(shard_idx); + } } fn stable_view_len(view: &UtxoSetView<'_>) -> usize { view.len() From bfab190ce12c36e27acd505c59893c59810b7117 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 15:50:26 +0000 Subject: [PATCH 19/37] test(coinstats): cover empty utxo commit no-op Add a regression oracle for the zero-dirty-shard UTXO commit path. The test seeds both folded CoinStats base state and an unfinished dirty delta, then proves an empty BlockChanges commit preserves the UTXO set and listener snapshot. Op: correct Restores: spec:empty UTXO commits preserve existing set and CoinStats listener state --- .../coinstats/tests/snapshot_with_muhash.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/coinstats/tests/snapshot_with_muhash.rs b/crates/coinstats/tests/snapshot_with_muhash.rs index 4f7bfb7..cf54265 100644 --- a/crates/coinstats/tests/snapshot_with_muhash.rs +++ b/crates/coinstats/tests/snapshot_with_muhash.rs @@ -233,6 +233,46 @@ fn failed_block_prevalidation_leaves_listener_clean_for_retry() Ok(()) } +#[test] +fn empty_block_commit_preserves_listener_and_set() -> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let first_outpoint = OutPoint::new(txid(72), 0); + let second_outpoint = OutPoint::new(txid(73), 0); + let first_txout = txout(72); + let second_txout = txout(73); + + let mut first = BlockChanges::default(); + first.add(UtxoAdd::new(first_outpoint, first_txout.clone(), true, 32)); + set.commit_block(&first, &txid(502))?; + listener.finish_block(32, 1); + + let mut second = BlockChanges::default(); + second.add(UtxoAdd::new( + second_outpoint, + second_txout.clone(), + false, + 33, + )); + set.commit_block(&second, &txid(503))?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&first_outpoint, &first_txout, 32, true); + expected.finish_block(32, 1); + expected.insert_utxo(&second_outpoint, &second_txout, 33, false); + assert_eq!(listener.snapshot(), expected); + + set.commit_block(&BlockChanges::default(), &txid(504))?; + + assert_eq!(set.len(), 2); + assert_eq!(set.get(&first_outpoint), Some(first_txout)); + assert_eq!(set.get(&second_outpoint), Some(second_txout)); + assert_eq!(listener.snapshot(), expected); + Ok(()) +} + #[test] fn undo_block_reverses_unfinished_listener_deltas() -> Result<(), Box> { let listener = CoinStatsListener::new(CoinStats::new()); From b07e923af3aea3dac2a30ab911ba6a5f5bca41a6 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 16:30:56 +0000 Subject: [PATCH 20/37] test(node): make G1 header parity fail closed Replace the empty ignored G1 scaffold with a HEAD-bound evidence gate. The gate now requires declared mainnet header samples from bitcoin-rs and bitcoind, verifies complete 0..=tip coverage, rejects malformed or duplicate hashes, and compares the maps byte-for-byte. This is an offline evidence validator only; it does not claim live tip freshness or implement a headers-only runtime mode. Op: correct Restores: spec:G1 header parity gate must not pass without bound evidence --- .../tests/gates/g01_headers_only_sync.rs | 302 +++++++++++++++++- 1 file changed, 294 insertions(+), 8 deletions(-) diff --git a/bin/bitcoin-rs/tests/gates/g01_headers_only_sync.rs b/bin/bitcoin-rs/tests/gates/g01_headers_only_sync.rs index 659032a..50e9992 100644 --- a/bin/bitcoin-rs/tests/gates/g01_headers_only_sync.rs +++ b/bin/bitcoin-rs/tests/gates/g01_headers_only_sync.rs @@ -1,14 +1,300 @@ //! G1 — Headers-only sync parity. -//! **G1 — Headers-only sync parity.** `bitcoin-rs --headers-only mainnet` → header chain hash matches `bitcoind`'s `getblockhash` for every height 0..tip. +//! **G1 — Headers-only sync parity.** Externally collected bitcoin-rs mainnet active-header-chain hashes match `bitcoind`'s `getblockhash` for every height `0..=tip`. +//! +//! This ignored gate does not run live mainnet header sync itself. It verifies +//! externally collected bitcoin-rs and Bitcoin Core header hash evidence, binds +//! it to the current clean git `HEAD`, and fails closed when that evidence +//! contract is missing, malformed, incomplete, or mismatched. -#![allow(clippy::let_unit_value)] +use std::{collections::BTreeMap, env, fs, process::Command}; -/// Gate G1 manual run instructions: set `BITCOIND_RPC_URL` and -/// `BITCOIND_RPC_COOKIE`, then run -/// `cargo test -p bitcoin-rs --test g01_headers_only_sync -- --ignored --nocapture`. -/// The gate compares every headers-only mainnet height against bitcoind. +const EVIDENCE_HELP: &str = "required G1 evidence env: \ +G1_COMMIT_SHA=, \ +G1_MEASUREMENT_TARGET=mainnet-headers, \ +G1_REFERENCE_IMPL=bitcoind, \ +G1_HASH_TYPE=blockhash, \ +G1_TIP_HEIGHT=, \ +G1_BITCOIN_RS_HEADER_HASHES= \ + or G1_BITCOIN_RS_HEADER_HASHES_FILE=, \ +G1_BITCOIND_BLOCK_HASHES= \ + or G1_BITCOIND_BLOCK_HASHES_FILE="; + +type Samples = BTreeMap; + +struct G1Evidence { + commit_sha: String, + tip_height: u32, + bitcoin_rs_samples: Samples, + bitcoind_samples: Samples, +} + +/// Gate G1 manual run instructions: run +/// `cargo test -p bitcoin-rs --test g01_headers_only_sync -- --ignored --nocapture` +/// with externally collected mainnet active-header-chain hashes from bitcoin-rs +/// and `bitcoin-cli getblockhash ` for every height `0..=tip`. #[test] -#[ignore = "requires live bitcoind mainnet RPC for cross-check"] +#[ignore = "requires externally collected bitcoin-rs and bitcoind mainnet header evidence"] fn headers_only_sync_parity() { - // Compare bitcoin-rs headers-only mainnet block hashes to bitcoind getblockhash for 0..tip. + let evidence = G1Evidence::from_env(); + evidence.assert_samples(); + evidence.report(); +} + +impl G1Evidence { + fn from_env() -> Self { + let commit_sha = required_commit_sha(); + require_literal("G1_MEASUREMENT_TARGET", "mainnet-headers"); + require_literal("G1_REFERENCE_IMPL", "bitcoind"); + require_literal("G1_HASH_TYPE", "blockhash"); + let tip_height = positive_u32("G1_TIP_HEIGHT"); + let bitcoin_rs_samples = samples_from_evidence( + "G1_BITCOIN_RS_HEADER_HASHES", + "G1_BITCOIN_RS_HEADER_HASHES_FILE", + ); + let bitcoind_samples = + samples_from_evidence("G1_BITCOIND_BLOCK_HASHES", "G1_BITCOIND_BLOCK_HASHES_FILE"); + Self { + commit_sha, + tip_height, + bitcoin_rs_samples, + bitcoind_samples, + } + } + + fn assert_samples(&self) { + assert_complete_heights("G1 bitcoin-rs", &self.bitcoin_rs_samples, self.tip_height); + assert_complete_heights("G1 bitcoind", &self.bitcoind_samples, self.tip_height); + assert_eq!( + self.bitcoin_rs_samples, self.bitcoind_samples, + "G1 header parity failed: bitcoin-rs active-header-chain hashes differ from bitcoind getblockhash results", + ); + } + + fn report(&self) { + let commit_sha = &self.commit_sha; + let tip_height = self.tip_height; + let sample_count = self.bitcoin_rs_samples.len(); + println!("G1 header evidence accepted for current git HEAD {commit_sha}"); + println!("tip_height={tip_height}"); + println!("sample_count={sample_count}"); + } +} + +fn assert_complete_heights(label: &str, samples: &Samples, tip_height: u32) { + for height in 0..=tip_height { + assert!( + samples.contains_key(&height), + "{label} samples missing height {height}; expected every height 0..={tip_height}", + ); + } + assert_eq!( + samples.len(), + usize::try_from(tip_height) + .ok() + .and_then(|height| height.checked_add(1)) + .unwrap_or(usize::MAX), + "{label} samples must contain exactly every height 0..={tip_height}", + ); +} + +fn samples_from_evidence(env_name: &str, file_env_name: &str) -> Samples { + let raw = evidence_string(env_name, file_env_name); + let mut samples = Samples::new(); + for entry in raw.split(|ch: char| ch == ',' || ch.is_whitespace()) { + let entry = entry.trim(); + if entry.is_empty() { + continue; + } + let Some((height_raw, hash_raw)) = entry.split_once(':') else { + panic!( + "{env_name} sample {entry:?} must use height:64-lowerhex format; {EVIDENCE_HELP}" + ); + }; + let height = parse_u32(env_name, height_raw); + let hash = hash_raw.trim(); + assert!( + is_lower_hex_hash(hash), + "{env_name} sample at height {height} must be a 64-character lowercase hex block hash, got {hash:?}", + ); + assert!( + samples.insert(height, hash.to_owned()).is_none(), + "{env_name} contains duplicate height {height}", + ); + } + assert!( + !samples.is_empty(), + "{env_name} must contain at least one sample; {EVIDENCE_HELP}", + ); + samples +} + +fn evidence_string(env_name: &str, file_env_name: &str) -> String { + match env::var(env_name) { + Ok(value) if !value.trim().is_empty() => return value, + Ok(_) => panic!("{env_name} must not be empty; {EVIDENCE_HELP}"), + Err(env::VarError::NotUnicode(_)) => { + panic!("{env_name} must be valid UTF-8; {EVIDENCE_HELP}") + } + Err(env::VarError::NotPresent) => {} + } + + let path = match env::var(file_env_name) { + Ok(value) if !value.trim().is_empty() => value, + Ok(_) => panic!("{file_env_name} must not be empty; {EVIDENCE_HELP}"), + Err(env::VarError::NotPresent) => { + panic!("missing {env_name} or {file_env_name}; {EVIDENCE_HELP}") + } + Err(env::VarError::NotUnicode(_)) => { + panic!("{file_env_name} must be valid UTF-8; {EVIDENCE_HELP}") + } + }; + match fs::read_to_string(&path) { + Ok(raw) => raw, + Err(error) => panic!("failed to read {file_env_name} path {path:?}: {error}"), + } +} + +fn required_env(name: &str) -> String { + match env::var(name) { + Ok(value) if !value.trim().is_empty() => value, + Ok(_) => panic!("{name} must not be empty; {EVIDENCE_HELP}"), + Err(env::VarError::NotPresent) => panic!("missing {name}; {EVIDENCE_HELP}"), + Err(env::VarError::NotUnicode(_)) => panic!("{name} must be valid UTF-8; {EVIDENCE_HELP}"), + } +} + +fn required_commit_sha() -> String { + let value = required_env("G1_COMMIT_SHA"); + assert!( + is_lower_hex_sha(&value), + "G1_COMMIT_SHA must be a 40-character lowercase hex commit sha, got {value:?}", + ); + let current_head = current_git_head(); + assert_eq!( + value, current_head, + "G1_COMMIT_SHA must match current git HEAD; evidence {value}, current HEAD {current_head}", + ); + assert_clean_tracked_tree(); + value +} + +fn current_git_head() -> String { + let output = match Command::new("git") + .args([ + "-C", + env!("CARGO_MANIFEST_DIR"), + "rev-parse", + "--verify", + "HEAD", + ]) + .output() + { + Ok(output) => output, + Err(error) => panic!("failed to run git rev-parse for G1_COMMIT_SHA binding: {error}"), + }; + assert!( + output.status.success(), + "git rev-parse HEAD failed while validating G1_COMMIT_SHA: {}", + String::from_utf8_lossy(&output.stderr), + ); + let stdout = match String::from_utf8(output.stdout) { + Ok(stdout) => stdout, + Err(error) => panic!("git rev-parse HEAD did not return UTF-8: {error}"), + }; + let head = stdout.trim().to_owned(); + assert!( + is_lower_hex_sha(&head), + "git rev-parse HEAD returned invalid sha {head:?}", + ); + head +} + +fn assert_clean_tracked_tree() { + let repo_root = current_git_root(); + let output = match Command::new("git") + .args([ + "-C", + repo_root.as_str(), + "status", + "--porcelain=v1", + "--untracked-files=no", + ]) + .output() + { + Ok(output) => output, + Err(error) => panic!("failed to run git status for G1 evidence binding: {error}"), + }; + assert!( + output.status.success(), + "git status failed while validating G1 evidence binding: {}", + String::from_utf8_lossy(&output.stderr), + ); + let status = match String::from_utf8(output.stdout) { + Ok(status) => status, + Err(error) => panic!("git status did not return UTF-8: {error}"), + }; + assert!( + status.trim().is_empty(), + "G1 evidence requires a clean tracked git tree for current HEAD; dirty entries:\n{status}", + ); +} + +fn current_git_root() -> String { + let output = match Command::new("git") + .args([ + "-C", + env!("CARGO_MANIFEST_DIR"), + "rev-parse", + "--show-toplevel", + ]) + .output() + { + Ok(output) => output, + Err(error) => panic!("failed to run git rev-parse --show-toplevel: {error}"), + }; + assert!( + output.status.success(), + "git rev-parse --show-toplevel failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + match String::from_utf8(output.stdout) { + Ok(stdout) => stdout.trim().to_owned(), + Err(error) => panic!("git rev-parse --show-toplevel did not return UTF-8: {error}"), + } +} + +fn require_literal(name: &str, expected: &str) { + let value = required_env(name); + assert_eq!( + value, expected, + "{name} must be {expected:?} for G1 evidence, got {value:?}", + ); +} + +fn positive_u32(name: &str) -> u32 { + let raw = required_env(name); + let value = parse_u32(name, &raw); + assert_ne!(value, 0, "{name} must be positive"); + value +} + +fn parse_u32(name: &str, raw: &str) -> u32 { + match raw.trim().parse::() { + Ok(value) => value, + Err(error) => panic!("{name} must contain a u32 integer, got {raw:?}: {error}"), + } +} + +fn is_lower_hex_sha(value: &str) -> bool { + value.len() == 40 + && value + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) +} + +fn is_lower_hex_hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) } From d101f573e957e42784b3d64502abab3333b1264f Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 18:31:20 +0000 Subject: [PATCH 21/37] feat(node): add headers-only runtime mode Add opt-in headers-only runtime plumbing through config, CLI, env, and production BlockSync construction. In headers-only mode BlockSync anchors genesis as a header, advances by header tip height, sends headers requests without body getdata, and drains inbound block bodies without buffering or applying them. RPC height lookup now resolves the published active header tree so getblockhash works without block records and stale records cannot override active headers. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-rpc --test handler_smoke getblockhash_ -- --nocapture - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node headers_only_tick -- --nocapture - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --test config_layered headers_only_defaults_false_and_layers_enable -- --exact --nocapture - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --features redb,bitcoinconsensus --test sync_smoke -- --nocapture - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-rpc --test handler_smoke -- --nocapture - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast Op: extend --- crates/node/src/config.rs | 14 ++ crates/node/src/state.rs | 1 + crates/node/src/sync.rs | 251 ++++++++++++++++++++++++++-- crates/node/tests/config_layered.rs | 35 ++++ crates/node/tests/sync_smoke.rs | 5 + crates/rpc/src/context.rs | 6 + crates/rpc/tests/handler_smoke.rs | 77 ++++++++- 7 files changed, 378 insertions(+), 11 deletions(-) diff --git a/crates/node/src/config.rs b/crates/node/src/config.rs index dec7e0d..7738145 100644 --- a/crates/node/src/config.rs +++ b/crates/node/src/config.rs @@ -112,6 +112,8 @@ pub struct Config { pub txindex: bool, /// Whether the compact block filter index is enabled. pub blockfilterindex: bool, + /// Whether P2P sync stops at headers and drops block bodies from the sync channel. + pub headers_only: bool, /// Database cache target in MiB. pub dbcache_mb: u64, /// Tracing filter level used when `RUST_LOG` is unset. @@ -158,6 +160,7 @@ impl fmt::Debug for Config { .field("utreexo_mode", &self.utreexo_mode) .field("txindex", &self.txindex) .field("blockfilterindex", &self.blockfilterindex) + .field("headers_only", &self.headers_only) .field("dbcache_mb", &self.dbcache_mb) .field("log_level", &self.log_level) .field("metrics_bind", &self.metrics_bind) @@ -199,6 +202,7 @@ impl Config { utreexo_mode: false, txindex: false, blockfilterindex: false, + headers_only: false, dbcache_mb: DEFAULT_DBCACHE_MB, log_level: DEFAULT_LOG_LEVEL.to_owned(), metrics_bind: None, @@ -408,6 +412,9 @@ impl Config { if let Some(blockfilterindex) = layer.blockfilterindex { self.blockfilterindex = blockfilterindex; } + if let Some(headers_only) = layer.headers_only { + self.headers_only = headers_only; + } if let Some(dbcache_mb) = layer.dbcache_mb { self.dbcache_mb = dbcache_mb; } @@ -496,6 +503,12 @@ pub(crate) struct ConfigLayer { pub(crate) txindex: Option, #[arg(long)] pub(crate) blockfilterindex: Option, + #[arg( + long = "headers-only", + num_args = 0..=1, + default_missing_value = "true" + )] + pub(crate) headers_only: Option, #[arg(long = "dbcache-mb")] pub(crate) dbcache_mb: Option, #[arg(long = "log-level")] @@ -561,6 +574,7 @@ impl ConfigLayer { "BITCOIN_RS_UTREEXO_MODE" => layer.utreexo_mode = Some(parse_bool(value)?), "BITCOIN_RS_TXINDEX" => layer.txindex = Some(parse_bool(value)?), "BITCOIN_RS_BLOCKFILTERINDEX" => layer.blockfilterindex = Some(parse_bool(value)?), + "BITCOIN_RS_HEADERS_ONLY" => layer.headers_only = Some(parse_bool(value)?), "BITCOIN_RS_DBCACHE_MB" => layer.dbcache_mb = Some(value.parse()?), "BITCOIN_RS_LOG_LEVEL" => layer.log_level = Some(value.to_owned()), "BITCOIN_RS_METRICS_BIND" => layer.metrics_bind = Some(value.parse()?), diff --git a/crates/node/src/state.rs b/crates/node/src/state.rs index 135ce69..d3758d5 100644 --- a/crates/node/src/state.rs +++ b/crates/node/src/state.rs @@ -728,6 +728,7 @@ impl NodeState { Arc::clone(&peer_outbound), Arc::clone(&inbound_headers_rx), Arc::clone(&inbound_blocks_rx), + config.headers_only, )); let prune_service = if config.prune_target_mb > 0 { Some(storage.prune_service(Arc::clone(&blocks), Arc::clone(&transactions))?) diff --git a/crates/node/src/sync.rs b/crates/node/src/sync.rs index ad7f104..03b79f2 100644 --- a/crates/node/src/sync.rs +++ b/crates/node/src/sync.rs @@ -4,7 +4,7 @@ //! and, when a peer reports a longer chain, sends `getheaders` toward //! that peer. Inbound `headers` batches are drained into the shared //! [`bitcoin_rs_chain::BlockTree`]; inbound full blocks are applied through -//! [`crate::apply::apply_block`]. +//! [`crate::apply::apply_block`] unless headers-only mode is enabled. use alloc::sync::Arc; use alloc::vec::Vec; @@ -48,6 +48,7 @@ pub struct BlockSync { inbound_blocks_rx: Arc>>, pending_blocks: Arc>>, received_blocks: Arc>>, + headers_only: bool, } impl BlockSync { @@ -59,6 +60,7 @@ impl BlockSync { peer_outbound: Arc>>>, inbound_headers_rx: Arc>>>, inbound_blocks_rx: Arc>>, + headers_only: bool, ) -> Self { Self { handles, @@ -68,6 +70,7 @@ impl BlockSync { inbound_blocks_rx, pending_blocks: Arc::new(Mutex::new(HashMap::new())), received_blocks: Arc::new(Mutex::new(HashMap::new())), + headers_only, } } @@ -78,18 +81,30 @@ impl BlockSync { self.ensure_genesis_tip(); self.drain_inbound_blocks(); - let applied_height = self - .handles - .applied_tip - .load_full() - .map_or(0, |tip| tip.height); - let Some(target) = self.pick_sync_peer(applied_height) else { - tracing::trace!(applied_height, "block sync: no peer above current height"); + let sync_height = self.sync_height(); + let Some(target) = self.pick_sync_peer(sync_height) else { + tracing::trace!(sync_height, "block sync: no peer above current height"); return; }; - self.send_getdata_for_pending_blocks(target.addr); - self.send_getheaders(target.addr, applied_height, target.start_height); + if !self.headers_only { + self.send_getdata_for_pending_blocks(target.addr); + } + self.send_getheaders(target.addr, sync_height, target.start_height); + } + + fn sync_height(&self) -> u32 { + if self.headers_only { + return self + .handles + .chain_tip + .load_full() + .map_or(0, |tip| tip.height); + } + self.handles + .applied_tip + .load_full() + .map_or(0, |tip| tip.height) } fn drain_inbound_headers(&self) { @@ -122,6 +137,17 @@ impl BlockSync { } fn drain_inbound_blocks(&self) { + if self.headers_only { + let dropped = self.drain_and_drop_inbound_blocks(); + if dropped > 0 { + tracing::debug!( + dropped, + "block sync: dropped inbound blocks in headers-only mode" + ); + } + return; + } + let receiver = self.inbound_blocks_rx.lock(); let mut received = 0_usize; while let Ok(block) = receiver.try_recv() { @@ -146,6 +172,16 @@ impl BlockSync { } } + fn drain_and_drop_inbound_blocks(&self) -> usize { + let receiver = self.inbound_blocks_rx.lock(); + let mut dropped = 0_usize; + while receiver.try_recv().is_ok() { + dropped = dropped.saturating_add(1); + } + self.received_blocks.lock().clear(); + dropped + } + fn buffer_received_block(&self, block: bitcoin::Block) { let now = Instant::now(); let hash = Hash256::from_le_bytes(block.block_hash().as_byte_array()); @@ -361,6 +397,11 @@ impl BlockSync { } fn ensure_genesis_tip(&self) { + if self.headers_only { + self.ensure_genesis_header(); + return; + } + if self.handles.applied_tip.load_full().is_some() { return; } @@ -379,6 +420,24 @@ impl BlockSync { } } } + + fn ensure_genesis_header(&self) { + let genesis_hash = self.handles.network.genesis_block_hash(); + let mut tree = self.handles.block_tree.write(); + if tree.lookup(genesis_hash).is_some() { + return; + } + + let genesis = + bitcoin::blockdata::constants::genesis_block(bitcoin_network(self.handles.network)); + match tree.insert_header( + genesis.header, + bitcoin_rs_chain::node::NodeStatus::HeaderValid, + ) { + Ok(_id) => tracing::debug!(%genesis_hash, "block sync: bootstrapped genesis header"), + Err(error) => tracing::warn!(%error, "block sync: failed to bootstrap genesis header"), + } + } } fn bitcoin_network(network: bitcoin_rs_primitives::Network) -> bitcoin::Network { @@ -470,6 +529,7 @@ mod tests { Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, ); let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); peers.write().push(synthetic_peer(addr, 100)); @@ -541,6 +601,7 @@ mod tests { Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, ); let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); peers.write().push(synthetic_peer(addr, 100)); @@ -622,6 +683,7 @@ mod tests { Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, ); let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); peers.write().push(synthetic_peer(addr, 100)); @@ -734,6 +796,107 @@ mod tests { Ok(()) } + #[test] + fn headers_only_tick_sends_getheaders_without_getdata() -> Result<(), Box> + { + let (sync, peers, peer_outbound, block_tree, applied_tip, _expected) = + sync_with_header_chain_for_mode(3, true)?; + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); + peers.write().push(synthetic_peer(addr, 100)); + let (tx, rx) = unbounded::(); + peer_outbound.write().insert(addr, tx); + + sync.tick(); + + assert_headers_only_no_body_apply(&applied_tip, &block_tree, &sync.handles)?; + assert!(sync.pending_blocks.lock().is_empty()); + let first = rx.try_recv()?; + if !matches!(first, NetworkMessage::GetHeaders(_)) { + return Err(std::io::Error::other("expected headers-only getheaders").into()); + } + match rx.try_recv() { + Ok(NetworkMessage::GetData(_)) => { + Err(std::io::Error::other("headers-only tick sent getdata").into()) + } + Ok(_) => Err(std::io::Error::other("unexpected extra headers-only message").into()), + Err(crossbeam_channel::TryRecvError::Empty) => Ok(()), + Err(crossbeam_channel::TryRecvError::Disconnected) => { + Err(std::io::Error::other("outbound channel disconnected").into()) + } + } + } + + #[test] + fn headers_only_tick_uses_header_tip_height_for_progress() + -> Result<(), Box> { + let (sync, peers, peer_outbound, block_tree, applied_tip, _expected) = + sync_with_header_chain_for_mode(3, true)?; + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); + peers.write().push(synthetic_peer(addr, 3)); + let (tx, rx) = unbounded::(); + peer_outbound.write().insert(addr, tx); + + sync.tick(); + + assert_headers_only_no_body_apply(&applied_tip, &block_tree, &sync.handles)?; + assert!(sync.pending_blocks.lock().is_empty()); + if !matches!(rx.try_recv(), Err(crossbeam_channel::TryRecvError::Empty)) { + return Err( + std::io::Error::other("headers-only tick requested current header height").into(), + ); + } + Ok(()) + } + + #[test] + fn headers_only_tick_drains_inbound_blocks_without_buffering() + -> Result<(), Box> { + let genesis_block = bitcoin::blockdata::constants::genesis_block(bitcoin::Network::Regtest); + let genesis_hash = bitcoin_rs_primitives::Hash256::from_le_bytes( + genesis_block.block_hash().as_byte_array(), + ); + let block_tree = Arc::new(RwLock::new(BlockTree::new())); + let chain_tip = block_tree.read().tip_handle(); + let applied_tip = Arc::new(ArcSwapOption::empty()); + let peers = Arc::new(RwLock::new(Vec::new())); + let peer_outbound = Arc::new(RwLock::new(HashMap::new())); + let (_inbound_headers_tx, inbound_headers_rx_raw) = unbounded::>(); + let inbound_headers_rx = Arc::new(Mutex::new(inbound_headers_rx_raw)); + let (inbound_blocks_tx, inbound_blocks_rx_raw) = unbounded::(); + let inbound_blocks_rx = Arc::new(Mutex::new(inbound_blocks_rx_raw)); + let handles = apply_handles( + Arc::clone(&chain_tip), + Arc::clone(&applied_tip), + Arc::clone(&block_tree), + ); + let sync = BlockSync::new( + handles, + Arc::clone(&peers), + Arc::clone(&peer_outbound), + inbound_headers_rx, + inbound_blocks_rx, + true, + ); + inbound_blocks_tx.send(genesis_block.clone())?; + sync.received_blocks.lock().insert( + genesis_hash, + super::ReceivedBlock { + block: genesis_block, + received_at: Instant::now(), + }, + ); + + sync.tick(); + + assert_headers_only_no_body_apply(&applied_tip, &block_tree, &sync.handles)?; + assert!(sync.received_blocks.lock().is_empty()); + assert!(matches!( + sync.inbound_blocks_rx.lock().try_recv(), + Err(crossbeam_channel::TryRecvError::Empty), + )); + Ok(()) + } + #[test] fn drain_inbound_blocks_prunes_stale_received_blocks_without_new_arrivals() -> Result<(), Box> { @@ -800,6 +963,59 @@ mod tests { Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, + ); + + Ok(( + sync, + peers, + peer_outbound, + block_tree, + applied_tip, + expected, + )) + } + + fn sync_with_header_chain_for_mode( + height: u32, + headers_only: bool, + ) -> Result> { + let mut tree = BlockTree::new(); + let genesis = genesis_header(); + let genesis_id = tree.insert_node(None, genesis, NodeStatus::HeaderValid)?; + let mut tip_id = genesis_id; + let mut expected = Vec::new(); + + for height in 1_u32..=height { + let parent_hash = BlockHash::from_byte_array(tree.node(tip_id)?.hash.to_le_bytes()); + let header = test_header(parent_hash, height); + tip_id = tree.insert_node(Some(tip_id), header, NodeStatus::HeaderValid)?; + expected.push(BlockHash::from_byte_array( + tree.node(tip_id)?.hash.to_le_bytes(), + )); + } + + let chain_tip = tree.tip_handle(); + let block_tree = Arc::new(RwLock::new(tree)); + let applied_tip = Arc::new(ArcSwapOption::empty()); + let peers = Arc::new(RwLock::new(Vec::new())); + let peer_outbound = Arc::new(RwLock::new(HashMap::new())); + let (_inbound_headers_tx, inbound_headers_rx_raw) = unbounded::>(); + let inbound_headers_rx = Arc::new(Mutex::new(inbound_headers_rx_raw)); + let (_inbound_blocks_tx, inbound_blocks_rx_raw) = unbounded::(); + let inbound_blocks_rx = Arc::new(Mutex::new(inbound_blocks_rx_raw)); + let handles = apply_handles( + Arc::clone(&chain_tip), + Arc::clone(&applied_tip), + Arc::clone(&block_tree), + ); + let sync = BlockSync::new( + handles, + Arc::clone(&peers), + Arc::clone(&peer_outbound), + inbound_headers_rx, + inbound_blocks_rx, + headers_only, ); Ok(( @@ -932,6 +1148,21 @@ mod tests { Ok(()) } + fn assert_headers_only_no_body_apply( + applied_tip: &Arc>, + block_tree: &Arc>, + handles: &ApplyHandles, + ) -> Result<(), Box> { + let genesis_hash = Network::Regtest.genesis_block_hash(); + if applied_tip.load_full().is_some() { + return Err(std::io::Error::other("headers-only mode applied a block body").into()); + } + assert_eq!(block_tree.read().height_of_hash(genesis_hash), Some(0)); + assert!(handles.blocks.read().is_empty()); + assert_eq!(handles.utxo.len(), 0); + Ok(()) + } + fn synthetic_peer(addr: SocketAddr, start_height: i32) -> PeerInfo { PeerInfo { addr, diff --git a/crates/node/tests/config_layered.rs b/crates/node/tests/config_layered.rs index 1227734..fd83f97 100644 --- a/crates/node/tests/config_layered.rs +++ b/crates/node/tests/config_layered.rs @@ -96,6 +96,41 @@ fn cli_can_override_socket_and_vector_fields() -> Result<()> { Ok(()) } +#[test] +fn headers_only_defaults_false_and_layers_enable() -> Result<()> { + let temp = tempfile::tempdir()?; + let toml_path = temp.path().join("node.toml"); + fs::write(&toml_path, "headers_only = true\n")?; + + let default_config = Config::default_for_network(Network::Regtest); + assert!(!default_config.headers_only); + + let toml_config = Config::from_layered_sources( + Some(&toml_path), + None, + core::iter::empty::(), + ["bitcoin-rs-node"], + )?; + assert!(toml_config.headers_only); + + let env_config = Config::from_layered_sources( + None, + None, + [("BITCOIN_RS_HEADERS_ONLY", "true")], + ["bitcoin-rs-node"], + )?; + assert!(env_config.headers_only); + + let cli_config = Config::from_layered_sources( + None, + None, + core::iter::empty::(), + ["bitcoin-rs-node", "--headers-only"], + )?; + assert!(cli_config.headers_only); + Ok(()) +} + #[test] fn zmq_layers_parse_precedence_and_publication_order() -> Result<()> { let temp = tempfile::tempdir()?; diff --git a/crates/node/tests/sync_smoke.rs b/crates/node/tests/sync_smoke.rs index 82a9bea..4b1afe6 100644 --- a/crates/node/tests/sync_smoke.rs +++ b/crates/node/tests/sync_smoke.rs @@ -52,6 +52,7 @@ fn tick_sends_getheaders_to_best_peer_above_our_height() -> Result<(), Box Result<(), Box Result<(), Box Option { + let tree = self.block_tree.read(); + if tree.tip().is_some() { + return tree.active_node_at_height(height).map(|node| node.hash); + } + drop(tree); + self.blocks .read() .iter() diff --git a/crates/rpc/tests/handler_smoke.rs b/crates/rpc/tests/handler_smoke.rs index 5f29f0d..3529141 100644 --- a/crates/rpc/tests/handler_smoke.rs +++ b/crates/rpc/tests/handler_smoke.rs @@ -8,7 +8,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use bitcoin::consensus::encode::serialize_hex; use bitcoin::hashes::Hash as _; use bitcoin::{Amount, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness}; -use bitcoin_rs_chain::{ChainWork, NodeId, TipSnapshot}; +use bitcoin_rs_chain::{ChainWork, NodeId, NodeStatus, TipSnapshot}; use bitcoin_rs_filters::{FilterIndexError, FilterIndexLike}; use bitcoin_rs_index::{BlockSource, IndexError, IndexRowCounts, IndexerLike}; use bitcoin_rs_mempool::MempoolEntry; @@ -250,6 +250,55 @@ fn gettxoutsetinfo_hash_type_modes_match_core_shapes() -> Result<(), Box Result<(), Box> { + let ctx = active_regtest_header_context()?; + let genesis_hash = bitcoin_rs_primitives::Network::Regtest.genesis_block_hash(); + assert!(ctx.blocks.read().is_empty()); + let handler = Handler::new(Arc::new(ctx)); + + let result = handler.dispatch("getblockhash", &json!([0]))?; + + assert_eq!(result.as_str(), Some(genesis_hash.to_string_be().as_str())); + Ok(()) +} + +#[test] +fn getblockhash_prefers_active_header_over_stale_block_record() +-> Result<(), Box> { + let ctx = active_regtest_header_context()?; + let genesis_hash = bitcoin_rs_primitives::Network::Regtest.genesis_block_hash(); + ctx.add_block(BlockRecord::synthetic( + 0, + Hash256::from_le_bytes(&[0x99; 32]), + )); + let handler = Handler::new(Arc::new(ctx)); + + let result = handler.dispatch("getblockhash", &json!([0]))?; + + assert_eq!(result.as_str(), Some(genesis_hash.to_string_be().as_str())); + Ok(()) +} + +#[test] +fn getblockhash_rejects_stale_block_record_above_active_tip() +-> Result<(), Box> { + let ctx = active_regtest_header_context()?; + ctx.add_block(BlockRecord::synthetic( + 2, + Hash256::from_le_bytes(&[0x77; 32]), + )); + let handler = Handler::new(Arc::new(ctx)); + + let error = handler + .dispatch("getblockhash", &json!([2])) + .expect_err("stale block record above active tip unexpectedly resolved"); + + assert_eq!(error.code(), RpcError::CORE_NOT_FOUND); + Ok(()) +} + #[test] fn getblockfilter_reads_filter_index() -> Result<(), Box> { let block_hash = Hash256::from_le_bytes(&[9_u8; 32]); @@ -768,6 +817,32 @@ fn context_with_peers(peers: Arc>>) -> Arc { Arc::new(ctx) } +fn active_regtest_header_context() -> Result> { + let ctx = Context::new(); + let genesis = bitcoin::blockdata::constants::genesis_block(bitcoin::Network::Regtest); + let child_header = bitcoin::block::Header { + version: bitcoin::block::Version::ONE, + prev_blockhash: genesis.block_hash(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: genesis.header.time.saturating_add(1), + bits: genesis.header.bits, + nonce: genesis.header.nonce.saturating_add(1), + }; + let child_hash = Hash256::from_le_bytes(child_header.block_hash().as_byte_array()); + let child_id = { + let mut tree = ctx.block_tree.write(); + tree.insert_header(genesis.header, NodeStatus::HeaderValid)?; + tree.insert_header(child_header, NodeStatus::HeaderValid)? + }; + ctx.set_chain_tip(TipSnapshot { + tip_id: child_id, + height: 1, + chainwork: ChainWork::ZERO, + hash: child_hash, + }); + Ok(ctx) +} + fn tx(label: u8, script_pubkey: ScriptBuf) -> Transaction { Transaction { version: bitcoin::transaction::Version::TWO, From a4cc4a2bc12fcfd2d1e23bc75992ea45d9bfa885 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 19:20:19 +0000 Subject: [PATCH 22/37] perf(coinstats): add listener attribution probes Add benchmark-only controls for sharded counters, encoding, and MuHash arithmetic to the synthetic UTXO commit CoinStats benchmark. Production code is unchanged; this is an attribution probe only and makes no runtime speed claim. Evidence:\n- cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check\n- cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --benches -- -D warnings\n- cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats -- --sample-size 10 --measurement-time 1 --warm-up-time 1 Caveat: benchmark-only attribution probe, not production behavior or proof of node/runtime speed. Op: extend --- .../benches/utxo_commit_coinstats.rs | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs index 1f772ff..25f8462 100644 --- a/crates/coinstats/benches/utxo_commit_coinstats.rs +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -25,6 +25,9 @@ enum ListenerKind { None, Noop, Accounting, + ShardedCounters, + ShardedEncodeOnly, + ShardedMuhashOnly, ShardedCoinStats, CoinStats, } @@ -48,6 +51,24 @@ struct AccountingListener { stats: RwLock, } +#[derive(Default)] +struct EncodeOnlyStats { + total_bytes: u64, + operations: u64, +} + +struct ShardedCountersListener { + shards: Vec>, +} + +struct ShardedEncodeOnlyListener { + shards: Vec>, +} + +struct ShardedMuhashOnlyListener { + shards: Vec>, +} + struct ShardedCoinStatsListener { shards: Vec>, } @@ -71,6 +92,58 @@ impl AccountingListener { } } +impl ShardedCountersListener { + fn new() -> Self { + let shards = (0..UtxoKey::SHARD_COUNT) + .map(|_| RwLock::new(AccountingStats::default())) + .collect(); + Self { shards } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.shards[index] + } +} + +impl ShardedEncodeOnlyListener { + fn new() -> Self { + let shards = (0..UtxoKey::SHARD_COUNT) + .map(|_| RwLock::new(EncodeOnlyStats::default())) + .collect(); + Self { shards } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.shards[index] + } + + fn encode(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let bytes = bench_coin_hash_bytes(op, txout, height, coinbase); + let byte_count = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + black_box(bytes.as_slice()); + + let mut stats = self.shard(op).write(); + stats.total_bytes = stats.total_bytes.saturating_add(byte_count); + stats.operations = stats.operations.saturating_add(1); + } +} + +impl ShardedMuhashOnlyListener { + fn new() -> Self { + let shards = (0..UtxoKey::SHARD_COUNT) + .map(|_| RwLock::new(MuHash3072::new())) + .collect(); + Self { shards } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.shards[index] + } +} + impl ShardedCoinStatsListener { fn new() -> Self { let shards = (0..UtxoKey::SHARD_COUNT) @@ -101,6 +174,52 @@ impl UtxoChangeListener for AccountingListener { } } +impl UtxoChangeListener for ShardedCountersListener { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, _height: u32, _coinbase: bool) { + let mut stats = self.shard(op).write(); + stats.total_amount = stats.total_amount.saturating_add(txout.value.to_sat()); + stats.bogo_size = stats.bogo_size.saturating_add(simple_bogo_size(txout)); + stats.utxo_count = stats.utxo_count.saturating_add(1); + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, _height: u32) { + let mut stats = self.shard(op).write(); + stats.total_amount = stats.total_amount.saturating_sub(txout.value.to_sat()); + stats.bogo_size = stats.bogo_size.saturating_sub(simple_bogo_size(txout)); + stats.utxo_count = stats.utxo_count.saturating_sub(1); + } +} + +impl UtxoChangeListener for ShardedEncodeOnlyListener { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.encode(op, txout, height, coinbase); + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { + self.encode(op, txout, height, false); + } + + fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.encode(op, txout, height, coinbase); + } +} + +impl UtxoChangeListener for ShardedMuhashOnlyListener { + fn on_insert(&self, op: &OutPoint, _txout: &TxOut, _height: u32, _coinbase: bool) { + self.shard(op).write().insert(op.as_bytes()); + } + + fn on_remove(&self, op: &OutPoint, _txout: &TxOut, _height: u32) { + self.shard(op).write().remove(op.as_bytes()); + } + + fn on_remove_coin(&self, op: &OutPoint, _txout: &TxOut, _height: u32, _coinbase: bool) { + // This isolates MuHash arithmetic in the commit callback shape; it is not + // measuring CoinStats' exact coin encoding semantics. + self.shard(op).write().remove(op.as_bytes()); + } +} + impl UtxoChangeListener for ShardedCoinStatsListener { fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { self.shard(op) @@ -171,6 +290,15 @@ fn synthetic_case(seed: u64, listener_kind: ListenerKind) -> (UtxoSet, BlockChan ListenerKind::None => {} ListenerKind::Noop => set.set_listener(Box::new(NoopListener)), ListenerKind::Accounting => set.set_listener(Box::new(AccountingListener::new())), + ListenerKind::ShardedCounters => { + set.set_listener(Box::new(ShardedCountersListener::new())); + } + ListenerKind::ShardedEncodeOnly => { + set.set_listener(Box::new(ShardedEncodeOnlyListener::new())); + } + ListenerKind::ShardedMuhashOnly => { + set.set_listener(Box::new(ShardedMuhashOnlyListener::new())); + } ListenerKind::ShardedCoinStats => { set.set_listener(Box::new(ShardedCoinStatsListener::new())); } @@ -359,6 +487,24 @@ fn utxo_commit_coinstats(c: &mut Criterion) { ListenerKind::Accounting, &block_hash, ); + bench_commit_case( + &mut group, + "sharded_counter_listener", + ListenerKind::ShardedCounters, + &block_hash, + ); + bench_commit_case( + &mut group, + "sharded_encode_only_listener", + ListenerKind::ShardedEncodeOnly, + &block_hash, + ); + bench_commit_case( + &mut group, + "sharded_muhash_only_listener", + ListenerKind::ShardedMuhashOnly, + &block_hash, + ); bench_commit_case( &mut group, "sharded_coinstats_listener", From fd0e3153f794d88863046bda05df568cda79bad2 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 19:32:11 +0000 Subject: [PATCH 23/37] perf(coinstats): add direct MuHash outpoint probe Add a benchmark-only direct MuHash probe using the same OutPoint::as_bytes payload as the callback-shaped sharded MuHash listener benchmark. This reconciles attribution evidence before any production optimization and makes no runtime speed claim. Evidence:\n- cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check\n- cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --benches -- -D warnings\n- cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-coinstats --bench utxo_commit_coinstats direct_muhash_outpoint_bytes -- --sample-size 10 --measurement-time 1 --warm-up-time 1 Caveat: benchmark-only attribution probe, not production behavior or proof of node/runtime speed. Task: #233 Op: extend --- .../benches/utxo_commit_coinstats.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs index 25f8462..76537ae 100644 --- a/crates/coinstats/benches/utxo_commit_coinstats.rs +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -456,6 +456,26 @@ fn bench_direct_encode_only(group: &mut BenchmarkGroup<'_, WallTime>) { }); } +fn direct_muhash_outpoint_bytes(group: &mut BenchmarkGroup<'_, WallTime>) { + group.bench_function("direct_muhash_outpoint_bytes", |b| { + b.iter_batched( + || synthetic_direct_case(CASE_SEED), + |case| { + let DirectCase { spends, adds, .. } = case; + let mut muhash = MuHash3072::new(); + for spend in &spends { + muhash.remove(spend.outpoint.as_bytes()); + } + for add in &adds { + muhash.insert(add.outpoint.as_bytes()); + } + black_box(muhash); + }, + BatchSize::SmallInput, + ); + }); +} + fn bench_direct_muhash_preencoded(group: &mut BenchmarkGroup<'_, WallTime>) { group.bench_function("direct_coinstats_muhash_preencoded", |b| { b.iter_batched( @@ -519,6 +539,7 @@ fn utxo_commit_coinstats(c: &mut Criterion) { ); bench_direct_coinstats(&mut group); bench_direct_encode_only(&mut group); + direct_muhash_outpoint_bytes(&mut group); bench_direct_muhash_preencoded(&mut group); group.finish(); From d7d621e8603075f6429ec50d993d3a75262a3c36 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 20:14:44 +0000 Subject: [PATCH 24/37] perf(node): attribute script verification phase Add bounded apply-local attribution for script verification outliers. The new event keeps the existing script_verify_us contract intact and decomposes it into compute_verify_flags, verify_block_transactions, cumulative transaction verifier time, and residual block-local overhead only when a block crosses the outlier threshold. Evidence: - cargo fmt --all -- --check - cargo test -p bitcoin-rs-node verify_block_transactions --no-default-features --features rocksdb,bitcoinconsensus -- --nocapture - cargo clippy -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus - cargo test --workspace --no-fail-fast Op: correct Restores: spec:evidence-first attribution before script optimization --- crates/node/src/apply.rs | 84 ++++++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 12 deletions(-) diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index ff9f765..890f5b2 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -24,6 +24,7 @@ const BIP68_DISABLE_FLAG: u32 = 0x8000_0000; const BIP68_TYPE_FLAG: u32 = 0x0040_0000; const BIP68_MASK: u32 = 0x0000_ffff; const BIP68_TIME_GRANULARITY_SECONDS: u32 = 512; +const SCRIPT_VERIFY_OUTLIER_BLOCK_US: u128 = 100_000; pub(crate) trait PruneBodyStore: Send + Sync { fn persist_block_body( @@ -226,13 +227,33 @@ pub fn apply_block( bip113_result?; let script_verify_started = quanta::Instant::now(); + let verify_flags_started = quanta::Instant::now(); let verify_flags = compute_verify_flags(handles.network, height, softfork_state); + let verify_flags_dur = verify_flags_started.elapsed(); + let block_transactions_started = quanta::Instant::now(); let script_verify_result = verify_block_transactions(handles, block, height, locktime_cutoff, verify_flags); + let block_transactions_dur = block_transactions_started.elapsed(); let script_verify_dur = script_verify_started.elapsed(); metrics::histogram!("node.apply_block.script_verify_seconds") .record(script_verify_dur.as_secs_f64()); - script_verify_result?; + let script_verify_profile = script_verify_result?; + if script_verify_dur.as_micros() >= SCRIPT_VERIFY_OUTLIER_BLOCK_US { + let verify_block_transactions_us = block_transactions_dur.as_micros(); + tracing::info!( + height, + %block_hash, + tx_count = block.txdata.len(), + non_coinbase_tx_count = script_verify_profile.non_coinbase_tx_count, + total_input_count = script_verify_profile.total_input_count, + script_verify_us = script_verify_dur.as_micros(), + compute_verify_flags_us = verify_flags_dur.as_micros(), + verify_block_transactions_us, + tx_verify_call_us = script_verify_profile.tx_verify_call_us, + tx_nonverify_us = script_verify_profile.nonverify_us(verify_block_transactions_us), + "apply_block: script verify attribution" + ); + } let coinbase_maturity_started = quanta::Instant::now(); let coinbase_maturity_result = check_coinbase_maturity(handles, block, height); @@ -523,11 +544,12 @@ fn verify_block_transactions( height: u32, locktime_cutoff: u32, flags: bitcoin_rs_script::VerifyFlags, -) -> core::result::Result<(), ApplyError> { +) -> core::result::Result { // Consensus connects transactions in block order. A later transaction may // spend an output created earlier in the same block. Coinbase outputs enter // this view too, so maturity failures stay in the maturity pass instead of // degrading into bogus missing-prevout script checks. + let mut profile = ScriptVerifyProfile::default(); let mut view = BlockLocalUtxoView::new(Arc::clone(&handles.utxo)); for tx in &block.txdata { if tx.is_coinbase() { @@ -535,17 +557,45 @@ fn verify_block_transactions( view.add_outputs(tx, height)?; continue; } - bitcoin_rs_consensus::verify_tx::verify_transaction_borrowed_with_mtp( - tx, - &view, - height, - locktime_cutoff, - flags, - )?; + profile.observe_non_coinbase(tx); + let tx_verify_started = quanta::Instant::now(); + let tx_verify_result = + bitcoin_rs_consensus::verify_tx::verify_transaction_borrowed_with_mtp( + tx, + &view, + height, + locktime_cutoff, + flags, + ); + let tx_verify_dur = tx_verify_started.elapsed(); + profile.add_tx_verify_call_us(tx_verify_dur.as_micros()); + tx_verify_result?; view.spend_inputs(tx); view.add_outputs(tx, height)?; } - Ok(()) + Ok(profile) +} + +#[derive(Default)] +struct ScriptVerifyProfile { + non_coinbase_tx_count: usize, + total_input_count: usize, + tx_verify_call_us: u128, +} + +impl ScriptVerifyProfile { + fn observe_non_coinbase(&mut self, tx: &bitcoin::Transaction) { + self.non_coinbase_tx_count = self.non_coinbase_tx_count.saturating_add(1); + self.total_input_count = self.total_input_count.saturating_add(tx.input.len()); + } + + fn add_tx_verify_call_us(&mut self, elapsed_us: u128) { + self.tx_verify_call_us = self.tx_verify_call_us.saturating_add(elapsed_us); + } + + fn nonverify_us(&self, verify_block_transactions_us: u128) -> u128 { + verify_block_transactions_us.saturating_sub(self.tx_verify_call_us) + } } struct BlockLocalUtxoView { @@ -1119,7 +1169,15 @@ mod consensus_rule_tests { ); let block = block_with_transactions(vec![funding_tx, same_block_spend]); - verify_block_transactions(&handles, &block, 2, 0, bitcoin_rs_script::VerifyFlags::NONE)?; + let profile = verify_block_transactions( + &handles, + &block, + 2, + 0, + bitcoin_rs_script::VerifyFlags::NONE, + )?; + assert_eq!(profile.non_coinbase_tx_count, 2); + assert_eq!(profile.total_input_count, 2); Ok(()) } @@ -1137,7 +1195,9 @@ mod consensus_rule_tests { 0, bitcoin_rs_script::VerifyFlags::MANDATORY, ) { - Ok(()) => panic!("bad coinbase scriptSig length must fail transaction verification"), + Ok(_profile) => { + panic!("bad coinbase scriptSig length must fail transaction verification") + } Err(error) => error, }; From 5726a38e5610e2448ab724061f288c6ac5f8a25c Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 20:36:31 +0000 Subject: [PATCH 25/37] perf(node): attribute slow script transactions Record the slowest transaction verifier calls inside outlier script verification blocks. The probe stays apply-local, bounded to the existing outlier gate, and emits no metrics labels or consensus-surface changes. Evidence: - cargo fmt --all -- --check - cargo test -p bitcoin-rs-node verify_block_transactions --no-default-features --features rocksdb,bitcoinconsensus -- --nocapture - cargo test -p bitcoin-rs-node script_verify_profile_keeps_bounded_slowest_transactions --no-default-features --features rocksdb,bitcoinconsensus -- --nocapture - cargo clippy -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus - cargo test --workspace --no-fail-fast Op: correct Restores: spec:evidence-first attribution before script optimization --- crates/node/src/apply.rs | 137 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 2 deletions(-) diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index 890f5b2..ebaa497 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -25,6 +25,7 @@ const BIP68_TYPE_FLAG: u32 = 0x0040_0000; const BIP68_MASK: u32 = 0x0000_ffff; const BIP68_TIME_GRANULARITY_SECONDS: u32 = 512; const SCRIPT_VERIFY_OUTLIER_BLOCK_US: u128 = 100_000; +const SCRIPT_VERIFY_PROFILE_TOP_N: usize = 5; pub(crate) trait PruneBodyStore: Send + Sync { fn persist_block_body( @@ -253,6 +254,21 @@ pub fn apply_block( tx_nonverify_us = script_verify_profile.nonverify_us(verify_block_transactions_us), "apply_block: script verify attribution" ); + for (rank_index, tx_profile) in script_verify_profile.slow_txs.iter().enumerate() { + tracing::info!( + height, + %block_hash, + tx_rank = rank_index.saturating_add(1), + tx_index = tx_profile.tx_index, + txid = %tx_profile.txid, + input_count = tx_profile.input_count, + output_count = tx_profile.output_count, + tx_verify_us = tx_profile.tx_verify_us, + block_tx_verify_call_us = script_verify_profile.tx_verify_call_us, + block_script_verify_us = script_verify_dur.as_micros(), + "apply_block: script verify transaction attribution" + ); + } } let coinbase_maturity_started = quanta::Instant::now(); @@ -551,7 +567,7 @@ fn verify_block_transactions( // degrading into bogus missing-prevout script checks. let mut profile = ScriptVerifyProfile::default(); let mut view = BlockLocalUtxoView::new(Arc::clone(&handles.utxo)); - for tx in &block.txdata { + for (tx_index, tx) in block.txdata.iter().enumerate() { if tx.is_coinbase() { bitcoin_rs_consensus::verify_tx::verify_coinbase_script_sig_size(tx)?; view.add_outputs(tx, height)?; @@ -568,8 +584,10 @@ fn verify_block_transactions( flags, ); let tx_verify_dur = tx_verify_started.elapsed(); - profile.add_tx_verify_call_us(tx_verify_dur.as_micros()); + let tx_verify_us = tx_verify_dur.as_micros(); + profile.add_tx_verify_call_us(tx_verify_us); tx_verify_result?; + profile.observe_slow_tx(tx_index, tx, tx_verify_us); view.spend_inputs(tx); view.add_outputs(tx, height)?; } @@ -581,6 +599,15 @@ struct ScriptVerifyProfile { non_coinbase_tx_count: usize, total_input_count: usize, tx_verify_call_us: u128, + slow_txs: Vec, +} + +struct SlowScriptVerifyTxProfile { + tx_index: usize, + txid: Txid, + input_count: usize, + output_count: usize, + tx_verify_us: u128, } impl ScriptVerifyProfile { @@ -589,6 +616,29 @@ impl ScriptVerifyProfile { self.total_input_count = self.total_input_count.saturating_add(tx.input.len()); } + fn observe_slow_tx(&mut self, tx_index: usize, tx: &bitcoin::Transaction, tx_verify_us: u128) { + let insert_index = self + .slow_txs + .iter() + .position(|candidate| candidate.tx_verify_us < tx_verify_us) + .unwrap_or(self.slow_txs.len()); + if insert_index >= SCRIPT_VERIFY_PROFILE_TOP_N { + return; + } + + self.slow_txs.insert( + insert_index, + SlowScriptVerifyTxProfile { + tx_index, + txid: tx.compute_txid(), + input_count: tx.input.len(), + output_count: tx.output.len(), + tx_verify_us, + }, + ); + self.slow_txs.truncate(SCRIPT_VERIFY_PROFILE_TOP_N); + } + fn add_tx_verify_call_us(&mut self, elapsed_us: u128) { self.tx_verify_call_us = self.tx_verify_call_us.saturating_add(elapsed_us); } @@ -1178,9 +1228,92 @@ mod consensus_rule_tests { )?; assert_eq!(profile.non_coinbase_tx_count, 2); assert_eq!(profile.total_input_count, 2); + assert!(profile.slow_txs.len() <= SCRIPT_VERIFY_PROFILE_TOP_N); + assert_eq!(profile.slow_txs.len(), 2); + assert!( + profile + .slow_txs + .iter() + .any(|tx_profile| tx_profile.tx_index == 0) + ); + assert!( + profile + .slow_txs + .iter() + .any(|tx_profile| tx_profile.tx_index == 1) + ); Ok(()) } + #[test] + fn script_verify_profile_keeps_bounded_slowest_transactions() { + let mut profile = ScriptVerifyProfile::default(); + let transactions = [ + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x71; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x72; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x73; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x74; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x75; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x76; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + ]; + + let mut tx_verify_us = 0_u128; + for (tx_index, tx) in transactions.iter().enumerate() { + profile.observe_slow_tx(tx_index, tx, tx_verify_us); + tx_verify_us = tx_verify_us.saturating_add(1); + } + + assert_eq!(profile.slow_txs.len(), SCRIPT_VERIFY_PROFILE_TOP_N); + let indexes: Vec = profile + .slow_txs + .iter() + .map(|tx_profile| tx_profile.tx_index) + .collect(); + assert_eq!(indexes, vec![5, 4, 3, 2, 1]); + } + #[test] fn verify_block_transactions_rejects_bad_coinbase_script_sig() { let mut coinbase = coinbase_transaction(0x63); From ad615da924d4b6ff780618b5a50ea35cff059d8a Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 20:53:42 +0000 Subject: [PATCH 26/37] perf(consensus): attribute verifier internals Expose a profiled borrowed transaction verifier that preserves the existing verifier API and returns coarse timing buckets for node-owned attribution. The node keeps all logging behind the existing outlier and top-transaction gates. Evidence: - cargo fmt --all -- --check - cargo test -p bitcoin-rs-consensus --no-fail-fast - cargo test -p bitcoin-rs-node verify_block_transactions --no-default-features --features rocksdb,bitcoinconsensus -- --nocapture - cargo clippy -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus - cargo test --workspace --no-fail-fast Op: correct Restores: spec:evidence-first attribution before script optimization --- crates/consensus/src/verify_tx.rs | 195 ++++++++++++++++++++++++++---- crates/node/src/apply.rs | 36 +++++- 2 files changed, 202 insertions(+), 29 deletions(-) diff --git a/crates/consensus/src/verify_tx.rs b/crates/consensus/src/verify_tx.rs index d804585..b9c0c43 100644 --- a/crates/consensus/src/verify_tx.rs +++ b/crates/consensus/src/verify_tx.rs @@ -1,4 +1,5 @@ use std::collections::BTreeSet; +use std::time::Instant; use bitcoin_rs_primitives::Tx; use bitcoin_rs_script::{Interpreter, VerifyFlags}; @@ -11,6 +12,39 @@ const SEQUENCE_FINAL: u32 = 0xffff_ffff; const MIN_COINBASE_SCRIPT_SIG_SIZE: usize = 2; const MAX_COINBASE_SCRIPT_SIG_SIZE: usize = 100; +/// Coarse timing profile for borrowed transaction verification. +/// +/// Buckets are accumulated microsecond totals for selected verifier-internal +/// operations. They are intended for node-owned attribution logs only and do +/// not affect consensus validation behavior. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct TransactionVerifyProfile { + /// Total time spent looking up input prevouts. + pub prevout_lookup_us: u128, + /// Total time spent adding input values. + pub input_value_us: u128, + /// Total time spent allocating witness vectors for interpreter calls. + pub witness_to_vec_us: u128, + /// Total time spent executing input scripts. + pub interpreter_execute_us: u128, + /// Total time spent computing transaction sigop cost. + pub sigop_cost_us: u128, + /// Number of transaction inputs observed by the verifier. + pub input_count: usize, +} + +impl TransactionVerifyProfile { + /// Returns the saturating sum of all timed verifier-internal buckets. + #[must_use] + pub fn bucket_sum_us(&self) -> u128 { + self.prevout_lookup_us + .saturating_add(self.input_value_us) + .saturating_add(self.witness_to_vec_us) + .saturating_add(self.interpreter_execute_us) + .saturating_add(self.sigop_cost_us) + } +} + /// Returns `true` iff the transaction is locktime-final at `block_height` and the timestamp cutoff. /// /// Implements Bitcoin Core's `IsFinalTx`: @@ -110,6 +144,21 @@ pub fn verify_transaction_borrowed_with_mtp( locktime_cutoff: u32, flags: VerifyFlags, ) -> Result<(), ConsensusError> { + verify_transaction_borrowed_with_locktime_cutoff(tx, prevouts, height, locktime_cutoff, flags) + .map(|_profile| ()) +} + +/// Verifies non-contextual and input-script transaction rules for a borrowed transaction and returns a coarse timing profile. +/// +/// The historical `_with_mtp` suffix is retained for source compatibility. Callers pass block +/// header time before BIP113 activation and previous-tip MTP after activation. +pub fn verify_transaction_borrowed_with_mtp_profiled( + tx: &bitcoin::Transaction, + prevouts: &impl UtxoView, + height: u32, + locktime_cutoff: u32, + flags: VerifyFlags, +) -> Result { verify_transaction_borrowed_with_locktime_cutoff(tx, prevouts, height, locktime_cutoff, flags) } @@ -120,7 +169,7 @@ fn verify_transaction_borrowed_with_locktime_cutoff( height: u32, locktime_cutoff: u32, flags: VerifyFlags, -) -> Result<(), ConsensusError> { +) -> Result { if !is_final_tx_with_locktime_cutoff(tx, height, locktime_cutoff) { return Err(ConsensusError::Bip { bip: "BIP113", @@ -132,6 +181,11 @@ fn verify_transaction_borrowed_with_locktime_cutoff( }); } + let mut profile = TransactionVerifyProfile { + input_count: tx.input.len(), + ..TransactionVerifyProfile::default() + }; + if tx.input.is_empty() { return Err(ConsensusError::EmptyInputs); } @@ -142,7 +196,7 @@ fn verify_transaction_borrowed_with_locktime_cutoff( let output_value = total_output_value_borrowed(tx)?; if tx.is_coinbase() { verify_coinbase_script_sig_size(tx)?; - return Ok(()); + return Ok(profile); } let mut seen = BTreeSet::new(); @@ -158,27 +212,45 @@ fn verify_transaction_borrowed_with_locktime_cutoff( let mut input_value = 0u64; let interpreter = Interpreter; for (input_index, input) in tx.input.iter().enumerate() { - let prevout = prevouts - .lookup(&input.previous_output) - .ok_or(ConsensusError::MissingPrevout { input_index })?; - input_value = input_value + let prevout_lookup_started = Instant::now(); + let prevout_result = prevouts.lookup(&input.previous_output); + profile.prevout_lookup_us = profile + .prevout_lookup_us + .saturating_add(prevout_lookup_started.elapsed().as_micros()); + let prevout = prevout_result.ok_or(ConsensusError::MissingPrevout { input_index })?; + + let input_value_started = Instant::now(); + let next_input_value = input_value .checked_add(prevout.value.to_sat()) - .ok_or(ConsensusError::OutputValueOverflow)?; + .ok_or(ConsensusError::OutputValueOverflow); + profile.input_value_us = profile + .input_value_us + .saturating_add(input_value_started.elapsed().as_micros()); + input_value = next_input_value?; + + let witness_to_vec_started = Instant::now(); let witness = input.witness.to_vec(); - interpreter - .execute( - prevout.script_pubkey.as_bytes(), - input.script_sig.as_bytes(), - &witness, - flags, - &prevout, - tx, - input_index, - ) - .map_err(|error| ConsensusError::Script { - input_index, - reason: error.to_string(), - })?; + profile.witness_to_vec_us = profile + .witness_to_vec_us + .saturating_add(witness_to_vec_started.elapsed().as_micros()); + + let interpreter_execute_started = Instant::now(); + let interpreter_result = interpreter.execute( + prevout.script_pubkey.as_bytes(), + input.script_sig.as_bytes(), + &witness, + flags, + &prevout, + tx, + input_index, + ); + profile.interpreter_execute_us = profile + .interpreter_execute_us + .saturating_add(interpreter_execute_started.elapsed().as_micros()); + interpreter_result.map_err(|error| ConsensusError::Script { + input_index, + reason: error.to_string(), + })?; } if input_value < output_value { @@ -188,8 +260,12 @@ fn verify_transaction_borrowed_with_locktime_cutoff( }); } - let sigop_cost = u32::try_from(tx.total_sigop_cost(|outpoint| prevouts.lookup(outpoint))) - .unwrap_or(u32::MAX); + let sigop_cost_started = Instant::now(); + let sigop_cost_result = tx.total_sigop_cost(|outpoint| prevouts.lookup(outpoint)); + profile.sigop_cost_us = profile + .sigop_cost_us + .saturating_add(sigop_cost_started.elapsed().as_micros()); + let sigop_cost = u32::try_from(sigop_cost_result).unwrap_or(u32::MAX); if sigop_cost > MAX_BLOCK_SIGOPS_COST { return Err(ConsensusError::SigopsLimit { cost: sigop_cost, @@ -197,7 +273,7 @@ fn verify_transaction_borrowed_with_locktime_cutoff( }); } - Ok(()) + Ok(profile) } fn total_output_value_borrowed(tx: &bitcoin::Transaction) -> Result { @@ -228,6 +304,7 @@ mod tests { use super::{ is_final_tx_with_locktime_cutoff, verify_coinbase_script_sig_size, verify_transaction, + verify_transaction_borrowed_with_mtp, verify_transaction_borrowed_with_mtp_profiled, verify_transaction_with_mtp, }; use crate::ConsensusError; @@ -284,6 +361,76 @@ mod tests { } } + #[test] + fn profiled_borrowed_verifier_preserves_success_and_input_count() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([2; 32]), + vout: 0, + }; + let tx = Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }; + let mut utxos = BTreeMap::new(); + utxos.insert( + outpoint, + TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(1).into_script(), + }, + ); + + assert_eq!( + verify_transaction_borrowed_with_mtp(&tx, &utxos, 0, 0, VerifyFlags::NONE), + Ok(()) + ); + let profile = match verify_transaction_borrowed_with_mtp_profiled( + &tx, + &utxos, + 0, + 0, + VerifyFlags::NONE, + ) { + Ok(profile) => profile, + Err(error) => { + panic!("profiled verifier should preserve successful validation: {error:?}") + } + }; + + assert_eq!(profile.input_count, 1); + } + + #[test] + fn profiled_borrowed_verifier_preserves_error_behavior() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([3; 32]), + vout: 0, + }; + let tx = Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }; + let utxos = BTreeMap::new(); + assert_eq!( + verify_transaction_borrowed_with_mtp(&tx, &utxos, 0, 0, VerifyFlags::NONE), + Err(ConsensusError::MissingPrevout { input_index: 0 }) + ); + assert_eq!( + verify_transaction_borrowed_with_mtp_profiled(&tx, &utxos, 0, 0, VerifyFlags::NONE), + Err(ConsensusError::MissingPrevout { input_index: 0 }) + ); + } + #[test] fn duplicate_non_coinbase_input_is_rejected() { let outpoint = OutPoint { diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index ebaa497..137c05f 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -264,6 +264,12 @@ pub fn apply_block( input_count = tx_profile.input_count, output_count = tx_profile.output_count, tx_verify_us = tx_profile.tx_verify_us, + prevout_lookup_us = tx_profile.verify_profile.prevout_lookup_us, + input_value_us = tx_profile.verify_profile.input_value_us, + witness_to_vec_us = tx_profile.verify_profile.witness_to_vec_us, + interpreter_execute_us = tx_profile.verify_profile.interpreter_execute_us, + sigop_cost_us = tx_profile.verify_profile.sigop_cost_us, + tx_verify_residual_us = tx_profile.tx_verify_residual_us(), block_tx_verify_call_us = script_verify_profile.tx_verify_call_us, block_script_verify_us = script_verify_dur.as_micros(), "apply_block: script verify transaction attribution" @@ -576,7 +582,7 @@ fn verify_block_transactions( profile.observe_non_coinbase(tx); let tx_verify_started = quanta::Instant::now(); let tx_verify_result = - bitcoin_rs_consensus::verify_tx::verify_transaction_borrowed_with_mtp( + bitcoin_rs_consensus::verify_tx::verify_transaction_borrowed_with_mtp_profiled( tx, &view, height, @@ -586,8 +592,8 @@ fn verify_block_transactions( let tx_verify_dur = tx_verify_started.elapsed(); let tx_verify_us = tx_verify_dur.as_micros(); profile.add_tx_verify_call_us(tx_verify_us); - tx_verify_result?; - profile.observe_slow_tx(tx_index, tx, tx_verify_us); + let verify_profile = tx_verify_result?; + profile.observe_slow_tx(tx_index, tx, tx_verify_us, verify_profile); view.spend_inputs(tx); view.add_outputs(tx, height)?; } @@ -608,6 +614,14 @@ struct SlowScriptVerifyTxProfile { input_count: usize, output_count: usize, tx_verify_us: u128, + verify_profile: bitcoin_rs_consensus::verify_tx::TransactionVerifyProfile, +} + +impl SlowScriptVerifyTxProfile { + fn tx_verify_residual_us(&self) -> u128 { + self.tx_verify_us + .saturating_sub(self.verify_profile.bucket_sum_us()) + } } impl ScriptVerifyProfile { @@ -616,7 +630,13 @@ impl ScriptVerifyProfile { self.total_input_count = self.total_input_count.saturating_add(tx.input.len()); } - fn observe_slow_tx(&mut self, tx_index: usize, tx: &bitcoin::Transaction, tx_verify_us: u128) { + fn observe_slow_tx( + &mut self, + tx_index: usize, + tx: &bitcoin::Transaction, + tx_verify_us: u128, + verify_profile: bitcoin_rs_consensus::verify_tx::TransactionVerifyProfile, + ) { let insert_index = self .slow_txs .iter() @@ -634,6 +654,7 @@ impl ScriptVerifyProfile { input_count: tx.input.len(), output_count: tx.output.len(), tx_verify_us, + verify_profile, }, ); self.slow_txs.truncate(SCRIPT_VERIFY_PROFILE_TOP_N); @@ -1301,7 +1322,12 @@ mod consensus_rule_tests { let mut tx_verify_us = 0_u128; for (tx_index, tx) in transactions.iter().enumerate() { - profile.observe_slow_tx(tx_index, tx, tx_verify_us); + profile.observe_slow_tx( + tx_index, + tx, + tx_verify_us, + bitcoin_rs_consensus::verify_tx::TransactionVerifyProfile::default(), + ); tx_verify_us = tx_verify_us.saturating_add(1); } From fe306b349847a8eeef61329abc28cf7aa2ad81b8 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 21:31:19 +0000 Subject: [PATCH 27/37] refactor(consensus): remove verifier bucket profiling Remove the temporary public transaction verifier profiling API after collecting bounded signet attribution evidence. Keep node-level bounded top transaction timing and add a regression that underfunded transactions fail before script execution.\n\nEvidence:\n- cargo fmt --all -- --check\n- cargo test -p bitcoin-rs-consensus underfunded_transaction_fails_before_script_execution -- --nocapture\n- cargo test -p bitcoin-rs-consensus --no-fail-fast\n- cargo test -p bitcoin-rs-node verify_block_transactions --no-default-features --features rocksdb,bitcoinconsensus -- --nocapture\n- cargo test -p bitcoin-rs-node script_verify_profile_keeps_bounded_slowest_transactions --no-default-features --features rocksdb,bitcoinconsensus -- --nocapture\n- cargo clippy -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings\n- cargo test -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus\n- cargo test --workspace --no-fail-fast\n\nOp: compress --- crates/consensus/src/verify_tx.rs | 206 +++++++----------------------- crates/node/src/apply.rs | 36 +----- 2 files changed, 54 insertions(+), 188 deletions(-) diff --git a/crates/consensus/src/verify_tx.rs b/crates/consensus/src/verify_tx.rs index b9c0c43..2c42125 100644 --- a/crates/consensus/src/verify_tx.rs +++ b/crates/consensus/src/verify_tx.rs @@ -1,5 +1,4 @@ use std::collections::BTreeSet; -use std::time::Instant; use bitcoin_rs_primitives::Tx; use bitcoin_rs_script::{Interpreter, VerifyFlags}; @@ -12,39 +11,6 @@ const SEQUENCE_FINAL: u32 = 0xffff_ffff; const MIN_COINBASE_SCRIPT_SIG_SIZE: usize = 2; const MAX_COINBASE_SCRIPT_SIG_SIZE: usize = 100; -/// Coarse timing profile for borrowed transaction verification. -/// -/// Buckets are accumulated microsecond totals for selected verifier-internal -/// operations. They are intended for node-owned attribution logs only and do -/// not affect consensus validation behavior. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct TransactionVerifyProfile { - /// Total time spent looking up input prevouts. - pub prevout_lookup_us: u128, - /// Total time spent adding input values. - pub input_value_us: u128, - /// Total time spent allocating witness vectors for interpreter calls. - pub witness_to_vec_us: u128, - /// Total time spent executing input scripts. - pub interpreter_execute_us: u128, - /// Total time spent computing transaction sigop cost. - pub sigop_cost_us: u128, - /// Number of transaction inputs observed by the verifier. - pub input_count: usize, -} - -impl TransactionVerifyProfile { - /// Returns the saturating sum of all timed verifier-internal buckets. - #[must_use] - pub fn bucket_sum_us(&self) -> u128 { - self.prevout_lookup_us - .saturating_add(self.input_value_us) - .saturating_add(self.witness_to_vec_us) - .saturating_add(self.interpreter_execute_us) - .saturating_add(self.sigop_cost_us) - } -} - /// Returns `true` iff the transaction is locktime-final at `block_height` and the timestamp cutoff. /// /// Implements Bitcoin Core's `IsFinalTx`: @@ -144,21 +110,6 @@ pub fn verify_transaction_borrowed_with_mtp( locktime_cutoff: u32, flags: VerifyFlags, ) -> Result<(), ConsensusError> { - verify_transaction_borrowed_with_locktime_cutoff(tx, prevouts, height, locktime_cutoff, flags) - .map(|_profile| ()) -} - -/// Verifies non-contextual and input-script transaction rules for a borrowed transaction and returns a coarse timing profile. -/// -/// The historical `_with_mtp` suffix is retained for source compatibility. Callers pass block -/// header time before BIP113 activation and previous-tip MTP after activation. -pub fn verify_transaction_borrowed_with_mtp_profiled( - tx: &bitcoin::Transaction, - prevouts: &impl UtxoView, - height: u32, - locktime_cutoff: u32, - flags: VerifyFlags, -) -> Result { verify_transaction_borrowed_with_locktime_cutoff(tx, prevouts, height, locktime_cutoff, flags) } @@ -169,7 +120,7 @@ fn verify_transaction_borrowed_with_locktime_cutoff( height: u32, locktime_cutoff: u32, flags: VerifyFlags, -) -> Result { +) -> Result<(), ConsensusError> { if !is_final_tx_with_locktime_cutoff(tx, height, locktime_cutoff) { return Err(ConsensusError::Bip { bip: "BIP113", @@ -181,11 +132,6 @@ fn verify_transaction_borrowed_with_locktime_cutoff( }); } - let mut profile = TransactionVerifyProfile { - input_count: tx.input.len(), - ..TransactionVerifyProfile::default() - }; - if tx.input.is_empty() { return Err(ConsensusError::EmptyInputs); } @@ -196,7 +142,7 @@ fn verify_transaction_borrowed_with_locktime_cutoff( let output_value = total_output_value_borrowed(tx)?; if tx.is_coinbase() { verify_coinbase_script_sig_size(tx)?; - return Ok(profile); + return Ok(()); } let mut seen = BTreeSet::new(); @@ -210,47 +156,15 @@ fn verify_transaction_borrowed_with_locktime_cutoff( } let mut input_value = 0u64; - let interpreter = Interpreter; + let mut input_prevouts = Vec::with_capacity(tx.input.len()); for (input_index, input) in tx.input.iter().enumerate() { - let prevout_lookup_started = Instant::now(); - let prevout_result = prevouts.lookup(&input.previous_output); - profile.prevout_lookup_us = profile - .prevout_lookup_us - .saturating_add(prevout_lookup_started.elapsed().as_micros()); - let prevout = prevout_result.ok_or(ConsensusError::MissingPrevout { input_index })?; - - let input_value_started = Instant::now(); - let next_input_value = input_value + let prevout = prevouts + .lookup(&input.previous_output) + .ok_or(ConsensusError::MissingPrevout { input_index })?; + input_value = input_value .checked_add(prevout.value.to_sat()) - .ok_or(ConsensusError::OutputValueOverflow); - profile.input_value_us = profile - .input_value_us - .saturating_add(input_value_started.elapsed().as_micros()); - input_value = next_input_value?; - - let witness_to_vec_started = Instant::now(); - let witness = input.witness.to_vec(); - profile.witness_to_vec_us = profile - .witness_to_vec_us - .saturating_add(witness_to_vec_started.elapsed().as_micros()); - - let interpreter_execute_started = Instant::now(); - let interpreter_result = interpreter.execute( - prevout.script_pubkey.as_bytes(), - input.script_sig.as_bytes(), - &witness, - flags, - &prevout, - tx, - input_index, - ); - profile.interpreter_execute_us = profile - .interpreter_execute_us - .saturating_add(interpreter_execute_started.elapsed().as_micros()); - interpreter_result.map_err(|error| ConsensusError::Script { - input_index, - reason: error.to_string(), - })?; + .ok_or(ConsensusError::OutputValueOverflow)?; + input_prevouts.push(prevout); } if input_value < output_value { @@ -260,11 +174,27 @@ fn verify_transaction_borrowed_with_locktime_cutoff( }); } - let sigop_cost_started = Instant::now(); + let interpreter = Interpreter; + for (input_index, (input, prevout)) in tx.input.iter().zip(input_prevouts.iter()).enumerate() { + let witness = input.witness.to_vec(); + + interpreter + .execute( + prevout.script_pubkey.as_bytes(), + input.script_sig.as_bytes(), + &witness, + flags, + prevout, + tx, + input_index, + ) + .map_err(|error| ConsensusError::Script { + input_index, + reason: error.to_string(), + })?; + } + let sigop_cost_result = tx.total_sigop_cost(|outpoint| prevouts.lookup(outpoint)); - profile.sigop_cost_us = profile - .sigop_cost_us - .saturating_add(sigop_cost_started.elapsed().as_micros()); let sigop_cost = u32::try_from(sigop_cost_result).unwrap_or(u32::MAX); if sigop_cost > MAX_BLOCK_SIGOPS_COST { return Err(ConsensusError::SigopsLimit { @@ -273,7 +203,7 @@ fn verify_transaction_borrowed_with_locktime_cutoff( }); } - Ok(profile) + Ok(()) } fn total_output_value_borrowed(tx: &bitcoin::Transaction) -> Result { @@ -304,8 +234,7 @@ mod tests { use super::{ is_final_tx_with_locktime_cutoff, verify_coinbase_script_sig_size, verify_transaction, - verify_transaction_borrowed_with_mtp, verify_transaction_borrowed_with_mtp_profiled, - verify_transaction_with_mtp, + verify_transaction_borrowed_with_mtp, verify_transaction_with_mtp, }; use crate::ConsensusError; @@ -362,20 +291,20 @@ mod tests { } #[test] - fn profiled_borrowed_verifier_preserves_success_and_input_count() { + fn duplicate_non_coinbase_input_is_rejected() { let outpoint = OutPoint { - txid: Txid::from_byte_array([2; 32]), + txid: Txid::from_byte_array([1; 32]), vout: 0, }; - let tx = Transaction { + let tx = Tx(Transaction { version: transaction::Version(1), lock_time: absolute::LockTime::ZERO, - input: vec![spending_input(outpoint)], + input: vec![spending_input(outpoint), spending_input(outpoint)], output: vec![TxOut { value: Amount::from_sat(50), script_pubkey: ScriptBuf::new(), }], - }; + }); let mut utxos = BTreeMap::new(); utxos.insert( outpoint, @@ -384,65 +313,24 @@ mod tests { script_pubkey: Builder::new().push_int(1).into_script(), }, ); - - assert_eq!( - verify_transaction_borrowed_with_mtp(&tx, &utxos, 0, 0, VerifyFlags::NONE), - Ok(()) - ); - let profile = match verify_transaction_borrowed_with_mtp_profiled( - &tx, - &utxos, - 0, - 0, - VerifyFlags::NONE, - ) { - Ok(profile) => profile, - Err(error) => { - panic!("profiled verifier should preserve successful validation: {error:?}") - } - }; - - assert_eq!(profile.input_count, 1); - } - - #[test] - fn profiled_borrowed_verifier_preserves_error_behavior() { - let outpoint = OutPoint { - txid: Txid::from_byte_array([3; 32]), - vout: 0, - }; - let tx = Transaction { - version: transaction::Version(1), - lock_time: absolute::LockTime::ZERO, - input: vec![spending_input(outpoint)], - output: vec![TxOut { - value: Amount::from_sat(50), - script_pubkey: ScriptBuf::new(), - }], - }; - let utxos = BTreeMap::new(); - assert_eq!( - verify_transaction_borrowed_with_mtp(&tx, &utxos, 0, 0, VerifyFlags::NONE), - Err(ConsensusError::MissingPrevout { input_index: 0 }) - ); assert_eq!( - verify_transaction_borrowed_with_mtp_profiled(&tx, &utxos, 0, 0, VerifyFlags::NONE), - Err(ConsensusError::MissingPrevout { input_index: 0 }) + verify_transaction(&tx, &utxos, 0, VerifyFlags::NONE), + Err(ConsensusError::DuplicateInput { input_index: 1 }) ); } #[test] - fn duplicate_non_coinbase_input_is_rejected() { + fn underfunded_transaction_fails_before_script_execution() { let outpoint = OutPoint { - txid: Txid::from_byte_array([1; 32]), + txid: Txid::from_byte_array([4; 32]), vout: 0, }; let tx = Tx(Transaction { version: transaction::Version(1), lock_time: absolute::LockTime::ZERO, - input: vec![spending_input(outpoint), spending_input(outpoint)], + input: vec![spending_input(outpoint)], output: vec![TxOut { - value: Amount::from_sat(50), + value: Amount::from_sat(100), script_pubkey: ScriptBuf::new(), }], }); @@ -450,13 +338,17 @@ mod tests { utxos.insert( outpoint, TxOut { - value: Amount::from_sat(100), - script_pubkey: Builder::new().push_int(1).into_script(), + value: Amount::from_sat(50), + script_pubkey: Builder::new().push_int(0).into_script(), }, ); + assert_eq!( verify_transaction(&tx, &utxos, 0, VerifyFlags::NONE), - Err(ConsensusError::DuplicateInput { input_index: 1 }) + Err(ConsensusError::InputsLessThanOutputs { + input_value: 50, + output_value: 100, + }) ); } diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index 137c05f..ebaa497 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -264,12 +264,6 @@ pub fn apply_block( input_count = tx_profile.input_count, output_count = tx_profile.output_count, tx_verify_us = tx_profile.tx_verify_us, - prevout_lookup_us = tx_profile.verify_profile.prevout_lookup_us, - input_value_us = tx_profile.verify_profile.input_value_us, - witness_to_vec_us = tx_profile.verify_profile.witness_to_vec_us, - interpreter_execute_us = tx_profile.verify_profile.interpreter_execute_us, - sigop_cost_us = tx_profile.verify_profile.sigop_cost_us, - tx_verify_residual_us = tx_profile.tx_verify_residual_us(), block_tx_verify_call_us = script_verify_profile.tx_verify_call_us, block_script_verify_us = script_verify_dur.as_micros(), "apply_block: script verify transaction attribution" @@ -582,7 +576,7 @@ fn verify_block_transactions( profile.observe_non_coinbase(tx); let tx_verify_started = quanta::Instant::now(); let tx_verify_result = - bitcoin_rs_consensus::verify_tx::verify_transaction_borrowed_with_mtp_profiled( + bitcoin_rs_consensus::verify_tx::verify_transaction_borrowed_with_mtp( tx, &view, height, @@ -592,8 +586,8 @@ fn verify_block_transactions( let tx_verify_dur = tx_verify_started.elapsed(); let tx_verify_us = tx_verify_dur.as_micros(); profile.add_tx_verify_call_us(tx_verify_us); - let verify_profile = tx_verify_result?; - profile.observe_slow_tx(tx_index, tx, tx_verify_us, verify_profile); + tx_verify_result?; + profile.observe_slow_tx(tx_index, tx, tx_verify_us); view.spend_inputs(tx); view.add_outputs(tx, height)?; } @@ -614,14 +608,6 @@ struct SlowScriptVerifyTxProfile { input_count: usize, output_count: usize, tx_verify_us: u128, - verify_profile: bitcoin_rs_consensus::verify_tx::TransactionVerifyProfile, -} - -impl SlowScriptVerifyTxProfile { - fn tx_verify_residual_us(&self) -> u128 { - self.tx_verify_us - .saturating_sub(self.verify_profile.bucket_sum_us()) - } } impl ScriptVerifyProfile { @@ -630,13 +616,7 @@ impl ScriptVerifyProfile { self.total_input_count = self.total_input_count.saturating_add(tx.input.len()); } - fn observe_slow_tx( - &mut self, - tx_index: usize, - tx: &bitcoin::Transaction, - tx_verify_us: u128, - verify_profile: bitcoin_rs_consensus::verify_tx::TransactionVerifyProfile, - ) { + fn observe_slow_tx(&mut self, tx_index: usize, tx: &bitcoin::Transaction, tx_verify_us: u128) { let insert_index = self .slow_txs .iter() @@ -654,7 +634,6 @@ impl ScriptVerifyProfile { input_count: tx.input.len(), output_count: tx.output.len(), tx_verify_us, - verify_profile, }, ); self.slow_txs.truncate(SCRIPT_VERIFY_PROFILE_TOP_N); @@ -1322,12 +1301,7 @@ mod consensus_rule_tests { let mut tx_verify_us = 0_u128; for (tx_index, tx) in transactions.iter().enumerate() { - profile.observe_slow_tx( - tx_index, - tx, - tx_verify_us, - bitcoin_rs_consensus::verify_tx::TransactionVerifyProfile::default(), - ); + profile.observe_slow_tx(tx_index, tx, tx_verify_us); tx_verify_us = tx_verify_us.saturating_add(1); } From f8f8027a2a499d0fa11a57d0c1567bf6a33f16a2 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 22:00:39 +0000 Subject: [PATCH 28/37] perf(script): add interpreter boundary benchmark Add a dev-only Criterion benchmark target for the script interpreter boundary. The fixture separates non-empty witness conversion, transaction clone/mutation, serialization, bitcoinconsensus verification of a pre-serialized transaction, and full Interpreter::execute on a 400-input transaction. The benchmark target is explicitly gated on the bitcoinconsensus feature so no-default feature checks do not build the backend-only slice accidentally. This is diagnostic smoke evidence only, not a signet-wide root-cause or Bitcoin Core comparison. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-script --no-default-features --features bitcoinconsensus --bench interpreter_execute_profile --no-run - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-script --all-targets --no-default-features -- -D warnings - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-script --all-targets --no-default-features --features bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-script --no-default-features --features bitcoinconsensus --no-fail-fast - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-script --no-default-features --features bitcoinconsensus --bench interpreter_execute_profile - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus Op: extend --- Cargo.lock | 1 + crates/script/Cargo.toml | 6 + .../benches/interpreter_execute_profile.rs | 221 ++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 crates/script/benches/interpreter_execute_profile.rs diff --git a/Cargo.lock b/Cargo.lock index 39589c6..1b9bd3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -552,6 +552,7 @@ dependencies = [ "bitcoin", "bitcoin-rs-primitives", "bytemuck", + "criterion", "parking_lot", "proptest", "rayon", diff --git a/crates/script/Cargo.toml b/crates/script/Cargo.toml index eb5107d..74e7f3a 100644 --- a/crates/script/Cargo.toml +++ b/crates/script/Cargo.toml @@ -32,5 +32,11 @@ bytemuck.workspace = true serde.workspace = true [dev-dependencies] +criterion.workspace = true serde_json.workspace = true proptest.workspace = true + +[[bench]] +name = "interpreter_execute_profile" +harness = false +required-features = ["bitcoinconsensus"] diff --git a/crates/script/benches/interpreter_execute_profile.rs b/crates/script/benches/interpreter_execute_profile.rs new file mode 100644 index 0000000..f5356bf --- /dev/null +++ b/crates/script/benches/interpreter_execute_profile.rs @@ -0,0 +1,221 @@ +//! Diagnostic benchmark for `Interpreter::execute` boundary costs. +// PERF: Criterion emits public harness items whose docs are irrelevant to the benchmark report. +#![allow(missing_docs)] + +use std::{hint::black_box, time::Duration}; + +use bitcoin::consensus::encode; +use bitcoin::hashes::Hash as _; +use bitcoin::script::Builder; +use bitcoin::{ + Amount, OutPoint, Script, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness, + absolute, transaction, +}; +use bitcoin_rs_script::{Interpreter, VerifyFlags}; +use criterion::{ + BatchSize, BenchmarkGroup, Criterion, criterion_group, criterion_main, measurement::WallTime, +}; + +const INPUT_COUNT: usize = 400; +const SELECTED_INPUT: usize = INPUT_COUNT - 1; +const PREVOUT_VALUE: u64 = 50_000; +const OUTPUT_VALUE: u64 = 1_000; +const SAMPLE_SIZE: usize = 20; +const MEASUREMENT_SECONDS: u64 = 3; +const WITNESS_ITEM_COUNT: usize = 4; +const WITNESS_ITEM_LEN: usize = 72; + +struct SpendCase { + tx: Transaction, + prevout: TxOut, + script_pubkey: ScriptBuf, + script_sig: ScriptBuf, + witness_vec: Vec>, + flags: VerifyFlags, +} + +fn interpreter_execute_profile(c: &mut Criterion) { + let case = spend_case(); + let mut group = c.benchmark_group("interpreter_execute_profile"); + group.sample_size(SAMPLE_SIZE); + group.measurement_time(Duration::from_secs(MEASUREMENT_SECONDS)); + + bench_witness_to_vec(&mut group, &case); + bench_clone_mutate(&mut group, &case); + bench_serialize_mutated(&mut group, &case); + bench_bitcoinconsensus_verify_serialized(&mut group, &case); + bench_interpreter_execute(&mut group, &case); + + group.finish(); +} + +fn bench_witness_to_vec(group: &mut BenchmarkGroup<'_, WallTime>, case: &SpendCase) { + group.bench_function("witness_to_vec_400_input", |b| { + b.iter(|| black_box(selected_input(&case.tx).witness.to_vec())); + }); +} + +fn bench_clone_mutate(group: &mut BenchmarkGroup<'_, WallTime>, case: &SpendCase) { + group.bench_function("clone_mutate_400_input", |b| { + b.iter(|| black_box(cloned_spending(black_box(case)))); + }); +} + +fn bench_serialize_mutated(group: &mut BenchmarkGroup<'_, WallTime>, case: &SpendCase) { + group.bench_function("serialize_mutated_400_input", |b| { + b.iter_batched( + || cloned_spending(case), + |spending| black_box(encode::serialize(&spending)), + BatchSize::SmallInput, + ); + }); +} + +fn bench_bitcoinconsensus_verify_serialized( + group: &mut BenchmarkGroup<'_, WallTime>, + case: &SpendCase, +) { + let spending = cloned_spending(case); + let serialized = encode::serialize(&spending); + let script = Script::from_bytes(case.script_pubkey.as_bytes()); + + group.bench_function("bitcoinconsensus_verify_serialized_400_input", |b| { + b.iter(|| { + let result = script.verify_with_flags( + SELECTED_INPUT, + case.prevout.value, + black_box(serialized.as_slice()), + case.flags.consensus_bits(), + ); + match result { + Ok(()) => black_box(true), + Err(error) => panic!("bitcoinconsensus verification failed: {error}"), + } + }); + }); +} + +fn bench_interpreter_execute(group: &mut BenchmarkGroup<'_, WallTime>, case: &SpendCase) { + let interpreter = Interpreter; + group.bench_function("interpreter_execute_400_input", |b| { + b.iter(|| { + let result = interpreter.execute( + case.script_pubkey.as_bytes(), + case.script_sig.as_bytes(), + black_box(case.witness_vec.as_slice()), + case.flags, + &case.prevout, + &case.tx, + SELECTED_INPUT, + ); + match result { + Ok(value) => black_box(value), + Err(error) => panic!("interpreter execution failed: {error}"), + } + }); + }); +} + +fn spend_case() -> SpendCase { + let script_pubkey = op_true_script(); + let script_sig = ScriptBuf::new(); + let witness_vec = witness_vec(); + let mut tx = Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: (0..INPUT_COUNT) + .map(|index| TxIn { + previous_output: OutPoint { + txid: txid(usize_to_u64(index)), + vout: 0, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }) + .collect(), + output: vec![TxOut { + value: Amount::from_sat(OUTPUT_VALUE), + script_pubkey: ScriptBuf::new(), + }], + }; + selected_input_mut(&mut tx).witness = Witness::from_slice(&witness_vec); + + SpendCase { + tx, + prevout: TxOut { + value: Amount::from_sat(PREVOUT_VALUE), + script_pubkey: script_pubkey.clone(), + }, + script_pubkey, + script_sig, + witness_vec, + flags: VerifyFlags::NONE, + } +} + +fn cloned_spending(case: &SpendCase) -> Transaction { + let mut spending = case.tx.clone(); + let inputs = spending.input.len(); + let input = match spending.input.get_mut(SELECTED_INPUT) { + Some(input) => input, + None => panic!("selected input {SELECTED_INPUT} out of range for {inputs} inputs"), + }; + input.script_sig = case.script_sig.clone(); + input.witness = Witness::from_slice(&case.witness_vec); + spending +} + +fn selected_input(tx: &Transaction) -> &TxIn { + match tx.input.get(SELECTED_INPUT) { + Some(input) => input, + None => panic!( + "selected input {SELECTED_INPUT} out of range for {} inputs", + tx.input.len() + ), + } +} + +fn selected_input_mut(tx: &mut Transaction) -> &mut TxIn { + let inputs = tx.input.len(); + match tx.input.get_mut(SELECTED_INPUT) { + Some(input) => input, + None => panic!("selected input {SELECTED_INPUT} out of range for {inputs} inputs"), + } +} + +fn op_true_script() -> ScriptBuf { + Builder::new().push_int(1).into_script() +} + +fn witness_vec() -> Vec> { + (0..WITNESS_ITEM_COUNT) + .map(|index| vec![usize_to_u8(index); WITNESS_ITEM_LEN]) + .collect() +} + +fn txid(seed: u64) -> Txid { + let mut bytes = [0_u8; 32]; + bytes[..8].copy_from_slice(&seed.to_le_bytes()); + bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); + bytes[16..24].copy_from_slice(&seed.wrapping_mul(0x9e37_79b9_7f4a_7c15).to_le_bytes()); + bytes[24..32].copy_from_slice(&seed.wrapping_add(0xd1b5_4a32_d192_ed03).to_le_bytes()); + Txid::from_byte_array(bytes) +} + +fn usize_to_u64(value: usize) -> u64 { + match u64::try_from(value) { + Ok(value) => value, + Err(error) => panic!("usize to u64 conversion failed: {error}"), + } +} + +fn usize_to_u8(value: usize) -> u8 { + match u8::try_from(value) { + Ok(value) => value, + Err(error) => panic!("usize to u8 conversion failed: {error}"), + } +} + +criterion_group!(benches, interpreter_execute_profile); +criterion_main!(benches); From 97dcc417fa53833897d28c4f13004eec63c58c53 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 22:21:50 +0000 Subject: [PATCH 29/37] chore(consensus): remove unused verifier test import Drop a test-module import that is unused under the no-default all-targets clippy graph. This keeps the consensus clippy gate warning-free without changing verifier behavior. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --all-targets --no-default-features -- -D warnings - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --all-targets --no-default-features --features bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-fail-fast - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus Op: compress --- crates/consensus/src/verify_tx.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/consensus/src/verify_tx.rs b/crates/consensus/src/verify_tx.rs index 2c42125..d1b7f8d 100644 --- a/crates/consensus/src/verify_tx.rs +++ b/crates/consensus/src/verify_tx.rs @@ -234,7 +234,7 @@ mod tests { use super::{ is_final_tx_with_locktime_cutoff, verify_coinbase_script_sig_size, verify_transaction, - verify_transaction_borrowed_with_mtp, verify_transaction_with_mtp, + verify_transaction_with_mtp, }; use crate::ConsensusError; From 82eab36d30277a2f2101563565f5bc923cf61159 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 22:22:22 +0000 Subject: [PATCH 30/37] perf(consensus): add verifier path benchmark Add a bitcoinconsensus-gated Criterion benchmark for the production RustValidator::verify_tx path. The fixture uses a 400-input OP_TRUE transaction with non-empty witness data and a BTreeMap UTXO view, so the timed path includes the current verifier bookkeeping, UTXO lookups, per-input interpreter boundary, and sigop cost path. This is evidence-only diagnostic coverage for #191. It is not representative signet/mainnet proof, not a root-cause claim, and not a Bitcoin Core comparison. Benchmark sample: - verify_transaction_profile/verify_tx_op_true_400_input: [384.56 ms 387.83 ms 390.87 ms] on one tuned local Criterion run Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-default-features --features bitcoinconsensus --bench verify_transaction_profile --no-run - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --all-targets --no-default-features -- -D warnings - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --all-targets --no-default-features --features bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-fail-fast - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-default-features --features bitcoinconsensus --bench verify_transaction_profile - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus Op: extend --- Cargo.lock | 1 + crates/consensus/Cargo.toml | 8 + .../benches/verify_transaction_profile.rs | 138 ++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 crates/consensus/benches/verify_transaction_profile.rs diff --git a/Cargo.lock b/Cargo.lock index 1b9bd3f..3b5a730 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -307,6 +307,7 @@ dependencies = [ "bitcoin-rs-primitives", "bitcoin-rs-script", "bitcoinkernel", + "criterion", "secp256k1 0.31.1", "serde", "serde_json", diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml index cb3e5b5..0d327e8 100644 --- a/crates/consensus/Cargo.toml +++ b/crates/consensus/Cargo.toml @@ -30,3 +30,11 @@ tracing.workspace = true serde.workspace = true serde_json.workspace = true bitcoinkernel = { workspace = true, optional = true } + +[dev-dependencies] +criterion.workspace = true + +[[bench]] +name = "verify_transaction_profile" +harness = false +required-features = ["bitcoinconsensus"] diff --git a/crates/consensus/benches/verify_transaction_profile.rs b/crates/consensus/benches/verify_transaction_profile.rs new file mode 100644 index 0000000..eb7ec5e --- /dev/null +++ b/crates/consensus/benches/verify_transaction_profile.rs @@ -0,0 +1,138 @@ +//! Diagnostic benchmark for the production transaction verifier path. +// PERF: Criterion emits public harness items whose docs are irrelevant to the benchmark report. +#![allow(missing_docs)] + +use std::{collections::BTreeMap, hint::black_box, time::Duration}; + +use bitcoin::hashes::Hash as _; +use bitcoin::script::Builder; +use bitcoin::{ + Amount, Network, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness, + absolute, transaction, +}; +use bitcoin_rs_consensus::RustValidator; +use bitcoin_rs_primitives::Tx; +use bitcoin_rs_script::VerifyFlags; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; + +const INPUT_COUNT: usize = 400; +const PREVOUT_VALUE: u64 = 50_000; +const OUTPUT_VALUE: u64 = 1_000; +const SAMPLE_SIZE: usize = 10; +const MEASUREMENT_SECONDS: u64 = 5; +const WITNESS_ITEM_COUNT: usize = 4; +const WITNESS_ITEM_LEN: usize = 72; + +struct VerifyCase { + validator: RustValidator, + tx: Tx, + prevouts: BTreeMap, + height: u32, + flags: VerifyFlags, +} + +fn verify_transaction_profile(c: &mut Criterion) { + let mut group = c.benchmark_group("verify_transaction_profile"); + group.sample_size(SAMPLE_SIZE); + group.measurement_time(Duration::from_secs(MEASUREMENT_SECONDS)); + + group.bench_function("verify_tx_op_true_400_input", |b| { + b.iter_batched( + verify_case, + |case| { + let result = case.validator.verify_tx( + black_box(&case.tx), + black_box(&case.prevouts), + case.height, + case.flags, + ); + match result { + Ok(()) => black_box(()), + Err(error) => panic!("transaction verification failed: {error}"), + } + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +fn verify_case() -> VerifyCase { + let script_pubkey = op_true_script(); + let mut prevouts = BTreeMap::new(); + let input = (0..INPUT_COUNT) + .map(|index| { + let outpoint = OutPoint { + txid: txid(usize_to_u64(index)), + vout: 0, + }; + prevouts.insert( + outpoint, + TxOut { + value: Amount::from_sat(PREVOUT_VALUE), + script_pubkey: script_pubkey.clone(), + }, + ); + TxIn { + previous_output: outpoint, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::from_slice(&witness_vec()), + } + }) + .collect(); + let tx = Tx(Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input, + output: vec![TxOut { + value: Amount::from_sat(OUTPUT_VALUE), + script_pubkey: ScriptBuf::new(), + }], + }); + + VerifyCase { + validator: RustValidator::new(Network::Signet), + tx, + prevouts, + height: 1, + flags: VerifyFlags::NONE, + } +} + +fn op_true_script() -> ScriptBuf { + Builder::new().push_int(1).into_script() +} + +fn witness_vec() -> Vec> { + (0..WITNESS_ITEM_COUNT) + .map(|index| vec![usize_to_u8(index); WITNESS_ITEM_LEN]) + .collect() +} + +fn txid(seed: u64) -> Txid { + let mut bytes = [0_u8; 32]; + bytes[..8].copy_from_slice(&seed.to_le_bytes()); + bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); + bytes[16..24].copy_from_slice(&seed.wrapping_mul(0x9e37_79b9_7f4a_7c15).to_le_bytes()); + bytes[24..32].copy_from_slice(&seed.wrapping_add(0xd1b5_4a32_d192_ed03).to_le_bytes()); + Txid::from_byte_array(bytes) +} + +fn usize_to_u64(value: usize) -> u64 { + match u64::try_from(value) { + Ok(value) => value, + Err(error) => panic!("usize to u64 conversion failed: {error}"), + } +} + +fn usize_to_u8(value: usize) -> u8 { + match u8::try_from(value) { + Ok(value) => value, + Err(error) => panic!("usize to u8 conversion failed: {error}"), + } +} + +criterion_group!(benches, verify_transaction_profile); +criterion_main!(benches); From 7675b74be6ef68d2103936d9551015ae85b800b2 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 22:37:54 +0000 Subject: [PATCH 31/37] perf(consensus): reuse serialized tx for script checks Serialize each non-coinbase transaction once in the bitcoinconsensus verifier path and feed those bytes directly to rust-bitcoin for non-taproot script checks. The current consensus caller passes the input's own scriptSig and witness back into Interpreter::execute, so the cloned-and-mutated transaction was byte-equivalent to the original transaction for this non-taproot path. Keep the interpreter fallback for P2TR+TAPROOT and for no-bitcoinconsensus builds so taproot behavior and disabled-backend behavior stay on the existing code path. Add OP_TRUE/OP_FALSE consensus tests for the direct path. Local benchmark evidence only: - before: verify_transaction_profile/verify_tx_op_true_400_input [384.56 ms 387.83 ms 390.87 ms] - after: verify_transaction_profile/verify_tx_op_true_400_input [354.94 ms 359.52 ms 364.62 ms] This is a synthetic OP_TRUE 400-input fixture; it is not signet/mainnet proof, not a root-cause claim, and not a Bitcoin Core comparison. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --all-targets --no-default-features -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-default-features --no-fail-fast - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-default-features --features kernel -- --include-ignored - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-fail-fast - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --all-targets --no-default-features --features bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-script --no-fail-fast - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-default-features --features bitcoinconsensus --bench verify_transaction_profile - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --workspace --no-fail-fast - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --no-fail-fast --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus Op: compress --- crates/consensus/src/verify_tx.rs | 162 ++++++++++++++++++++++++++---- 1 file changed, 145 insertions(+), 17 deletions(-) diff --git a/crates/consensus/src/verify_tx.rs b/crates/consensus/src/verify_tx.rs index d1b7f8d..88c3ea4 100644 --- a/crates/consensus/src/verify_tx.rs +++ b/crates/consensus/src/verify_tx.rs @@ -1,5 +1,10 @@ use std::collections::BTreeSet; +#[cfg(feature = "bitcoinconsensus")] +use bitcoin::Script; +#[cfg(feature = "bitcoinconsensus")] +use bitcoin::consensus::encode; + use bitcoin_rs_primitives::Tx; use bitcoin_rs_script::{Interpreter, VerifyFlags}; @@ -174,24 +179,21 @@ fn verify_transaction_borrowed_with_locktime_cutoff( }); } - let interpreter = Interpreter; + #[cfg(feature = "bitcoinconsensus")] + let serialized_tx = encode::serialize(tx); + for (input_index, (input, prevout)) in tx.input.iter().zip(input_prevouts.iter()).enumerate() { - let witness = input.witness.to_vec(); - - interpreter - .execute( - prevout.script_pubkey.as_bytes(), - input.script_sig.as_bytes(), - &witness, - flags, - prevout, - tx, - input_index, - ) - .map_err(|error| ConsensusError::Script { - input_index, - reason: error.to_string(), - })?; + #[cfg(feature = "bitcoinconsensus")] + verify_input_script( + input_index, + input, + prevout, + tx, + serialized_tx.as_slice(), + flags, + )?; + #[cfg(not(feature = "bitcoinconsensus"))] + verify_input_script(input_index, input, prevout, tx, flags)?; } let sigop_cost_result = tx.total_sigop_cost(|outpoint| prevouts.lookup(outpoint)); @@ -206,6 +208,70 @@ fn verify_transaction_borrowed_with_locktime_cutoff( Ok(()) } +#[cfg(feature = "bitcoinconsensus")] +fn verify_input_script( + input_index: usize, + input: &bitcoin::TxIn, + prevout: &bitcoin::TxOut, + tx: &bitcoin::Transaction, + serialized_tx: &[u8], + flags: VerifyFlags, +) -> Result<(), ConsensusError> { + let script = Script::from_bytes(prevout.script_pubkey.as_bytes()); + if script.is_p2tr() && flags.contains(VerifyFlags::TAPROOT) { + return verify_input_script_with_interpreter(input_index, input, prevout, tx, flags); + } + + script + .verify_with_flags( + input_index, + prevout.value, + serialized_tx, + flags.consensus_bits(), + ) + .map_err(|error| ConsensusError::Script { + input_index, + reason: format!("script verification failed: {error}"), + }) +} + +#[cfg(not(feature = "bitcoinconsensus"))] +fn verify_input_script( + input_index: usize, + input: &bitcoin::TxIn, + prevout: &bitcoin::TxOut, + tx: &bitcoin::Transaction, + flags: VerifyFlags, +) -> Result<(), ConsensusError> { + verify_input_script_with_interpreter(input_index, input, prevout, tx, flags) +} + +fn verify_input_script_with_interpreter( + input_index: usize, + input: &bitcoin::TxIn, + prevout: &bitcoin::TxOut, + tx: &bitcoin::Transaction, + flags: VerifyFlags, +) -> Result<(), ConsensusError> { + let witness = input.witness.to_vec(); + + Interpreter + .execute( + prevout.script_pubkey.as_bytes(), + input.script_sig.as_bytes(), + &witness, + flags, + prevout, + tx, + input_index, + ) + .map(|_| ()) + .map_err(|error| ConsensusError::Script { + input_index, + reason: error.to_string(), + }) +} + fn total_output_value_borrowed(tx: &bitcoin::Transaction) -> Result { tx.output.iter().try_fold(0u64, |sum, output| { let next = sum @@ -319,6 +385,68 @@ mod tests { ); } + #[cfg(feature = "bitcoinconsensus")] + #[test] + fn non_coinbase_true_script_passes() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([3; 32]), + vout: 0, + }; + let tx = Tx(Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }); + let mut utxos = BTreeMap::new(); + utxos.insert( + outpoint, + TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(1).into_script(), + }, + ); + + assert_eq!( + verify_transaction(&tx, &utxos, 0, VerifyFlags::NONE), + Ok(()) + ); + } + + #[cfg(feature = "bitcoinconsensus")] + #[test] + fn non_coinbase_false_script_reports_script_error() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([5; 32]), + vout: 0, + }; + let tx = Tx(Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }); + let mut utxos = BTreeMap::new(); + utxos.insert( + outpoint, + TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(0).into_script(), + }, + ); + + assert!(matches!( + verify_transaction(&tx, &utxos, 0, VerifyFlags::NONE), + Err(ConsensusError::Script { input_index: 0, .. }) + )); + } + #[test] fn underfunded_transaction_fails_before_script_execution() { let outpoint = OutPoint { From 6f278ae438b93ae5701eba033a4c629cf605a69c Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Tue, 26 May 2026 23:19:47 +0000 Subject: [PATCH 32/37] test(consensus): cover verifier backend branches Add branch-boundary regressions for the serialized bitcoinconsensus verifier path. The tests pin the non-taproot direct serialized transaction input, the P2TR+TAPROOT local fallback, and the feature-disabled interpreter backend path. Evidence: cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check; cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --all-targets --no-default-features -- -D warnings; cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --all-targets --no-default-features --features bitcoinconsensus -- -D warnings; cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-default-features --no-fail-fast; cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-default-features --features bitcoinconsensus --no-fail-fast; cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --no-default-features --features kernel -- --include-ignored. Op: correct Restores: spec:verifier backend branch invariants --- crates/consensus/src/verify_tx.rs | 140 ++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/crates/consensus/src/verify_tx.rs b/crates/consensus/src/verify_tx.rs index 88c3ea4..64dcff2 100644 --- a/crates/consensus/src/verify_tx.rs +++ b/crates/consensus/src/verify_tx.rs @@ -291,6 +291,8 @@ mod tests { use bitcoin::hashes::Hash as _; use bitcoin::script::Builder; + #[cfg(feature = "bitcoinconsensus")] + use bitcoin::secp256k1::{Keypair, Secp256k1, SecretKey}; use bitcoin::{ Amount, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness, absolute, transaction, @@ -447,6 +449,93 @@ mod tests { )); } + #[cfg(feature = "bitcoinconsensus")] + #[test] + fn non_taproot_input_uses_supplied_serialized_tx() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([6; 32]), + vout: 0, + }; + let tx = Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }; + let prevout = TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(1).into_script(), + }; + + assert!(matches!( + super::verify_input_script(0, &tx.input[0], &prevout, &tx, &[], VerifyFlags::NONE), + Err(ConsensusError::Script { input_index: 0, .. }) + )); + } + + #[cfg(feature = "bitcoinconsensus")] + #[test] + fn p2tr_taproot_input_ignores_supplied_serialized_tx_for_local_fallback() { + let Some(fixture) = p2tr_extra_witness_spend(7) else { + panic!("test secret key must produce a taproot spend fixture"); + }; + + assert!(VerifyFlags::MANDATORY.contains(VerifyFlags::TAPROOT)); + match super::verify_input_script( + 0, + &fixture.tx.0.input[0], + &fixture.prevout, + &fixture.tx.0, + &[], + VerifyFlags::MANDATORY, + ) { + Err(ConsensusError::Script { + input_index: 0, + reason, + }) => assert!( + reason.contains("taproot witness stack with 2 elements"), + "unexpected taproot fallback error: {reason}" + ), + other => panic!("expected taproot fallback script error, got {other:?}"), + } + } + + #[cfg(not(feature = "bitcoinconsensus"))] + #[test] + fn non_taproot_input_reaches_interpreter_disabled_backend() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([6; 32]), + vout: 0, + }; + let tx = Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }; + let prevout = TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(1).into_script(), + }; + + match super::verify_input_script(0, &tx.input[0], &prevout, &tx, VerifyFlags::NONE) { + Err(ConsensusError::Script { + input_index: 0, + reason, + }) => assert!( + reason.contains("bitcoinconsensus backend is disabled"), + "unexpected disabled-backend error: {reason}" + ), + other => panic!("expected disabled-backend script error, got {other:?}"), + } + } + #[test] fn underfunded_transaction_fails_before_script_execution() { let outpoint = OutPoint { @@ -552,4 +641,55 @@ mod tests { }], }) } + + #[cfg(feature = "bitcoinconsensus")] + struct SpendFixture { + prevout: TxOut, + tx: Tx, + } + + #[cfg(feature = "bitcoinconsensus")] + fn p2tr_extra_witness_spend(byte: u8) -> Option { + let secp = Secp256k1::new(); + let secret = secret_key(byte)?; + let keypair = Keypair::from_secret_key(&secp, &secret); + let tweaked = bitcoin::key::TapTweak::tap_tweak(keypair, &secp, None); + let (output_key, _) = tweaked.public_parts(); + let prevout = TxOut { + value: Amount::from_sat(50_000), + script_pubkey: ScriptBuf::new_p2tr_tweaked(output_key), + }; + let mut tx = unsigned_spend(byte); + tx.input[0].witness = Witness::from_slice(&[vec![0; 64], vec![0xaa]]); + Some(SpendFixture { + prevout, + tx: Tx(tx), + }) + } + + #[cfg(feature = "bitcoinconsensus")] + fn unsigned_spend(byte: u8) -> Transaction { + Transaction { + version: transaction::Version(2), + lock_time: absolute::LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { + txid: Txid::from_byte_array([byte; 32]), + vout: 0, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(49_000), + script_pubkey: Builder::new().push_int(1).into_script(), + }], + } + } + + #[cfg(feature = "bitcoinconsensus")] + fn secret_key(byte: u8) -> Option { + SecretKey::from_slice(&[byte; 32]).ok() + } } From 53fbb22b87e8324e436302f45b77b3346f75c014 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Wed, 27 May 2026 00:00:51 +0000 Subject: [PATCH 33/37] test(consensus): add core vector verifier benchmark Add one explicit tx_valid-derived workload for the production RustValidator::verify_tx path. The fixture hardcodes the selected row, converts excluded Core flags to STANDARD-minus-excluded flags, and fails closed before Criterion measures it. This keeps the benchmark as one falsifier instead of a partial Core-vector parser. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --benches --features bitcoinconsensus -- -D warnings - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-consensus --bench verify_transaction_profile verify_tx_core_valid_vector -- --test Op: extend --- .../benches/verify_transaction_profile.rs | 123 +++++++++++++++++- 1 file changed, 121 insertions(+), 2 deletions(-) diff --git a/crates/consensus/benches/verify_transaction_profile.rs b/crates/consensus/benches/verify_transaction_profile.rs index eb7ec5e..8ec1467 100644 --- a/crates/consensus/benches/verify_transaction_profile.rs +++ b/crates/consensus/benches/verify_transaction_profile.rs @@ -2,19 +2,32 @@ // PERF: Criterion emits public harness items whose docs are irrelevant to the benchmark report. #![allow(missing_docs)] -use std::{collections::BTreeMap, hint::black_box, time::Duration}; +use std::{collections::BTreeMap, hint::black_box, str::FromStr, time::Duration}; use bitcoin::hashes::Hash as _; use bitcoin::script::Builder; use bitcoin::{ Amount, Network, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness, - absolute, transaction, + absolute, consensus, transaction, }; use bitcoin_rs_consensus::RustValidator; use bitcoin_rs_primitives::Tx; use bitcoin_rs_script::VerifyFlags; use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +const CORE_VECTOR_NAME: &str = + "tx_valid.json line 419: Unknown witness program version without discouragement"; +const CORE_VECTOR_HEIGHT: u32 = 1; +const CORE_VECTOR_PREVOUT_TXID: &str = + "0000000000000000000000000000000000000000000000000000000000000100"; +const CORE_VECTOR_PREVOUTS: &[(u32, &str, u64)] = &[ + (0, "51", 1_000), + (1, "60144c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2_000), + (2, "51", 3_000), +]; +const CORE_VECTOR_TX_HEX: &str = "0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b00000000000001510002483045022100a3cec69b52cba2d2de623ffffffffff1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000"; +const CORE_VECTOR_EXCLUDED_FLAGS: &str = "DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"; + const INPUT_COUNT: usize = 400; const PREVOUT_VALUE: u64 = 50_000; const OUTPUT_VALUE: u64 = 1_000; @@ -55,6 +68,25 @@ fn verify_transaction_profile(c: &mut Criterion) { ); }); + group.bench_function("verify_tx_core_valid_vector", |b| { + b.iter_batched( + core_valid_vector, + |case| { + let result = case.validator.verify_tx( + black_box(&case.tx), + black_box(&case.prevouts), + case.height, + case.flags, + ); + match result { + Ok(()) => black_box(()), + Err(error) => panic!("{CORE_VECTOR_NAME} verification failed: {error}"), + } + }, + BatchSize::SmallInput, + ); + }); + group.finish(); } @@ -111,6 +143,93 @@ fn witness_vec() -> Vec> { .collect() } +fn core_valid_vector() -> VerifyCase { + let txid = match Txid::from_str(CORE_VECTOR_PREVOUT_TXID) { + Ok(txid) => txid, + Err(error) => panic!("{CORE_VECTOR_NAME} prevout txid should parse: {error}"), + }; + + let mut prevouts = BTreeMap::new(); + for &(vout, script_hex, amount) in CORE_VECTOR_PREVOUTS { + let script_bytes = match decode_hex(script_hex) { + Ok(bytes) => bytes, + Err(error) => { + panic!("{CORE_VECTOR_NAME} prevout {vout} script hex should decode: {error}") + } + }; + prevouts.insert( + OutPoint { txid, vout }, + TxOut { + value: Amount::from_sat(amount), + script_pubkey: ScriptBuf::from_bytes(script_bytes), + }, + ); + } + + let tx_bytes = match decode_hex(CORE_VECTOR_TX_HEX) { + Ok(bytes) => bytes, + Err(error) => panic!("{CORE_VECTOR_NAME} transaction hex should decode: {error}"), + }; + let tx = match consensus::deserialize(&tx_bytes) { + Ok(tx) => Tx(tx), + Err(error) => panic!("{CORE_VECTOR_NAME} transaction should deserialize: {error}"), + }; + let flags = core_valid_vector_flags(); + + let case = VerifyCase { + validator: RustValidator::new(Network::Signet), + tx, + prevouts, + height: CORE_VECTOR_HEIGHT, + flags, + }; + if let Err(error) = case + .validator + .verify_tx(&case.tx, &case.prevouts, case.height, case.flags) + { + panic!("{CORE_VECTOR_NAME} should verify before benchmarking: {error}"); + } + case +} + +fn core_valid_vector_flags() -> VerifyFlags { + let excluded = match VerifyFlags::from_core_names(CORE_VECTOR_EXCLUDED_FLAGS) { + Ok(flags) => flags, + Err(error) => panic!("{CORE_VECTOR_NAME} excluded flags should parse: {error}"), + }; + let active = VerifyFlags::from_bits(VerifyFlags::STANDARD.bits() & !excluded.bits()); + assert!( + !active.contains(VerifyFlags::DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM), + "{CORE_VECTOR_NAME} excluded flag must not be active", + ); + assert!( + active.contains(VerifyFlags::WITNESS), + "{CORE_VECTOR_NAME} non-excluded standard flag must remain active", + ); + active +} + +fn decode_hex(hex: &str) -> Result, String> { + let mut bytes = Vec::with_capacity(hex.len() / 2); + let mut chars = hex.chars(); + while let Some(high) = chars.next() { + let low = chars + .next() + .ok_or_else(|| format!("odd-length hex string at high nibble '{high}'"))?; + let high = hex_nibble(high)?; + let low = hex_nibble(low)?; + bytes.push((high << 4) | low); + } + Ok(bytes) +} + +fn hex_nibble(ch: char) -> Result { + let value = ch + .to_digit(16) + .ok_or_else(|| format!("invalid hex character '{ch}'"))?; + u8::try_from(value).map_err(|error| format!("hex nibble conversion failed: {error}")) +} + fn txid(seed: u64) -> Txid { let mut bytes = [0_u8; 32]; bytes[..8].copy_from_slice(&seed.to_le_bytes()); From 1841287bc1dfbcaf9a00bdd4af925692a2181c9f Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Wed, 27 May 2026 00:14:16 +0000 Subject: [PATCH 34/37] fix(node): narrow script verify timing boundary Move the script_verify timer start after verify flag construction so script_verify_us and node.apply_block.script_verify_seconds no longer include compute_verify_flags_us. This preserves the apply/verification behavior and restores attribution exclusivity for profiling output. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs --all-targets --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus -- -D warnings - cargo test --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-node --no-default-features --features rocksdb,fjall,redb,mdbx,bitcoinconsensus --test sync_smoke bounded_apply_profile_replay -- --ignored Op: correct Restores: spec:apply profiling attribution exclusivity --- crates/node/src/apply.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index ebaa497..29bb260 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -227,10 +227,10 @@ pub fn apply_block( metrics::histogram!("node.apply_block.bip113_seconds").record(bip113_dur.as_secs_f64()); bip113_result?; - let script_verify_started = quanta::Instant::now(); let verify_flags_started = quanta::Instant::now(); let verify_flags = compute_verify_flags(handles.network, height, softfork_state); let verify_flags_dur = verify_flags_started.elapsed(); + let script_verify_started = quanta::Instant::now(); let block_transactions_started = quanta::Instant::now(); let script_verify_result = verify_block_transactions(handles, block, height, locktime_cutoff, verify_flags); From 6925b8239fd2acb82da6804f14f994aa716d00d6 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Wed, 27 May 2026 00:42:52 +0000 Subject: [PATCH 35/37] test(utxo): cover all shards in commit benchmark Avalanche the synthetic benchmark seed before deriving txid prefix bytes so the UTXO commit benchmark exercises all 256 shards instead of one modulo-4 residue class. Add a fail-closed distribution assertion before the benchmark summary so the measurement oracle cannot silently regress to the biased generator. This changes the synthetic workload and Criterion baseline; it is not a production performance regression or speed claim. Evidence: - cargo fmt --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml --all -- --check - cargo clippy --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-utxo --benches --no-default-features -- -D warnings - cargo bench --manifest-path /home/alpha/dev/bitcoin-rs/bitcoin-rs/Cargo.toml -p bitcoin-rs-utxo --no-default-features --bench utxo_commit -- --warm-up-time 1 --measurement-time 1 --sample-size 10 Op: correct Restores: spec:UTXO benchmark shard-distribution oracle --- crates/utxo/benches/utxo_commit.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/utxo/benches/utxo_commit.rs b/crates/utxo/benches/utxo_commit.rs index 4dccf5f..c3ab56d 100644 --- a/crates/utxo/benches/utxo_commit.rs +++ b/crates/utxo/benches/utxo_commit.rs @@ -19,7 +19,15 @@ const fn next_u64(state: &mut u64) -> u64 { *state } +fn mix_synthetic_seed(seed: u64) -> u64 { + let mut value = seed.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} + fn txid(seed: u64) -> Hash256 { + let seed = mix_synthetic_seed(seed); let mut bytes = [0_u8; 32]; bytes[..8].copy_from_slice(&seed.to_le_bytes()); bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); @@ -79,6 +87,10 @@ const fn percentile(samples: &[Duration], numerator: usize, denominator: usize) fn print_synthetic_summary() { let mut samples = Vec::with_capacity(9); let (_, _, distribution) = synthetic_case(0x5555_aaaa_ffff_0000); + assert!( + distribution.iter().all(|entries| *entries > 0), + "synthetic txids must exercise every UTXO shard" + ); for seed in 0_u64..9 { let (set, changes, _) = synthetic_case(seed + 1); let start = Instant::now(); From bfb4cbb5e29416185ae79154f82b9e6c15ce8163 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Wed, 27 May 2026 02:27:07 +0000 Subject: [PATCH 36/37] test(coinstats): cover all shards in commit benchmark Restore the synthetic benchmark invariant that generated coinstats UTXO keys exercise every shard instead of leaking raw seed low bytes into shard selection. The benchmark input distribution changes, so prior coinstats Criterion baselines are not apples-to-apples evidence. This commit is benchmark maintenance only; it makes no production speed or readiness claim. Evidence: cargo fmt --all -- --check; cargo clippy -p bitcoin-rs-coinstats --benches --no-default-features -- -D warnings; targeted Criterion bench completed with the shard assertion enabled. Op: correct Restores: spec:synthetic coinstats benchmark exercises all UTXO shards --- .../benches/utxo_commit_coinstats.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs index 76537ae..2af3737 100644 --- a/crates/coinstats/benches/utxo_commit_coinstats.rs +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -265,7 +265,15 @@ const fn next_u64(state: &mut u64) -> u64 { *state } +fn mix_synthetic_seed(seed: u64) -> u64 { + let mut value = seed.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} + fn txid(seed: u64) -> Hash256 { + let seed = mix_synthetic_seed(seed); let mut bytes = [0_u8; 32]; bytes[..8].copy_from_slice(&seed.to_le_bytes()); bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); @@ -336,6 +344,33 @@ fn synthetic_case(seed: u64, listener_kind: ListenerKind) -> (UtxoSet, BlockChan (set, changes) } +fn synthetic_shard_distribution(seed: u64) -> [usize; UtxoKey::SHARD_COUNT] { + let mut rng = seed; + let mut distribution = [0_usize; UtxoKey::SHARD_COUNT]; + + for _ in 0_u64..OP_COUNT { + let spend_seed = next_u64(&mut rng); + let shard = usize::from(UtxoKey::from_txid(&txid(spend_seed)).shard()); + distribution[shard] = distribution[shard].saturating_add(1); + } + + for i in 0_u64..OP_COUNT { + let add_seed = next_u64(&mut rng).wrapping_add(i); + let shard = usize::from(UtxoKey::from_txid(&txid(add_seed)).shard()); + distribution[shard] = distribution[shard].saturating_add(1); + } + + distribution +} + +fn assert_synthetic_shard_coverage() { + let distribution = synthetic_shard_distribution(CASE_SEED); + assert!( + distribution.iter().all(|entries| *entries > 0), + "synthetic coinstats txids must exercise every UTXO shard" + ); +} + fn synthetic_direct_case(seed: u64) -> DirectCase { let mut stats = CoinStats::new(); let mut spends = Vec::with_capacity(usize::try_from(OP_COUNT).unwrap_or(usize::MAX)); @@ -496,6 +531,7 @@ fn bench_direct_muhash_preencoded(group: &mut BenchmarkGroup<'_, WallTime>) { } fn utxo_commit_coinstats(c: &mut Criterion) { + assert_synthetic_shard_coverage(); let mut group = c.benchmark_group("utxo_commit_coinstats"); let block_hash = txid(COMMIT_BLOCK_SEED); From f3e04e2d825ad6d36aad6129c8064205d49c3707 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Wed, 27 May 2026 04:44:56 +0000 Subject: [PATCH 37/37] test(node): gate legacy spend replay on bitcoinconsensus The replay fixture spends a legacy OP_TRUE output. The no-bitcoinconsensus backend only carries the local Taproot key-path fallback, so this fixture has always depended on the bitcoinconsensus script backend. Gate the correctness test and bounded profiling harnesses on the feature instead of letting default node tests fail with a misleading final-tip mismatch at height 100. Evidence: cargo fmt --all -- --check; default exact sync_smoke legacy-spend test filtered out; bitcoinconsensus exact sync_smoke legacy-spend test passed; both bitcoinconsensus exact ignored replay harnesses passed; cargo clippy -p bitcoin-rs-node --test sync_smoke --features bitcoinconsensus -- -D warnings. Op: correct Restores: spec:legacy OP_TRUE replay tests require bitcoinconsensus script backend --- crates/node/tests/sync_smoke.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/node/tests/sync_smoke.rs b/crates/node/tests/sync_smoke.rs index 4b1afe6..d0ba1f0 100644 --- a/crates/node/tests/sync_smoke.rs +++ b/crates/node/tests/sync_smoke.rs @@ -266,6 +266,7 @@ fn tick_buffers_out_of_order_blocks_until_parent_arrives() -> Result<(), Box Result<(), Box> { let fixture = non_coinbase_spend_chain()?; @@ -314,6 +315,7 @@ fn tick_applies_non_coinbase_spend_and_updates_utxo_and_coinstats() } #[test] +#[cfg(feature = "bitcoinconsensus")] #[ignore = "bounded local profiling harness; run explicitly with RUST_LOG=bitcoin_rs_node::apply=info"] fn bounded_apply_profile_replay() -> Result<(), Box> { let _subscriber_already_set = tracing_subscriber::fmt() @@ -342,6 +344,7 @@ fn bounded_apply_profile_replay() -> Result<(), Box> { } #[test] +#[cfg(feature = "bitcoinconsensus")] #[ignore = "bounded local coinstats listener cost harness; run explicitly with RUST_LOG=bitcoin_rs_node::apply=info"] fn bounded_apply_profile_replay_coinstats_listener_cost() -> Result<(), Box> {