diff --git a/Cargo.lock b/Cargo.lock index 13213fc..3b5a730 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,6 +287,7 @@ dependencies = [ "bitcoin-rs-storage", "bitcoin-rs-utxo", "bytemuck", + "criterion", "parking_lot", "proptest", "ruint", @@ -306,6 +307,7 @@ dependencies = [ "bitcoin-rs-primitives", "bitcoin-rs-script", "bitcoinkernel", + "criterion", "secp256k1 0.31.1", "serde", "serde_json", @@ -551,6 +553,7 @@ dependencies = [ "bitcoin", "bitcoin-rs-primitives", "bytemuck", + "criterion", "parking_lot", "proptest", "rayon", diff --git a/bin/bitcoin-rs/tests/gates/g01_headers_only_sync.rs b/bin/bitcoin-rs/tests/gates/g01_headers_only_sync.rs index 659032a..50e9992 100644 --- a/bin/bitcoin-rs/tests/gates/g01_headers_only_sync.rs +++ b/bin/bitcoin-rs/tests/gates/g01_headers_only_sync.rs @@ -1,14 +1,300 @@ //! G1 — Headers-only sync parity. -//! **G1 — Headers-only sync parity.** `bitcoin-rs --headers-only mainnet` → header chain hash matches `bitcoind`'s `getblockhash` for every height 0..tip. +//! **G1 — Headers-only sync parity.** Externally collected bitcoin-rs mainnet active-header-chain hashes match `bitcoind`'s `getblockhash` for every height `0..=tip`. +//! +//! This ignored gate does not run live mainnet header sync itself. It verifies +//! externally collected bitcoin-rs and Bitcoin Core header hash evidence, binds +//! it to the current clean git `HEAD`, and fails closed when that evidence +//! contract is missing, malformed, incomplete, or mismatched. -#![allow(clippy::let_unit_value)] +use std::{collections::BTreeMap, env, fs, process::Command}; -/// Gate G1 manual run instructions: set `BITCOIND_RPC_URL` and -/// `BITCOIND_RPC_COOKIE`, then run -/// `cargo test -p bitcoin-rs --test g01_headers_only_sync -- --ignored --nocapture`. -/// The gate compares every headers-only mainnet height against bitcoind. +const EVIDENCE_HELP: &str = "required G1 evidence env: \ +G1_COMMIT_SHA=, \ +G1_MEASUREMENT_TARGET=mainnet-headers, \ +G1_REFERENCE_IMPL=bitcoind, \ +G1_HASH_TYPE=blockhash, \ +G1_TIP_HEIGHT=, \ +G1_BITCOIN_RS_HEADER_HASHES= \ + or G1_BITCOIN_RS_HEADER_HASHES_FILE=, \ +G1_BITCOIND_BLOCK_HASHES= \ + or G1_BITCOIND_BLOCK_HASHES_FILE="; + +type Samples = BTreeMap; + +struct G1Evidence { + commit_sha: String, + tip_height: u32, + bitcoin_rs_samples: Samples, + bitcoind_samples: Samples, +} + +/// Gate G1 manual run instructions: run +/// `cargo test -p bitcoin-rs --test g01_headers_only_sync -- --ignored --nocapture` +/// with externally collected mainnet active-header-chain hashes from bitcoin-rs +/// and `bitcoin-cli getblockhash ` for every height `0..=tip`. #[test] -#[ignore = "requires live bitcoind mainnet RPC for cross-check"] +#[ignore = "requires externally collected bitcoin-rs and bitcoind mainnet header evidence"] fn headers_only_sync_parity() { - // Compare bitcoin-rs headers-only mainnet block hashes to bitcoind getblockhash for 0..tip. + let evidence = G1Evidence::from_env(); + evidence.assert_samples(); + evidence.report(); +} + +impl G1Evidence { + fn from_env() -> Self { + let commit_sha = required_commit_sha(); + require_literal("G1_MEASUREMENT_TARGET", "mainnet-headers"); + require_literal("G1_REFERENCE_IMPL", "bitcoind"); + require_literal("G1_HASH_TYPE", "blockhash"); + let tip_height = positive_u32("G1_TIP_HEIGHT"); + let bitcoin_rs_samples = samples_from_evidence( + "G1_BITCOIN_RS_HEADER_HASHES", + "G1_BITCOIN_RS_HEADER_HASHES_FILE", + ); + let bitcoind_samples = + samples_from_evidence("G1_BITCOIND_BLOCK_HASHES", "G1_BITCOIND_BLOCK_HASHES_FILE"); + Self { + commit_sha, + tip_height, + bitcoin_rs_samples, + bitcoind_samples, + } + } + + fn assert_samples(&self) { + assert_complete_heights("G1 bitcoin-rs", &self.bitcoin_rs_samples, self.tip_height); + assert_complete_heights("G1 bitcoind", &self.bitcoind_samples, self.tip_height); + assert_eq!( + self.bitcoin_rs_samples, self.bitcoind_samples, + "G1 header parity failed: bitcoin-rs active-header-chain hashes differ from bitcoind getblockhash results", + ); + } + + fn report(&self) { + let commit_sha = &self.commit_sha; + let tip_height = self.tip_height; + let sample_count = self.bitcoin_rs_samples.len(); + println!("G1 header evidence accepted for current git HEAD {commit_sha}"); + println!("tip_height={tip_height}"); + println!("sample_count={sample_count}"); + } +} + +fn assert_complete_heights(label: &str, samples: &Samples, tip_height: u32) { + for height in 0..=tip_height { + assert!( + samples.contains_key(&height), + "{label} samples missing height {height}; expected every height 0..={tip_height}", + ); + } + assert_eq!( + samples.len(), + usize::try_from(tip_height) + .ok() + .and_then(|height| height.checked_add(1)) + .unwrap_or(usize::MAX), + "{label} samples must contain exactly every height 0..={tip_height}", + ); +} + +fn samples_from_evidence(env_name: &str, file_env_name: &str) -> Samples { + let raw = evidence_string(env_name, file_env_name); + let mut samples = Samples::new(); + for entry in raw.split(|ch: char| ch == ',' || ch.is_whitespace()) { + let entry = entry.trim(); + if entry.is_empty() { + continue; + } + let Some((height_raw, hash_raw)) = entry.split_once(':') else { + panic!( + "{env_name} sample {entry:?} must use height:64-lowerhex format; {EVIDENCE_HELP}" + ); + }; + let height = parse_u32(env_name, height_raw); + let hash = hash_raw.trim(); + assert!( + is_lower_hex_hash(hash), + "{env_name} sample at height {height} must be a 64-character lowercase hex block hash, got {hash:?}", + ); + assert!( + samples.insert(height, hash.to_owned()).is_none(), + "{env_name} contains duplicate height {height}", + ); + } + assert!( + !samples.is_empty(), + "{env_name} must contain at least one sample; {EVIDENCE_HELP}", + ); + samples +} + +fn evidence_string(env_name: &str, file_env_name: &str) -> String { + match env::var(env_name) { + Ok(value) if !value.trim().is_empty() => return value, + Ok(_) => panic!("{env_name} must not be empty; {EVIDENCE_HELP}"), + Err(env::VarError::NotUnicode(_)) => { + panic!("{env_name} must be valid UTF-8; {EVIDENCE_HELP}") + } + Err(env::VarError::NotPresent) => {} + } + + let path = match env::var(file_env_name) { + Ok(value) if !value.trim().is_empty() => value, + Ok(_) => panic!("{file_env_name} must not be empty; {EVIDENCE_HELP}"), + Err(env::VarError::NotPresent) => { + panic!("missing {env_name} or {file_env_name}; {EVIDENCE_HELP}") + } + Err(env::VarError::NotUnicode(_)) => { + panic!("{file_env_name} must be valid UTF-8; {EVIDENCE_HELP}") + } + }; + match fs::read_to_string(&path) { + Ok(raw) => raw, + Err(error) => panic!("failed to read {file_env_name} path {path:?}: {error}"), + } +} + +fn required_env(name: &str) -> String { + match env::var(name) { + Ok(value) if !value.trim().is_empty() => value, + Ok(_) => panic!("{name} must not be empty; {EVIDENCE_HELP}"), + Err(env::VarError::NotPresent) => panic!("missing {name}; {EVIDENCE_HELP}"), + Err(env::VarError::NotUnicode(_)) => panic!("{name} must be valid UTF-8; {EVIDENCE_HELP}"), + } +} + +fn required_commit_sha() -> String { + let value = required_env("G1_COMMIT_SHA"); + assert!( + is_lower_hex_sha(&value), + "G1_COMMIT_SHA must be a 40-character lowercase hex commit sha, got {value:?}", + ); + let current_head = current_git_head(); + assert_eq!( + value, current_head, + "G1_COMMIT_SHA must match current git HEAD; evidence {value}, current HEAD {current_head}", + ); + assert_clean_tracked_tree(); + value +} + +fn current_git_head() -> String { + let output = match Command::new("git") + .args([ + "-C", + env!("CARGO_MANIFEST_DIR"), + "rev-parse", + "--verify", + "HEAD", + ]) + .output() + { + Ok(output) => output, + Err(error) => panic!("failed to run git rev-parse for G1_COMMIT_SHA binding: {error}"), + }; + assert!( + output.status.success(), + "git rev-parse HEAD failed while validating G1_COMMIT_SHA: {}", + String::from_utf8_lossy(&output.stderr), + ); + let stdout = match String::from_utf8(output.stdout) { + Ok(stdout) => stdout, + Err(error) => panic!("git rev-parse HEAD did not return UTF-8: {error}"), + }; + let head = stdout.trim().to_owned(); + assert!( + is_lower_hex_sha(&head), + "git rev-parse HEAD returned invalid sha {head:?}", + ); + head +} + +fn assert_clean_tracked_tree() { + let repo_root = current_git_root(); + let output = match Command::new("git") + .args([ + "-C", + repo_root.as_str(), + "status", + "--porcelain=v1", + "--untracked-files=no", + ]) + .output() + { + Ok(output) => output, + Err(error) => panic!("failed to run git status for G1 evidence binding: {error}"), + }; + assert!( + output.status.success(), + "git status failed while validating G1 evidence binding: {}", + String::from_utf8_lossy(&output.stderr), + ); + let status = match String::from_utf8(output.stdout) { + Ok(status) => status, + Err(error) => panic!("git status did not return UTF-8: {error}"), + }; + assert!( + status.trim().is_empty(), + "G1 evidence requires a clean tracked git tree for current HEAD; dirty entries:\n{status}", + ); +} + +fn current_git_root() -> String { + let output = match Command::new("git") + .args([ + "-C", + env!("CARGO_MANIFEST_DIR"), + "rev-parse", + "--show-toplevel", + ]) + .output() + { + Ok(output) => output, + Err(error) => panic!("failed to run git rev-parse --show-toplevel: {error}"), + }; + assert!( + output.status.success(), + "git rev-parse --show-toplevel failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + match String::from_utf8(output.stdout) { + Ok(stdout) => stdout.trim().to_owned(), + Err(error) => panic!("git rev-parse --show-toplevel did not return UTF-8: {error}"), + } +} + +fn require_literal(name: &str, expected: &str) { + let value = required_env(name); + assert_eq!( + value, expected, + "{name} must be {expected:?} for G1 evidence, got {value:?}", + ); +} + +fn positive_u32(name: &str) -> u32 { + let raw = required_env(name); + let value = parse_u32(name, &raw); + assert_ne!(value, 0, "{name} must be positive"); + value +} + +fn parse_u32(name: &str, raw: &str) -> u32 { + match raw.trim().parse::() { + Ok(value) => value, + Err(error) => panic!("{name} must contain a u32 integer, got {raw:?}: {error}"), + } +} + +fn is_lower_hex_sha(value: &str) -> bool { + value.len() == 40 + && value + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) +} + +fn is_lower_hex_hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) } diff --git a/crates/coinstats/Cargo.toml b/crates/coinstats/Cargo.toml index e858979..0c57d20 100644 --- a/crates/coinstats/Cargo.toml +++ b/crates/coinstats/Cargo.toml @@ -32,6 +32,11 @@ redb = ["bitcoin-rs-storage/redb"] mdbx = ["bitcoin-rs-storage/mdbx"] [dev-dependencies] +criterion.workspace = true proptest.workspace = true serde_json.workspace = true tempfile = "3" + +[[bench]] +name = "utxo_commit_coinstats" +harness = false diff --git a/crates/coinstats/benches/utxo_commit_coinstats.rs b/crates/coinstats/benches/utxo_commit_coinstats.rs new file mode 100644 index 0000000..2af3737 --- /dev/null +++ b/crates/coinstats/benches/utxo_commit_coinstats.rs @@ -0,0 +1,585 @@ +//! Synthetic UTXO commit benchmark with and without the coinstats listener. +// PERF: Criterion emits public harness items whose docs are irrelevant to the benchmark report. +#![allow(missing_docs)] + +use std::hint::black_box; + +use bitcoin::{Amount, ScriptBuf, consensus::Encodable}; +use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener, MuHash3072}; +use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; +use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoChangeListener, UtxoKey, UtxoSet}; +use criterion::{ + BatchSize, BenchmarkGroup, Criterion, criterion_group, criterion_main, measurement::WallTime, +}; +use parking_lot::RwLock; +use zerocopy::IntoBytes; + +const PRELOAD_HEIGHT: u32 = 1; +const ADD_HEIGHT: u32 = 2; +const OP_COUNT: u64 = 10_000; +const CASE_SEED: u64 = 0x00ab_cdef; +const COMMIT_BLOCK_SEED: u64 = 0x0012_3456; + +#[derive(Clone, Copy)] +enum ListenerKind { + None, + Noop, + Accounting, + ShardedCounters, + ShardedEncodeOnly, + ShardedMuhashOnly, + ShardedCoinStats, + CoinStats, +} + +struct NoopListener; + +impl UtxoChangeListener for NoopListener { + fn on_insert(&self, _op: &OutPoint, _txout: &TxOut, _height: u32, _coinbase: bool) {} + + fn on_remove(&self, _op: &OutPoint, _txout: &TxOut, _height: u32) {} +} + +#[derive(Default)] +struct AccountingStats { + total_amount: u64, + bogo_size: u64, + utxo_count: u64, +} + +struct AccountingListener { + stats: RwLock, +} + +#[derive(Default)] +struct EncodeOnlyStats { + total_bytes: u64, + operations: u64, +} + +struct ShardedCountersListener { + shards: Vec>, +} + +struct ShardedEncodeOnlyListener { + shards: Vec>, +} + +struct ShardedMuhashOnlyListener { + shards: Vec>, +} + +struct ShardedCoinStatsListener { + shards: Vec>, +} + +struct DirectCase { + stats: CoinStats, + spends: Vec, + adds: Vec, +} + +struct PreEncodedCase { + spend_bytes: Vec>, + add_bytes: Vec>, +} + +impl AccountingListener { + fn new() -> Self { + Self { + stats: RwLock::new(AccountingStats::default()), + } + } +} + +impl ShardedCountersListener { + fn new() -> Self { + let shards = (0..UtxoKey::SHARD_COUNT) + .map(|_| RwLock::new(AccountingStats::default())) + .collect(); + Self { shards } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.shards[index] + } +} + +impl ShardedEncodeOnlyListener { + fn new() -> Self { + let shards = (0..UtxoKey::SHARD_COUNT) + .map(|_| RwLock::new(EncodeOnlyStats::default())) + .collect(); + Self { shards } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.shards[index] + } + + fn encode(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let bytes = bench_coin_hash_bytes(op, txout, height, coinbase); + let byte_count = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + black_box(bytes.as_slice()); + + let mut stats = self.shard(op).write(); + stats.total_bytes = stats.total_bytes.saturating_add(byte_count); + stats.operations = stats.operations.saturating_add(1); + } +} + +impl ShardedMuhashOnlyListener { + fn new() -> Self { + let shards = (0..UtxoKey::SHARD_COUNT) + .map(|_| RwLock::new(MuHash3072::new())) + .collect(); + Self { shards } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.shards[index] + } +} + +impl ShardedCoinStatsListener { + fn new() -> Self { + let shards = (0..UtxoKey::SHARD_COUNT) + .map(|_| RwLock::new(CoinStats::new())) + .collect(); + Self { shards } + } + + fn shard(&self, op: &OutPoint) -> &RwLock { + let index = usize::from(UtxoKey::from_txid(&op.txid).shard()); + &self.shards[index] + } +} + +impl UtxoChangeListener for AccountingListener { + fn on_insert(&self, _op: &OutPoint, txout: &TxOut, _height: u32, _coinbase: bool) { + let mut stats = self.stats.write(); + stats.total_amount = stats.total_amount.saturating_add(txout.value.to_sat()); + stats.bogo_size = stats.bogo_size.saturating_add(simple_bogo_size(txout)); + stats.utxo_count = stats.utxo_count.saturating_add(1); + } + + fn on_remove(&self, _op: &OutPoint, txout: &TxOut, _height: u32) { + let mut stats = self.stats.write(); + stats.total_amount = stats.total_amount.saturating_sub(txout.value.to_sat()); + stats.bogo_size = stats.bogo_size.saturating_sub(simple_bogo_size(txout)); + stats.utxo_count = stats.utxo_count.saturating_sub(1); + } +} + +impl UtxoChangeListener for ShardedCountersListener { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, _height: u32, _coinbase: bool) { + let mut stats = self.shard(op).write(); + stats.total_amount = stats.total_amount.saturating_add(txout.value.to_sat()); + stats.bogo_size = stats.bogo_size.saturating_add(simple_bogo_size(txout)); + stats.utxo_count = stats.utxo_count.saturating_add(1); + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, _height: u32) { + let mut stats = self.shard(op).write(); + stats.total_amount = stats.total_amount.saturating_sub(txout.value.to_sat()); + stats.bogo_size = stats.bogo_size.saturating_sub(simple_bogo_size(txout)); + stats.utxo_count = stats.utxo_count.saturating_sub(1); + } +} + +impl UtxoChangeListener for ShardedEncodeOnlyListener { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.encode(op, txout, height, coinbase); + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { + self.encode(op, txout, height, false); + } + + fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.encode(op, txout, height, coinbase); + } +} + +impl UtxoChangeListener for ShardedMuhashOnlyListener { + fn on_insert(&self, op: &OutPoint, _txout: &TxOut, _height: u32, _coinbase: bool) { + self.shard(op).write().insert(op.as_bytes()); + } + + fn on_remove(&self, op: &OutPoint, _txout: &TxOut, _height: u32) { + self.shard(op).write().remove(op.as_bytes()); + } + + fn on_remove_coin(&self, op: &OutPoint, _txout: &TxOut, _height: u32, _coinbase: bool) { + // This isolates MuHash arithmetic in the commit callback shape; it is not + // measuring CoinStats' exact coin encoding semantics. + self.shard(op).write().remove(op.as_bytes()); + } +} + +impl UtxoChangeListener for ShardedCoinStatsListener { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.shard(op) + .write() + .insert_utxo(op, txout, height, coinbase); + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { + self.shard(op).write().remove_utxo(op, txout, height, false); + } + + fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.shard(op) + .write() + .remove_utxo(op, txout, height, coinbase); + } +} + +fn simple_bogo_size(txout: &TxOut) -> u64 { + let script_len = u64::try_from(txout.script_pubkey.len()).unwrap_or(u64::MAX); + 36_u64 + .saturating_add(4) + .saturating_add(8) + .saturating_add(2) + .saturating_add(script_len) +} + +fn bench_coin_hash_bytes(op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) -> Vec { + let mut out = Vec::with_capacity(36 + 4 + txout.script_pubkey.len() + 16); + out.extend_from_slice(op.as_bytes()); + let coinbase_bit = u32::from(coinbase); + out.extend_from_slice(&((height << 1) | coinbase_bit).to_le_bytes()); + if txout.consensus_encode(&mut out).is_err() { + unreachable!("vec-backed consensus encoder is infallible"); + } + out +} + +const fn next_u64(state: &mut u64) -> u64 { + *state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + *state +} + +fn mix_synthetic_seed(seed: u64) -> u64 { + let mut value = seed.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} + +fn txid(seed: u64) -> Hash256 { + let seed = mix_synthetic_seed(seed); + let mut bytes = [0_u8; 32]; + bytes[..8].copy_from_slice(&seed.to_le_bytes()); + bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); + bytes[16..24].copy_from_slice(&seed.wrapping_mul(0x9e37_79b9_7f4a_7c15).to_le_bytes()); + bytes[24..32].copy_from_slice(&seed.wrapping_add(0xd1b5_4a32_d192_ed03).to_le_bytes()); + Hash256::from_le_bytes(&bytes) +} + +fn txout(seed: u64) -> TxOut { + let mut script = Vec::with_capacity(34); + script.extend_from_slice(&[0x00, 0x20]); + script.extend_from_slice(&txid(seed).to_le_bytes()); + TxOut { + value: Amount::from_sat(5_000 + seed), + script_pubkey: ScriptBuf::from_bytes(script), + } +} + +fn synthetic_case(seed: u64, listener_kind: ListenerKind) -> (UtxoSet, BlockChanges) { + let mut set = UtxoSet::new(); + match listener_kind { + ListenerKind::None => {} + ListenerKind::Noop => set.set_listener(Box::new(NoopListener)), + ListenerKind::Accounting => set.set_listener(Box::new(AccountingListener::new())), + ListenerKind::ShardedCounters => { + set.set_listener(Box::new(ShardedCountersListener::new())); + } + ListenerKind::ShardedEncodeOnly => { + set.set_listener(Box::new(ShardedEncodeOnlyListener::new())); + } + ListenerKind::ShardedMuhashOnly => { + set.set_listener(Box::new(ShardedMuhashOnlyListener::new())); + } + ListenerKind::ShardedCoinStats => { + set.set_listener(Box::new(ShardedCoinStatsListener::new())); + } + ListenerKind::CoinStats => { + set.set_listener(Box::new(CoinStatsListener::new(CoinStats::new()))); + } + } + + let mut preload = BlockChanges::default(); + let mut changes = BlockChanges::default(); + let mut rng = seed; + + for _ in 0_u64..OP_COUNT { + let spend_seed = next_u64(&mut rng); + let outpoint = OutPoint::new(txid(spend_seed), 0); + preload.add(UtxoAdd::new( + outpoint, + txout(spend_seed), + false, + PRELOAD_HEIGHT, + )); + changes.remove(outpoint); + } + + if let Err(error) = set.commit_block(&preload, &txid(seed)) { + panic!("synthetic preload failed: {error}"); + } + + for i in 0_u64..OP_COUNT { + let add_seed = next_u64(&mut rng).wrapping_add(i); + let outpoint = OutPoint::new(txid(add_seed), 0); + changes.add(UtxoAdd::new(outpoint, txout(add_seed), false, ADD_HEIGHT)); + } + + (set, changes) +} + +fn synthetic_shard_distribution(seed: u64) -> [usize; UtxoKey::SHARD_COUNT] { + let mut rng = seed; + let mut distribution = [0_usize; UtxoKey::SHARD_COUNT]; + + for _ in 0_u64..OP_COUNT { + let spend_seed = next_u64(&mut rng); + let shard = usize::from(UtxoKey::from_txid(&txid(spend_seed)).shard()); + distribution[shard] = distribution[shard].saturating_add(1); + } + + for i in 0_u64..OP_COUNT { + let add_seed = next_u64(&mut rng).wrapping_add(i); + let shard = usize::from(UtxoKey::from_txid(&txid(add_seed)).shard()); + distribution[shard] = distribution[shard].saturating_add(1); + } + + distribution +} + +fn assert_synthetic_shard_coverage() { + let distribution = synthetic_shard_distribution(CASE_SEED); + assert!( + distribution.iter().all(|entries| *entries > 0), + "synthetic coinstats txids must exercise every UTXO shard" + ); +} + +fn synthetic_direct_case(seed: u64) -> DirectCase { + let mut stats = CoinStats::new(); + let mut spends = Vec::with_capacity(usize::try_from(OP_COUNT).unwrap_or(usize::MAX)); + let mut adds = Vec::with_capacity(usize::try_from(OP_COUNT).unwrap_or(usize::MAX)); + let mut rng = seed; + + for _ in 0_u64..OP_COUNT { + let spend_seed = next_u64(&mut rng); + let outpoint = OutPoint::new(txid(spend_seed), 0); + let spend = UtxoAdd::new(outpoint, txout(spend_seed), false, PRELOAD_HEIGHT); + stats.insert_utxo(&spend.outpoint, &spend.txout, spend.height, spend.coinbase); + spends.push(spend); + } + + for i in 0_u64..OP_COUNT { + let add_seed = next_u64(&mut rng).wrapping_add(i); + let outpoint = OutPoint::new(txid(add_seed), 0); + adds.push(UtxoAdd::new(outpoint, txout(add_seed), false, ADD_HEIGHT)); + } + + DirectCase { + stats, + spends, + adds, + } +} + +fn synthetic_preencoded_case(seed: u64) -> PreEncodedCase { + let direct = synthetic_direct_case(seed); + let spend_bytes = direct + .spends + .iter() + .map(|spend| { + bench_coin_hash_bytes(&spend.outpoint, &spend.txout, spend.height, spend.coinbase) + }) + .collect(); + let add_bytes = direct + .adds + .iter() + .map(|add| bench_coin_hash_bytes(&add.outpoint, &add.txout, add.height, add.coinbase)) + .collect(); + + PreEncodedCase { + spend_bytes, + add_bytes, + } +} + +fn bench_commit_case( + group: &mut BenchmarkGroup<'_, WallTime>, + name: &'static str, + listener_kind: ListenerKind, + block_hash: &Hash256, +) { + group.bench_function(name, |b| { + b.iter_batched( + || synthetic_case(CASE_SEED, listener_kind), + |(set, changes)| { + if let Err(error) = set.commit_block(black_box(&changes), black_box(block_hash)) { + panic!("synthetic commit failed: {error}"); + } + }, + BatchSize::SmallInput, + ); + }); +} + +fn bench_direct_coinstats(group: &mut BenchmarkGroup<'_, WallTime>) { + group.bench_function("direct_coinstats_insert_remove", |b| { + b.iter_batched( + || synthetic_direct_case(CASE_SEED), + |case| { + let DirectCase { + mut stats, + spends, + adds, + } = case; + for spend in &spends { + stats.remove_utxo(&spend.outpoint, &spend.txout, spend.height, spend.coinbase); + } + for add in &adds { + stats.insert_utxo(&add.outpoint, &add.txout, add.height, add.coinbase); + } + black_box(stats); + }, + BatchSize::SmallInput, + ); + }); +} + +fn bench_direct_encode_only(group: &mut BenchmarkGroup<'_, WallTime>) { + group.bench_function("direct_coinstats_encode_only", |b| { + b.iter_batched( + || synthetic_direct_case(CASE_SEED), + |case| { + let DirectCase { spends, adds, .. } = case; + let mut encoded = Vec::with_capacity(spends.len().saturating_add(adds.len())); + for spend in &spends { + encoded.push(bench_coin_hash_bytes( + &spend.outpoint, + &spend.txout, + spend.height, + spend.coinbase, + )); + } + for add in &adds { + encoded.push(bench_coin_hash_bytes( + &add.outpoint, + &add.txout, + add.height, + add.coinbase, + )); + } + black_box(encoded); + }, + BatchSize::SmallInput, + ); + }); +} + +fn direct_muhash_outpoint_bytes(group: &mut BenchmarkGroup<'_, WallTime>) { + group.bench_function("direct_muhash_outpoint_bytes", |b| { + b.iter_batched( + || synthetic_direct_case(CASE_SEED), + |case| { + let DirectCase { spends, adds, .. } = case; + let mut muhash = MuHash3072::new(); + for spend in &spends { + muhash.remove(spend.outpoint.as_bytes()); + } + for add in &adds { + muhash.insert(add.outpoint.as_bytes()); + } + black_box(muhash); + }, + BatchSize::SmallInput, + ); + }); +} + +fn bench_direct_muhash_preencoded(group: &mut BenchmarkGroup<'_, WallTime>) { + group.bench_function("direct_coinstats_muhash_preencoded", |b| { + b.iter_batched( + || synthetic_preencoded_case(CASE_SEED), + |case| { + let mut muhash = MuHash3072::new(); + for bytes in &case.spend_bytes { + muhash.remove(bytes); + } + for bytes in &case.add_bytes { + muhash.insert(bytes); + } + black_box(muhash); + }, + BatchSize::SmallInput, + ); + }); +} + +fn utxo_commit_coinstats(c: &mut Criterion) { + assert_synthetic_shard_coverage(); + let mut group = c.benchmark_group("utxo_commit_coinstats"); + let block_hash = txid(COMMIT_BLOCK_SEED); + + bench_commit_case(&mut group, "no_listener", ListenerKind::None, &block_hash); + bench_commit_case(&mut group, "noop_listener", ListenerKind::Noop, &block_hash); + bench_commit_case( + &mut group, + "accounting_listener", + ListenerKind::Accounting, + &block_hash, + ); + bench_commit_case( + &mut group, + "sharded_counter_listener", + ListenerKind::ShardedCounters, + &block_hash, + ); + bench_commit_case( + &mut group, + "sharded_encode_only_listener", + ListenerKind::ShardedEncodeOnly, + &block_hash, + ); + bench_commit_case( + &mut group, + "sharded_muhash_only_listener", + ListenerKind::ShardedMuhashOnly, + &block_hash, + ); + bench_commit_case( + &mut group, + "sharded_coinstats_listener", + ListenerKind::ShardedCoinStats, + &block_hash, + ); + bench_commit_case( + &mut group, + "coinstats_listener", + ListenerKind::CoinStats, + &block_hash, + ); + bench_direct_coinstats(&mut group); + bench_direct_encode_only(&mut group); + direct_muhash_outpoint_bytes(&mut group); + bench_direct_muhash_preencoded(&mut group); + + group.finish(); +} + +criterion_group!(benches, utxo_commit_coinstats); +criterion_main!(benches); diff --git a/crates/coinstats/src/stats.rs b/crates/coinstats/src/stats.rs index cc085f2..f52dec7 100644 --- a/crates/coinstats/src/stats.rs +++ b/crates/coinstats/src/stats.rs @@ -1,9 +1,12 @@ use alloc::sync::Arc; -use core::convert::Infallible; +use core::{ + convert::Infallible, + sync::atomic::{AtomicBool, Ordering}, +}; use bitcoin::consensus::Encodable; use bitcoin_rs_primitives::{OutPoint, TxOut}; -use bitcoin_rs_utxo::UtxoChangeListener; +use bitcoin_rs_utxo::{UtxoChangeListener, UtxoKey}; use parking_lot::RwLock; use zerocopy::IntoBytes; @@ -126,7 +129,25 @@ pub enum CoinStatsDecodeError { /// UTXO listener that maintains [`CoinStats`]. #[derive(Clone, Debug)] pub struct CoinStatsListener { - stats: Arc>, + inner: Arc, +} + +#[derive(Debug)] +struct CoinStatsListenerInner { + linearize: RwLock<()>, + base: RwLock, + deltas: [RwLock; UtxoKey::SHARD_COUNT], + // Dirty bits are hints guarded by `linearize`: callbacks set the bit before + // dropping the read guard; snapshot/fold read it under the write guard. + dirty: [AtomicBool; UtxoKey::SHARD_COUNT], +} + +#[derive(Clone, Debug, Default)] +struct CoinStatsDelta { + muhash: MuHash3072, + total_amount: i128, + bogo_size: i128, + utxo_count: i128, } impl CoinStatsListener { @@ -134,40 +155,138 @@ impl CoinStatsListener { #[must_use] pub fn new(stats: CoinStats) -> Self { Self { - stats: Arc::new(RwLock::new(stats)), + inner: Arc::new(CoinStatsListenerInner::new(stats)), } } /// Returns a point-in-time copy of the current stats. #[must_use] pub fn snapshot(&self) -> CoinStats { - self.stats.read().clone() + let _linearized = self.inner.linearize.write(); + self.inner.materialize() } /// Applies a per-block delta to the wrapped stats. pub fn finish_block(&self, height: u32, tx_delta: u64) { - self.stats.write().finish_block(height, tx_delta); + let _linearized = self.inner.linearize.write(); + self.inner.fold_deltas(height, tx_delta); + } +} + +impl CoinStatsListenerInner { + fn new(base: CoinStats) -> Self { + Self { + linearize: RwLock::new(()), + base: RwLock::new(base), + deltas: core::array::from_fn(|_| RwLock::new(CoinStatsDelta::default())), + dirty: core::array::from_fn(|_| AtomicBool::new(false)), + } + } + + fn shard_index(op: &OutPoint) -> usize { + usize::from(UtxoKey::from_txid(&op.txid).shard()) + } + + fn materialize(&self) -> CoinStats { + let mut stats = self.base.read().clone(); + for (delta, dirty) in self.deltas.iter().zip(&self.dirty) { + if dirty.load(Ordering::Relaxed) { + delta.read().apply_to(&mut stats); + } + } + stats + } + + fn fold_deltas(&self, height: u32, tx_delta: u64) { + let mut base = self.base.write(); + for (delta, dirty) in self.deltas.iter().zip(&self.dirty) { + if dirty.load(Ordering::Relaxed) { + let mut delta = delta.write(); + dirty.store(false, Ordering::Relaxed); + delta.apply_to(&mut base); + *delta = CoinStatsDelta::default(); + } + } + base.finish_block(height, tx_delta); + } + + fn insert_utxo(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let index = Self::shard_index(op); + self.deltas[index] + .write() + .insert_utxo(op, txout, height, coinbase); + self.dirty[index].store(true, Ordering::Relaxed); + } + + fn remove_utxo(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let index = Self::shard_index(op); + self.deltas[index] + .write() + .remove_utxo(op, txout, height, coinbase); + self.dirty[index].store(true, Ordering::Relaxed); + } +} + +impl CoinStatsDelta { + fn insert_utxo(&mut self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let encoded = coin_hash_bytes(op, txout, height, coinbase); + self.muhash.insert(&encoded); + self.total_amount = self + .total_amount + .saturating_add(i128::from(txout.value.to_sat())); + self.bogo_size = self.bogo_size.saturating_add(i128::from(bogo_size(txout))); + self.utxo_count = self.utxo_count.saturating_add(1); + } + + fn remove_utxo(&mut self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + let encoded = coin_hash_bytes(op, txout, height, coinbase); + self.muhash.remove(&encoded); + self.total_amount = self + .total_amount + .saturating_sub(i128::from(txout.value.to_sat())); + self.bogo_size = self.bogo_size.saturating_sub(i128::from(bogo_size(txout))); + self.utxo_count = self.utxo_count.saturating_sub(1); + } + + fn apply_to(&self, stats: &mut CoinStats) { + stats.muhash.combine(&self.muhash); + apply_signed_delta(&mut stats.total_amount, self.total_amount); + apply_signed_delta(&mut stats.bogo_size, self.bogo_size); + apply_signed_delta(&mut stats.utxo_count, self.utxo_count); } } impl UtxoChangeListener for CoinStatsListener { fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { - self.stats.write().insert_utxo(op, txout, height, coinbase); + let _linearized = self.inner.linearize.read(); + self.inner.insert_utxo(op, txout, height, coinbase); } fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { - self.stats.write().remove_utxo(op, txout, height, false); + let _linearized = self.inner.linearize.read(); + self.inner.remove_utxo(op, txout, height, false); } fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { - self.stats.write().remove_utxo(op, txout, height, coinbase); + let _linearized = self.inner.linearize.read(); + self.inner.remove_utxo(op, txout, height, coinbase); } fn muhash3072(&self) -> Option<[u8; 384]> { - Some(self.stats.read().muhash.finalize()) + Some(self.snapshot().muhash.finalize()) } } +fn apply_signed_delta(value: &mut u64, delta: i128) { + if let Ok(increment) = u64::try_from(delta) { + *value = value.saturating_add(increment); + return; + } + + let decrement = u64::try_from(delta.saturating_neg()).unwrap_or(u64::MAX); + *value = value.saturating_sub(decrement); +} + fn coin_hash_bytes(op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) -> Vec { let mut out = Vec::with_capacity(OUTPOINT_BYTES + 4 + txout.script_pubkey.len() + 16); out.extend_from_slice(op.as_bytes()); diff --git a/crates/coinstats/tests/snapshot_with_muhash.rs b/crates/coinstats/tests/snapshot_with_muhash.rs index 93b02fe..cf54265 100644 --- a/crates/coinstats/tests/snapshot_with_muhash.rs +++ b/crates/coinstats/tests/snapshot_with_muhash.rs @@ -1,8 +1,19 @@ //! Snapshot trailer integration tests for coinstats. +use std::{ + sync::{ + Arc, Barrier, + atomic::{AtomicBool, Ordering}, + }, + thread, +}; + use bitcoin::{Amount, ScriptBuf}; use bitcoin_rs_coinstats::{CoinStats, CoinStatsListener}; use bitcoin_rs_primitives::{Hash256, OutPoint, TxOut}; -use bitcoin_rs_utxo::{BlockChanges, UtxoAdd, UtxoSet, write_snapshot}; +use bitcoin_rs_utxo::{ + BlockChanges, UndoBatch, UtxoAdd, UtxoChangeListener, UtxoError, UtxoKey, UtxoSet, + write_snapshot, +}; #[test] fn snapshot_trailer_uses_listener_muhash() -> Result<(), Box> { @@ -77,6 +88,243 @@ fn snapshot_trailer_tracks_listener_after_removal() -> Result<(), Box Result<(), Box> { + let outpoint = OutPoint::new(txid(20), 0); + let txout = txout(20); + let height = 17; + let mut set = UtxoSet::new(); + let mut preload = BlockChanges::default(); + preload.add(UtxoAdd::new(outpoint, txout.clone(), false, height)); + set.commit_block(&preload, &txid(200))?; + + let mut base = CoinStats::new(); + base.insert_utxo(&outpoint, &txout, height, false); + let listener = CoinStatsListener::new(base.clone()); + set.set_listener(Box::new(listener.clone())); + + let mut removes = BlockChanges::default(); + removes.remove(outpoint); + set.commit_block(&removes, &txid(201))?; + + let mut expected = base; + expected.remove_utxo(&outpoint, &txout, height, false); + let snapshot = listener.snapshot(); + assert_eq!(snapshot, expected); + assert_eq!(snapshot.height, 0); + assert_eq!(snapshot.tx_count, 0); + Ok(()) +} + +#[test] +fn finish_block_folds_deltas_once_before_next_block() -> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let first_outpoint = OutPoint::new(txid(40), 0); + let second_outpoint = OutPoint::new(txid(41), 0); + let first_txout = txout(40); + let second_txout = txout(41); + + let mut first = BlockChanges::default(); + first.add(UtxoAdd::new(first_outpoint, first_txout.clone(), false, 11)); + set.commit_block(&first, &txid(300))?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&first_outpoint, &first_txout, 11, false); + assert_eq!(listener.snapshot(), expected); + + listener.finish_block(11, 1); + expected.finish_block(11, 1); + assert_eq!(listener.snapshot(), expected); + assert_eq!(listener.snapshot(), expected); + + let mut second = BlockChanges::default(); + second.add(UtxoAdd::new( + second_outpoint, + second_txout.clone(), + true, + 12, + )); + set.commit_block(&second, &txid(301))?; + + expected.insert_utxo(&second_outpoint, &second_txout, 12, true); + assert_eq!(listener.snapshot(), expected); + + listener.finish_block(12, 1); + expected.finish_block(12, 1); + assert_eq!(listener.snapshot(), expected); + Ok(()) +} + +#[test] +fn listener_muhash_matches_snapshot_after_remove_and_readd() +-> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let outpoint = OutPoint::new(txid(60), 0); + let original = txout(60); + let replacement = txout(61); + + let mut add = BlockChanges::default(); + add.add(UtxoAdd::new(outpoint, original.clone(), false, 21)); + set.commit_block(&add, &txid(400))?; + + let mut remove = BlockChanges::default(); + remove.remove(outpoint); + set.commit_block(&remove, &txid(401))?; + + let mut readd = BlockChanges::default(); + readd.add(UtxoAdd::new(outpoint, replacement.clone(), true, 22)); + set.commit_block(&readd, &txid(402))?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&outpoint, &original, 21, false); + expected.remove_utxo(&outpoint, &original, 21, false); + expected.insert_utxo(&outpoint, &replacement, 22, true); + + let snapshot = listener.snapshot(); + let trailer = UtxoChangeListener::muhash3072(&listener).unwrap_or([0_u8; 384]); + assert_eq!(snapshot, expected); + assert_eq!(trailer, snapshot.muhash.finalize()); + assert_ne!(trailer, [0_u8; 384]); + Ok(()) +} + +#[test] +fn failed_block_prevalidation_leaves_listener_clean_for_retry() +-> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let valid_outpoint = OutPoint::new(txid(70), 0); + let valid_txout = txout(70); + let invalid_outpoint = OutPoint::new(txid(71), 0); + let invalid_txout = TxOut { + value: Amount::from_sat(71_000), + script_pubkey: ScriptBuf::from_bytes(vec![0x51; usize::from(u16::MAX) + 1]), + }; + + let mut failed = BlockChanges::default(); + failed.add(UtxoAdd::new(valid_outpoint, valid_txout.clone(), false, 31)); + failed.add(UtxoAdd::new(invalid_outpoint, invalid_txout, false, 31)); + let error = match set.commit_block(&failed, &txid(500)) { + Ok(()) => return Err("oversized script block unexpectedly committed".into()), + Err(error) => error, + }; + + assert!(matches!(error, UtxoError::ScriptTooLarge { .. })); + assert_eq!(set.get(&valid_outpoint), None); + assert_eq!(listener.snapshot(), CoinStats::new()); + + let mut retry = BlockChanges::default(); + retry.add(UtxoAdd::new(valid_outpoint, valid_txout.clone(), false, 31)); + set.commit_block(&retry, &txid(501))?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&valid_outpoint, &valid_txout, 31, false); + assert_eq!(set.get(&valid_outpoint), Some(valid_txout)); + assert_eq!(listener.snapshot(), expected); + Ok(()) +} + +#[test] +fn empty_block_commit_preserves_listener_and_set() -> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let first_outpoint = OutPoint::new(txid(72), 0); + let second_outpoint = OutPoint::new(txid(73), 0); + let first_txout = txout(72); + let second_txout = txout(73); + + let mut first = BlockChanges::default(); + first.add(UtxoAdd::new(first_outpoint, first_txout.clone(), true, 32)); + set.commit_block(&first, &txid(502))?; + listener.finish_block(32, 1); + + let mut second = BlockChanges::default(); + second.add(UtxoAdd::new( + second_outpoint, + second_txout.clone(), + false, + 33, + )); + set.commit_block(&second, &txid(503))?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&first_outpoint, &first_txout, 32, true); + expected.finish_block(32, 1); + expected.insert_utxo(&second_outpoint, &second_txout, 33, false); + assert_eq!(listener.snapshot(), expected); + + set.commit_block(&BlockChanges::default(), &txid(504))?; + + assert_eq!(set.len(), 2); + assert_eq!(set.get(&first_outpoint), Some(first_txout)); + assert_eq!(set.get(&second_outpoint), Some(second_txout)); + assert_eq!(listener.snapshot(), expected); + Ok(()) +} + +#[test] +fn undo_block_reverses_unfinished_listener_deltas() -> Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + set.set_listener(Box::new(listener.clone())); + + let spent_outpoint = OutPoint::new(txid(80), 0); + let spent_txout = txout(80); + let created_outpoint = OutPoint::new(txid(81), 0); + let created_txout = txout(81); + + let mut preload = BlockChanges::default(); + preload.add(UtxoAdd::new(spent_outpoint, spent_txout.clone(), false, 40)); + set.commit_block(&preload, &txid(600))?; + + let mut connected = BlockChanges::default(); + connected.remove(spent_outpoint); + connected.add(UtxoAdd::new( + created_outpoint, + created_txout.clone(), + true, + 41, + )); + set.commit_block(&connected, &txid(601))?; + + let mut after_connect = CoinStats::new(); + after_connect.insert_utxo(&spent_outpoint, &spent_txout, 40, false); + after_connect.remove_utxo(&spent_outpoint, &spent_txout, 40, false); + after_connect.insert_utxo(&created_outpoint, &created_txout, 41, true); + assert_eq!(listener.snapshot(), after_connect); + + let mut undo = UndoBatch::default(); + undo.restore(UtxoAdd::new(spent_outpoint, spent_txout.clone(), false, 40)); + undo.remove(created_outpoint); + set.undo_block(&undo)?; + + let mut expected = CoinStats::new(); + expected.insert_utxo(&spent_outpoint, &spent_txout, 40, false); + expected.remove_utxo(&spent_outpoint, &spent_txout, 40, false); + expected.insert_utxo(&created_outpoint, &created_txout, 41, true); + expected.insert_utxo(&spent_outpoint, &spent_txout, 40, false); + expected.remove_utxo(&created_outpoint, &created_txout, 41, true); + let mut canonical = CoinStats::new(); + canonical.insert_utxo(&spent_outpoint, &spent_txout, 40, false); + let snapshot = listener.snapshot(); + assert_eq!(set.get(&spent_outpoint), Some(spent_txout)); + assert_eq!(set.get(&created_outpoint), None); + assert_eq!(snapshot, expected); + assert_eq!(snapshot.muhash.finalize(), canonical.muhash.finalize()); + Ok(()) +} + #[test] fn listener_tracks_duplicate_txid_overwrite() -> Result<(), Box> { let listener = CoinStatsListener::new(CoinStats::new()); @@ -108,6 +356,117 @@ fn listener_tracks_duplicate_txid_overwrite() -> Result<(), Box Result<(), Box> { + let listener = CoinStatsListener::new(CoinStats::new()); + let mut set = UtxoSet::new(); + let gate = Arc::new(AtomicBool::new(false)); + let barrier = Arc::new(Barrier::new(2)); + set.set_listener(Box::new(FirstInsertGate { + inner: listener.clone(), + gate: Arc::clone(&gate), + barrier: Arc::clone(&barrier), + })); + + let first_txid = txid(1); + let second_txid = txid(257); + assert_eq!( + UtxoKey::from_txid(&first_txid).shard(), + UtxoKey::from_txid(&second_txid).shard() + ); + + let first_outpoint = OutPoint::new(first_txid, 0); + let second_outpoint = OutPoint::new(second_txid, 0); + let first_txout = txout(1); + let second_txout = txout(257); + let height = 7; + let tx_delta = 2; + + let mut changes = BlockChanges::default(); + changes.add(UtxoAdd::new( + first_outpoint, + first_txout.clone(), + false, + height, + )); + changes.add(UtxoAdd::new( + second_outpoint, + second_txout.clone(), + true, + height, + )); + + let block_hash = txid(1_000); + thread::scope(|scope| { + let commit = scope.spawn(|| set.commit_block(&changes, &block_hash)); + + barrier.wait(); + let mid_block = listener.snapshot(); + let mut expected_mid_block = CoinStats::new(); + expected_mid_block.insert_utxo(&first_outpoint, &first_txout, height, false); + assert_eq!(mid_block, expected_mid_block); + assert_eq!(mid_block.height, 0); + assert_eq!(mid_block.tx_count, 0); + + barrier.wait(); + match commit.join() { + Ok(result) => result, + Err(payload) => std::panic::resume_unwind(payload), + } + })?; + + let post_commit = listener.snapshot(); + let mut expected_post_commit = CoinStats::new(); + expected_post_commit.insert_utxo(&first_outpoint, &first_txout, height, false); + expected_post_commit.insert_utxo(&second_outpoint, &second_txout, height, true); + assert_eq!(post_commit, expected_post_commit); + assert_eq!(post_commit.height, 0); + assert_eq!(post_commit.tx_count, 0); + + listener.finish_block(height, tx_delta); + let final_snapshot = listener.snapshot(); + let mut expected_final = expected_post_commit; + expected_final.finish_block(height, tx_delta); + assert_eq!(final_snapshot, expected_final); + assert_eq!(final_snapshot.total_amount, post_commit.total_amount); + assert_eq!(final_snapshot.bogo_size, post_commit.bogo_size); + assert_eq!(final_snapshot.utxo_count, post_commit.utxo_count); + Ok(()) +} + +struct FirstInsertGate { + inner: CoinStatsListener, + gate: Arc, + barrier: Arc, +} + +impl UtxoChangeListener for FirstInsertGate { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.inner.on_insert(op, txout, height, coinbase); + if self + .gate + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + self.barrier.wait(); + self.barrier.wait(); + } + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { + self.inner.on_remove(op, txout, height); + } + + fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.inner.on_remove_coin(op, txout, height, coinbase); + } + + fn muhash3072(&self) -> Option<[u8; 384]> { + self.inner.muhash3072() + } +} + fn txout(index: u32) -> TxOut { TxOut { value: Amount::from_sat(50_000 + u64::from(index)), diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml index cb3e5b5..0d327e8 100644 --- a/crates/consensus/Cargo.toml +++ b/crates/consensus/Cargo.toml @@ -30,3 +30,11 @@ tracing.workspace = true serde.workspace = true serde_json.workspace = true bitcoinkernel = { workspace = true, optional = true } + +[dev-dependencies] +criterion.workspace = true + +[[bench]] +name = "verify_transaction_profile" +harness = false +required-features = ["bitcoinconsensus"] diff --git a/crates/consensus/benches/verify_transaction_profile.rs b/crates/consensus/benches/verify_transaction_profile.rs new file mode 100644 index 0000000..8ec1467 --- /dev/null +++ b/crates/consensus/benches/verify_transaction_profile.rs @@ -0,0 +1,257 @@ +//! Diagnostic benchmark for the production transaction verifier path. +// PERF: Criterion emits public harness items whose docs are irrelevant to the benchmark report. +#![allow(missing_docs)] + +use std::{collections::BTreeMap, hint::black_box, str::FromStr, time::Duration}; + +use bitcoin::hashes::Hash as _; +use bitcoin::script::Builder; +use bitcoin::{ + Amount, Network, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness, + absolute, consensus, transaction, +}; +use bitcoin_rs_consensus::RustValidator; +use bitcoin_rs_primitives::Tx; +use bitcoin_rs_script::VerifyFlags; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; + +const CORE_VECTOR_NAME: &str = + "tx_valid.json line 419: Unknown witness program version without discouragement"; +const CORE_VECTOR_HEIGHT: u32 = 1; +const CORE_VECTOR_PREVOUT_TXID: &str = + "0000000000000000000000000000000000000000000000000000000000000100"; +const CORE_VECTOR_PREVOUTS: &[(u32, &str, u64)] = &[ + (0, "51", 1_000), + (1, "60144c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2_000), + (2, "51", 3_000), +]; +const CORE_VECTOR_TX_HEX: &str = "0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b00000000000001510002483045022100a3cec69b52cba2d2de623ffffffffff1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000"; +const CORE_VECTOR_EXCLUDED_FLAGS: &str = "DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"; + +const INPUT_COUNT: usize = 400; +const PREVOUT_VALUE: u64 = 50_000; +const OUTPUT_VALUE: u64 = 1_000; +const SAMPLE_SIZE: usize = 10; +const MEASUREMENT_SECONDS: u64 = 5; +const WITNESS_ITEM_COUNT: usize = 4; +const WITNESS_ITEM_LEN: usize = 72; + +struct VerifyCase { + validator: RustValidator, + tx: Tx, + prevouts: BTreeMap, + height: u32, + flags: VerifyFlags, +} + +fn verify_transaction_profile(c: &mut Criterion) { + let mut group = c.benchmark_group("verify_transaction_profile"); + group.sample_size(SAMPLE_SIZE); + group.measurement_time(Duration::from_secs(MEASUREMENT_SECONDS)); + + group.bench_function("verify_tx_op_true_400_input", |b| { + b.iter_batched( + verify_case, + |case| { + let result = case.validator.verify_tx( + black_box(&case.tx), + black_box(&case.prevouts), + case.height, + case.flags, + ); + match result { + Ok(()) => black_box(()), + Err(error) => panic!("transaction verification failed: {error}"), + } + }, + BatchSize::SmallInput, + ); + }); + + group.bench_function("verify_tx_core_valid_vector", |b| { + b.iter_batched( + core_valid_vector, + |case| { + let result = case.validator.verify_tx( + black_box(&case.tx), + black_box(&case.prevouts), + case.height, + case.flags, + ); + match result { + Ok(()) => black_box(()), + Err(error) => panic!("{CORE_VECTOR_NAME} verification failed: {error}"), + } + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +fn verify_case() -> VerifyCase { + let script_pubkey = op_true_script(); + let mut prevouts = BTreeMap::new(); + let input = (0..INPUT_COUNT) + .map(|index| { + let outpoint = OutPoint { + txid: txid(usize_to_u64(index)), + vout: 0, + }; + prevouts.insert( + outpoint, + TxOut { + value: Amount::from_sat(PREVOUT_VALUE), + script_pubkey: script_pubkey.clone(), + }, + ); + TxIn { + previous_output: outpoint, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::from_slice(&witness_vec()), + } + }) + .collect(); + let tx = Tx(Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input, + output: vec![TxOut { + value: Amount::from_sat(OUTPUT_VALUE), + script_pubkey: ScriptBuf::new(), + }], + }); + + VerifyCase { + validator: RustValidator::new(Network::Signet), + tx, + prevouts, + height: 1, + flags: VerifyFlags::NONE, + } +} + +fn op_true_script() -> ScriptBuf { + Builder::new().push_int(1).into_script() +} + +fn witness_vec() -> Vec> { + (0..WITNESS_ITEM_COUNT) + .map(|index| vec![usize_to_u8(index); WITNESS_ITEM_LEN]) + .collect() +} + +fn core_valid_vector() -> VerifyCase { + let txid = match Txid::from_str(CORE_VECTOR_PREVOUT_TXID) { + Ok(txid) => txid, + Err(error) => panic!("{CORE_VECTOR_NAME} prevout txid should parse: {error}"), + }; + + let mut prevouts = BTreeMap::new(); + for &(vout, script_hex, amount) in CORE_VECTOR_PREVOUTS { + let script_bytes = match decode_hex(script_hex) { + Ok(bytes) => bytes, + Err(error) => { + panic!("{CORE_VECTOR_NAME} prevout {vout} script hex should decode: {error}") + } + }; + prevouts.insert( + OutPoint { txid, vout }, + TxOut { + value: Amount::from_sat(amount), + script_pubkey: ScriptBuf::from_bytes(script_bytes), + }, + ); + } + + let tx_bytes = match decode_hex(CORE_VECTOR_TX_HEX) { + Ok(bytes) => bytes, + Err(error) => panic!("{CORE_VECTOR_NAME} transaction hex should decode: {error}"), + }; + let tx = match consensus::deserialize(&tx_bytes) { + Ok(tx) => Tx(tx), + Err(error) => panic!("{CORE_VECTOR_NAME} transaction should deserialize: {error}"), + }; + let flags = core_valid_vector_flags(); + + let case = VerifyCase { + validator: RustValidator::new(Network::Signet), + tx, + prevouts, + height: CORE_VECTOR_HEIGHT, + flags, + }; + if let Err(error) = case + .validator + .verify_tx(&case.tx, &case.prevouts, case.height, case.flags) + { + panic!("{CORE_VECTOR_NAME} should verify before benchmarking: {error}"); + } + case +} + +fn core_valid_vector_flags() -> VerifyFlags { + let excluded = match VerifyFlags::from_core_names(CORE_VECTOR_EXCLUDED_FLAGS) { + Ok(flags) => flags, + Err(error) => panic!("{CORE_VECTOR_NAME} excluded flags should parse: {error}"), + }; + let active = VerifyFlags::from_bits(VerifyFlags::STANDARD.bits() & !excluded.bits()); + assert!( + !active.contains(VerifyFlags::DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM), + "{CORE_VECTOR_NAME} excluded flag must not be active", + ); + assert!( + active.contains(VerifyFlags::WITNESS), + "{CORE_VECTOR_NAME} non-excluded standard flag must remain active", + ); + active +} + +fn decode_hex(hex: &str) -> Result, String> { + let mut bytes = Vec::with_capacity(hex.len() / 2); + let mut chars = hex.chars(); + while let Some(high) = chars.next() { + let low = chars + .next() + .ok_or_else(|| format!("odd-length hex string at high nibble '{high}'"))?; + let high = hex_nibble(high)?; + let low = hex_nibble(low)?; + bytes.push((high << 4) | low); + } + Ok(bytes) +} + +fn hex_nibble(ch: char) -> Result { + let value = ch + .to_digit(16) + .ok_or_else(|| format!("invalid hex character '{ch}'"))?; + u8::try_from(value).map_err(|error| format!("hex nibble conversion failed: {error}")) +} + +fn txid(seed: u64) -> Txid { + let mut bytes = [0_u8; 32]; + bytes[..8].copy_from_slice(&seed.to_le_bytes()); + bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); + bytes[16..24].copy_from_slice(&seed.wrapping_mul(0x9e37_79b9_7f4a_7c15).to_le_bytes()); + bytes[24..32].copy_from_slice(&seed.wrapping_add(0xd1b5_4a32_d192_ed03).to_le_bytes()); + Txid::from_byte_array(bytes) +} + +fn usize_to_u64(value: usize) -> u64 { + match u64::try_from(value) { + Ok(value) => value, + Err(error) => panic!("usize to u64 conversion failed: {error}"), + } +} + +fn usize_to_u8(value: usize) -> u8 { + match u8::try_from(value) { + Ok(value) => value, + Err(error) => panic!("usize to u8 conversion failed: {error}"), + } +} + +criterion_group!(benches, verify_transaction_profile); +criterion_main!(benches); diff --git a/crates/consensus/src/verify_tx.rs b/crates/consensus/src/verify_tx.rs index d804585..64dcff2 100644 --- a/crates/consensus/src/verify_tx.rs +++ b/crates/consensus/src/verify_tx.rs @@ -1,5 +1,10 @@ use std::collections::BTreeSet; +#[cfg(feature = "bitcoinconsensus")] +use bitcoin::Script; +#[cfg(feature = "bitcoinconsensus")] +use bitcoin::consensus::encode; + use bitcoin_rs_primitives::Tx; use bitcoin_rs_script::{Interpreter, VerifyFlags}; @@ -156,7 +161,7 @@ fn verify_transaction_borrowed_with_locktime_cutoff( } let mut input_value = 0u64; - let interpreter = Interpreter; + let mut input_prevouts = Vec::with_capacity(tx.input.len()); for (input_index, input) in tx.input.iter().enumerate() { let prevout = prevouts .lookup(&input.previous_output) @@ -164,21 +169,7 @@ fn verify_transaction_borrowed_with_locktime_cutoff( input_value = input_value .checked_add(prevout.value.to_sat()) .ok_or(ConsensusError::OutputValueOverflow)?; - let witness = input.witness.to_vec(); - interpreter - .execute( - prevout.script_pubkey.as_bytes(), - input.script_sig.as_bytes(), - &witness, - flags, - &prevout, - tx, - input_index, - ) - .map_err(|error| ConsensusError::Script { - input_index, - reason: error.to_string(), - })?; + input_prevouts.push(prevout); } if input_value < output_value { @@ -188,8 +179,25 @@ fn verify_transaction_borrowed_with_locktime_cutoff( }); } - let sigop_cost = u32::try_from(tx.total_sigop_cost(|outpoint| prevouts.lookup(outpoint))) - .unwrap_or(u32::MAX); + #[cfg(feature = "bitcoinconsensus")] + let serialized_tx = encode::serialize(tx); + + for (input_index, (input, prevout)) in tx.input.iter().zip(input_prevouts.iter()).enumerate() { + #[cfg(feature = "bitcoinconsensus")] + verify_input_script( + input_index, + input, + prevout, + tx, + serialized_tx.as_slice(), + flags, + )?; + #[cfg(not(feature = "bitcoinconsensus"))] + verify_input_script(input_index, input, prevout, tx, flags)?; + } + + let sigop_cost_result = tx.total_sigop_cost(|outpoint| prevouts.lookup(outpoint)); + let sigop_cost = u32::try_from(sigop_cost_result).unwrap_or(u32::MAX); if sigop_cost > MAX_BLOCK_SIGOPS_COST { return Err(ConsensusError::SigopsLimit { cost: sigop_cost, @@ -200,6 +208,70 @@ fn verify_transaction_borrowed_with_locktime_cutoff( Ok(()) } +#[cfg(feature = "bitcoinconsensus")] +fn verify_input_script( + input_index: usize, + input: &bitcoin::TxIn, + prevout: &bitcoin::TxOut, + tx: &bitcoin::Transaction, + serialized_tx: &[u8], + flags: VerifyFlags, +) -> Result<(), ConsensusError> { + let script = Script::from_bytes(prevout.script_pubkey.as_bytes()); + if script.is_p2tr() && flags.contains(VerifyFlags::TAPROOT) { + return verify_input_script_with_interpreter(input_index, input, prevout, tx, flags); + } + + script + .verify_with_flags( + input_index, + prevout.value, + serialized_tx, + flags.consensus_bits(), + ) + .map_err(|error| ConsensusError::Script { + input_index, + reason: format!("script verification failed: {error}"), + }) +} + +#[cfg(not(feature = "bitcoinconsensus"))] +fn verify_input_script( + input_index: usize, + input: &bitcoin::TxIn, + prevout: &bitcoin::TxOut, + tx: &bitcoin::Transaction, + flags: VerifyFlags, +) -> Result<(), ConsensusError> { + verify_input_script_with_interpreter(input_index, input, prevout, tx, flags) +} + +fn verify_input_script_with_interpreter( + input_index: usize, + input: &bitcoin::TxIn, + prevout: &bitcoin::TxOut, + tx: &bitcoin::Transaction, + flags: VerifyFlags, +) -> Result<(), ConsensusError> { + let witness = input.witness.to_vec(); + + Interpreter + .execute( + prevout.script_pubkey.as_bytes(), + input.script_sig.as_bytes(), + &witness, + flags, + prevout, + tx, + input_index, + ) + .map(|_| ()) + .map_err(|error| ConsensusError::Script { + input_index, + reason: error.to_string(), + }) +} + fn total_output_value_borrowed(tx: &bitcoin::Transaction) -> Result { tx.output.iter().try_fold(0u64, |sum, output| { let next = sum @@ -219,6 +291,8 @@ mod tests { use bitcoin::hashes::Hash as _; use bitcoin::script::Builder; + #[cfg(feature = "bitcoinconsensus")] + use bitcoin::secp256k1::{Keypair, Secp256k1, SecretKey}; use bitcoin::{ Amount, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness, absolute, transaction, @@ -313,6 +387,188 @@ mod tests { ); } + #[cfg(feature = "bitcoinconsensus")] + #[test] + fn non_coinbase_true_script_passes() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([3; 32]), + vout: 0, + }; + let tx = Tx(Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }); + let mut utxos = BTreeMap::new(); + utxos.insert( + outpoint, + TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(1).into_script(), + }, + ); + + assert_eq!( + verify_transaction(&tx, &utxos, 0, VerifyFlags::NONE), + Ok(()) + ); + } + + #[cfg(feature = "bitcoinconsensus")] + #[test] + fn non_coinbase_false_script_reports_script_error() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([5; 32]), + vout: 0, + }; + let tx = Tx(Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }); + let mut utxos = BTreeMap::new(); + utxos.insert( + outpoint, + TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(0).into_script(), + }, + ); + + assert!(matches!( + verify_transaction(&tx, &utxos, 0, VerifyFlags::NONE), + Err(ConsensusError::Script { input_index: 0, .. }) + )); + } + + #[cfg(feature = "bitcoinconsensus")] + #[test] + fn non_taproot_input_uses_supplied_serialized_tx() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([6; 32]), + vout: 0, + }; + let tx = Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }; + let prevout = TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(1).into_script(), + }; + + assert!(matches!( + super::verify_input_script(0, &tx.input[0], &prevout, &tx, &[], VerifyFlags::NONE), + Err(ConsensusError::Script { input_index: 0, .. }) + )); + } + + #[cfg(feature = "bitcoinconsensus")] + #[test] + fn p2tr_taproot_input_ignores_supplied_serialized_tx_for_local_fallback() { + let Some(fixture) = p2tr_extra_witness_spend(7) else { + panic!("test secret key must produce a taproot spend fixture"); + }; + + assert!(VerifyFlags::MANDATORY.contains(VerifyFlags::TAPROOT)); + match super::verify_input_script( + 0, + &fixture.tx.0.input[0], + &fixture.prevout, + &fixture.tx.0, + &[], + VerifyFlags::MANDATORY, + ) { + Err(ConsensusError::Script { + input_index: 0, + reason, + }) => assert!( + reason.contains("taproot witness stack with 2 elements"), + "unexpected taproot fallback error: {reason}" + ), + other => panic!("expected taproot fallback script error, got {other:?}"), + } + } + + #[cfg(not(feature = "bitcoinconsensus"))] + #[test] + fn non_taproot_input_reaches_interpreter_disabled_backend() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([6; 32]), + vout: 0, + }; + let tx = Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(50), + script_pubkey: ScriptBuf::new(), + }], + }; + let prevout = TxOut { + value: Amount::from_sat(100), + script_pubkey: Builder::new().push_int(1).into_script(), + }; + + match super::verify_input_script(0, &tx.input[0], &prevout, &tx, VerifyFlags::NONE) { + Err(ConsensusError::Script { + input_index: 0, + reason, + }) => assert!( + reason.contains("bitcoinconsensus backend is disabled"), + "unexpected disabled-backend error: {reason}" + ), + other => panic!("expected disabled-backend script error, got {other:?}"), + } + } + + #[test] + fn underfunded_transaction_fails_before_script_execution() { + let outpoint = OutPoint { + txid: Txid::from_byte_array([4; 32]), + vout: 0, + }; + let tx = Tx(Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: vec![spending_input(outpoint)], + output: vec![TxOut { + value: Amount::from_sat(100), + script_pubkey: ScriptBuf::new(), + }], + }); + let mut utxos = BTreeMap::new(); + utxos.insert( + outpoint, + TxOut { + value: Amount::from_sat(50), + script_pubkey: Builder::new().push_int(0).into_script(), + }, + ); + + assert_eq!( + verify_transaction(&tx, &utxos, 0, VerifyFlags::NONE), + Err(ConsensusError::InputsLessThanOutputs { + input_value: 50, + output_value: 100, + }) + ); + } + #[test] fn verify_transaction_rejects_non_final_height_lock() { let tx = Tx(Transaction { @@ -385,4 +641,55 @@ mod tests { }], }) } + + #[cfg(feature = "bitcoinconsensus")] + struct SpendFixture { + prevout: TxOut, + tx: Tx, + } + + #[cfg(feature = "bitcoinconsensus")] + fn p2tr_extra_witness_spend(byte: u8) -> Option { + let secp = Secp256k1::new(); + let secret = secret_key(byte)?; + let keypair = Keypair::from_secret_key(&secp, &secret); + let tweaked = bitcoin::key::TapTweak::tap_tweak(keypair, &secp, None); + let (output_key, _) = tweaked.public_parts(); + let prevout = TxOut { + value: Amount::from_sat(50_000), + script_pubkey: ScriptBuf::new_p2tr_tweaked(output_key), + }; + let mut tx = unsigned_spend(byte); + tx.input[0].witness = Witness::from_slice(&[vec![0; 64], vec![0xaa]]); + Some(SpendFixture { + prevout, + tx: Tx(tx), + }) + } + + #[cfg(feature = "bitcoinconsensus")] + fn unsigned_spend(byte: u8) -> Transaction { + Transaction { + version: transaction::Version(2), + lock_time: absolute::LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { + txid: Txid::from_byte_array([byte; 32]), + vout: 0, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(49_000), + script_pubkey: Builder::new().push_int(1).into_script(), + }], + } + } + + #[cfg(feature = "bitcoinconsensus")] + fn secret_key(byte: u8) -> Option { + SecretKey::from_slice(&[byte; 32]).ok() + } } diff --git a/crates/node/src/apply.rs b/crates/node/src/apply.rs index 4165a7d..29bb260 100644 --- a/crates/node/src/apply.rs +++ b/crates/node/src/apply.rs @@ -24,6 +24,8 @@ const BIP68_DISABLE_FLAG: u32 = 0x8000_0000; const BIP68_TYPE_FLAG: u32 = 0x0040_0000; const BIP68_MASK: u32 = 0x0000_ffff; const BIP68_TIME_GRANULARITY_SECONDS: u32 = 512; +const SCRIPT_VERIFY_OUTLIER_BLOCK_US: u128 = 100_000; +const SCRIPT_VERIFY_PROFILE_TOP_N: usize = 5; pub(crate) trait PruneBodyStore: Send + Sync { fn persist_block_body( @@ -65,10 +67,10 @@ pub struct ApplyHandles { pub utxo: Arc, /// Shared coinstats listener. pub coin_stats: Arc, - /// Shared best-effort confirmed transaction indexer. - pub tx_index: Arc>>, - /// Shared best-effort compact-filter indexer. - pub filter_index: Arc>, + /// Shared best-effort confirmed transaction indexer when txindex is enabled. + pub tx_index: Option>>>, + /// Shared best-effort compact-filter indexer when blockfilterindex is enabled. + pub filter_index: Option>>, /// Shared mempool. pub mempool: Arc>, /// Shared block records exposed to RPC handlers. @@ -92,8 +94,8 @@ impl ApplyHandles { block_tree: Arc>, utxo: Arc, coin_stats: Arc, - tx_index: Arc>>, - filter_index: Arc>, + tx_index: Option>>>, + filter_index: Option>>, mempool: Arc>, blocks: Arc>>, transactions: Arc>>, @@ -225,14 +227,49 @@ pub fn apply_block( metrics::histogram!("node.apply_block.bip113_seconds").record(bip113_dur.as_secs_f64()); bip113_result?; - let script_verify_started = quanta::Instant::now(); + let verify_flags_started = quanta::Instant::now(); let verify_flags = compute_verify_flags(handles.network, height, softfork_state); + let verify_flags_dur = verify_flags_started.elapsed(); + let script_verify_started = quanta::Instant::now(); + let block_transactions_started = quanta::Instant::now(); let script_verify_result = verify_block_transactions(handles, block, height, locktime_cutoff, verify_flags); + let block_transactions_dur = block_transactions_started.elapsed(); let script_verify_dur = script_verify_started.elapsed(); metrics::histogram!("node.apply_block.script_verify_seconds") .record(script_verify_dur.as_secs_f64()); - script_verify_result?; + let script_verify_profile = script_verify_result?; + if script_verify_dur.as_micros() >= SCRIPT_VERIFY_OUTLIER_BLOCK_US { + let verify_block_transactions_us = block_transactions_dur.as_micros(); + tracing::info!( + height, + %block_hash, + tx_count = block.txdata.len(), + non_coinbase_tx_count = script_verify_profile.non_coinbase_tx_count, + total_input_count = script_verify_profile.total_input_count, + script_verify_us = script_verify_dur.as_micros(), + compute_verify_flags_us = verify_flags_dur.as_micros(), + verify_block_transactions_us, + tx_verify_call_us = script_verify_profile.tx_verify_call_us, + tx_nonverify_us = script_verify_profile.nonverify_us(verify_block_transactions_us), + "apply_block: script verify attribution" + ); + for (rank_index, tx_profile) in script_verify_profile.slow_txs.iter().enumerate() { + tracing::info!( + height, + %block_hash, + tx_rank = rank_index.saturating_add(1), + tx_index = tx_profile.tx_index, + txid = %tx_profile.txid, + input_count = tx_profile.input_count, + output_count = tx_profile.output_count, + tx_verify_us = tx_profile.tx_verify_us, + block_tx_verify_call_us = script_verify_profile.tx_verify_call_us, + block_script_verify_us = script_verify_dur.as_micros(), + "apply_block: script verify transaction attribution" + ); + } + } let coinbase_maturity_started = quanta::Instant::now(); let coinbase_maturity_result = check_coinbase_maturity(handles, block, height); @@ -254,7 +291,11 @@ pub fn apply_block( metrics::histogram!("node.apply_block.bip68_seconds").record(bip68_dur.as_secs_f64()); bip68_result?; - let filter_bytes = compute_basic_filter(block, handles, block_hash, height)?; + let filter_bytes = if handles.filter_index.is_some() { + compute_basic_filter(block, handles, block_hash, height)? + } else { + None + }; let block_bytes = bitcoin::consensus::encode::serialize(block); @@ -310,40 +351,39 @@ pub fn apply_block( metrics::histogram!("node.apply_block.coin_stats_finish_seconds") .record(coin_stats_dur.as_secs_f64()); let tx_index_ingest_started = quanta::Instant::now(); - let tx_index_ingest_result = handles.tx_index.lock().ingest_block(&block_bytes, height); - match tx_index_ingest_result { - Ok(counts) => { - tracing::debug!( - height, - txids = counts.txids, - funding = counts.funding, - spending = counts.spending, - headers = counts.headers, - "tx_index ingested block" - ); - } - Err(error) => { - tracing::warn!( - height, - %error, - "tx_index failed to ingest block; best-effort path continues" - ); + if let Some(tx_index) = &handles.tx_index { + let tx_index_ingest_result = tx_index.lock().ingest_block(&block_bytes, height); + match tx_index_ingest_result { + Ok(counts) => { + tracing::debug!( + height, + txids = counts.txids, + funding = counts.funding, + spending = counts.spending, + headers = counts.headers, + "tx_index ingested block" + ); + } + Err(error) => { + tracing::warn!( + height, + %error, + "tx_index failed to ingest block; best-effort path continues" + ); + } } } let tx_index_ingest_dur = tx_index_ingest_started.elapsed(); metrics::histogram!("node.apply_block.tx_index_ingest_seconds") .record(tx_index_ingest_dur.as_secs_f64()); let filter_started = quanta::Instant::now(); - if let Some(filter_bytes) = filter_bytes { + if let (Some(filter_index), Some(filter_bytes)) = (&handles.filter_index, filter_bytes) { let prev_filter_header = handles .applied_tip .load_full() - .and_then(|tip| handles.filter_index.filter_header(tip.hash).ok().flatten()) + .and_then(|tip| filter_index.filter_header(tip.hash).ok().flatten()) .unwrap_or_default(); - match handles - .filter_index - .put_filter(block_hash, prev_filter_header, &filter_bytes) - { + match filter_index.put_filter(block_hash, prev_filter_header, &filter_bytes) { Ok(filter_header) => { tracing::debug!( height, @@ -520,29 +560,92 @@ fn verify_block_transactions( height: u32, locktime_cutoff: u32, flags: bitcoin_rs_script::VerifyFlags, -) -> core::result::Result<(), ApplyError> { +) -> core::result::Result { // Consensus connects transactions in block order. A later transaction may // spend an output created earlier in the same block. Coinbase outputs enter // this view too, so maturity failures stay in the maturity pass instead of // degrading into bogus missing-prevout script checks. + let mut profile = ScriptVerifyProfile::default(); let mut view = BlockLocalUtxoView::new(Arc::clone(&handles.utxo)); - for tx in &block.txdata { + for (tx_index, tx) in block.txdata.iter().enumerate() { if tx.is_coinbase() { bitcoin_rs_consensus::verify_tx::verify_coinbase_script_sig_size(tx)?; view.add_outputs(tx, height)?; continue; } - bitcoin_rs_consensus::verify_tx::verify_transaction_borrowed_with_mtp( - tx, - &view, - height, - locktime_cutoff, - flags, - )?; + profile.observe_non_coinbase(tx); + let tx_verify_started = quanta::Instant::now(); + let tx_verify_result = + bitcoin_rs_consensus::verify_tx::verify_transaction_borrowed_with_mtp( + tx, + &view, + height, + locktime_cutoff, + flags, + ); + let tx_verify_dur = tx_verify_started.elapsed(); + let tx_verify_us = tx_verify_dur.as_micros(); + profile.add_tx_verify_call_us(tx_verify_us); + tx_verify_result?; + profile.observe_slow_tx(tx_index, tx, tx_verify_us); view.spend_inputs(tx); view.add_outputs(tx, height)?; } - Ok(()) + Ok(profile) +} + +#[derive(Default)] +struct ScriptVerifyProfile { + non_coinbase_tx_count: usize, + total_input_count: usize, + tx_verify_call_us: u128, + slow_txs: Vec, +} + +struct SlowScriptVerifyTxProfile { + tx_index: usize, + txid: Txid, + input_count: usize, + output_count: usize, + tx_verify_us: u128, +} + +impl ScriptVerifyProfile { + fn observe_non_coinbase(&mut self, tx: &bitcoin::Transaction) { + self.non_coinbase_tx_count = self.non_coinbase_tx_count.saturating_add(1); + self.total_input_count = self.total_input_count.saturating_add(tx.input.len()); + } + + fn observe_slow_tx(&mut self, tx_index: usize, tx: &bitcoin::Transaction, tx_verify_us: u128) { + let insert_index = self + .slow_txs + .iter() + .position(|candidate| candidate.tx_verify_us < tx_verify_us) + .unwrap_or(self.slow_txs.len()); + if insert_index >= SCRIPT_VERIFY_PROFILE_TOP_N { + return; + } + + self.slow_txs.insert( + insert_index, + SlowScriptVerifyTxProfile { + tx_index, + txid: tx.compute_txid(), + input_count: tx.input.len(), + output_count: tx.output.len(), + tx_verify_us, + }, + ); + self.slow_txs.truncate(SCRIPT_VERIFY_PROFILE_TOP_N); + } + + fn add_tx_verify_call_us(&mut self, elapsed_us: u128) { + self.tx_verify_call_us = self.tx_verify_call_us.saturating_add(elapsed_us); + } + + fn nonverify_us(&self, verify_block_transactions_us: u128) -> u128 { + verify_block_transactions_us.saturating_sub(self.tx_verify_call_us) + } } struct BlockLocalUtxoView { @@ -1091,6 +1194,7 @@ mod consensus_rule_tests { } #[test] + #[cfg(feature = "bitcoinconsensus")] fn verify_block_transactions_accepts_same_block_spend() -> Result<(), Box> { let base_prevout = bitcoin::OutPoint { @@ -1115,10 +1219,101 @@ mod consensus_rule_tests { ); let block = block_with_transactions(vec![funding_tx, same_block_spend]); - verify_block_transactions(&handles, &block, 2, 0, bitcoin_rs_script::VerifyFlags::NONE)?; + let profile = verify_block_transactions( + &handles, + &block, + 2, + 0, + bitcoin_rs_script::VerifyFlags::NONE, + )?; + assert_eq!(profile.non_coinbase_tx_count, 2); + assert_eq!(profile.total_input_count, 2); + assert!(profile.slow_txs.len() <= SCRIPT_VERIFY_PROFILE_TOP_N); + assert_eq!(profile.slow_txs.len(), 2); + assert!( + profile + .slow_txs + .iter() + .any(|tx_profile| tx_profile.tx_index == 0) + ); + assert!( + profile + .slow_txs + .iter() + .any(|tx_profile| tx_profile.tx_index == 1) + ); Ok(()) } + #[test] + fn script_verify_profile_keeps_bounded_slowest_transactions() { + let mut profile = ScriptVerifyProfile::default(); + let transactions = [ + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x71; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x72; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x73; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x74; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x75; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + spending_transaction_to_script( + bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0x76; 32]), + vout: 0, + }, + Sequence::MAX.to_consensus_u32(), + op_true_script(), + ), + ]; + + let mut tx_verify_us = 0_u128; + for (tx_index, tx) in transactions.iter().enumerate() { + profile.observe_slow_tx(tx_index, tx, tx_verify_us); + tx_verify_us = tx_verify_us.saturating_add(1); + } + + assert_eq!(profile.slow_txs.len(), SCRIPT_VERIFY_PROFILE_TOP_N); + let indexes: Vec = profile + .slow_txs + .iter() + .map(|tx_profile| tx_profile.tx_index) + .collect(); + assert_eq!(indexes, vec![5, 4, 3, 2, 1]); + } + #[test] fn verify_block_transactions_rejects_bad_coinbase_script_sig() { let mut coinbase = coinbase_transaction(0x63); @@ -1133,7 +1328,9 @@ mod consensus_rule_tests { 0, bitcoin_rs_script::VerifyFlags::MANDATORY, ) { - Ok(()) => panic!("bad coinbase scriptSig length must fail transaction verification"), + Ok(_profile) => { + panic!("bad coinbase scriptSig length must fail transaction verification") + } Err(error) => error, }; @@ -1206,6 +1403,7 @@ mod consensus_rule_tests { } #[test] + #[cfg(feature = "bitcoinconsensus")] fn verify_block_transactions_defers_same_block_coinbase_spend_to_maturity() { let mut coinbase = coinbase_transaction(0x65); coinbase.output[0].script_pubkey = op_true_script(); @@ -1857,6 +2055,7 @@ mod consensus_rule_tests { } #[test] + #[cfg(feature = "bitcoinconsensus")] fn apply_block_persists_non_empty_filter_for_valid_same_block_spend() -> Result<(), Box> { let genesis = bitcoin::blockdata::constants::genesis_block(bitcoin::Network::Regtest); @@ -2021,6 +2220,7 @@ mod consensus_rule_tests { } } + #[cfg(feature = "bitcoinconsensus")] fn block_with_prev_hash_and_transactions( prev_blockhash: bitcoin::BlockHash, txdata: Vec, @@ -2042,6 +2242,7 @@ mod consensus_rule_tests { block } + #[cfg(feature = "bitcoinconsensus")] fn mined_block_with_prev_hash_and_transactions( prev_blockhash: bitcoin::BlockHash, txdata: Vec, @@ -2360,8 +2561,8 @@ mod consensus_rule_tests { Arc::new(bitcoin_rs_coinstats::CoinStatsListener::new( bitcoin_rs_coinstats::CoinStats::default(), )), - noop_tx_index(), - filter_index, + Some(noop_tx_index()), + Some(filter_index), Arc::new(RwLock::new(Mempool::new(MempoolLimits::default()))), Arc::new(RwLock::new(Vec::new())), Arc::new(RwLock::new(HashMap::::new())), @@ -2379,8 +2580,8 @@ mod consensus_rule_tests { Arc::new(bitcoin_rs_coinstats::CoinStatsListener::new( bitcoin_rs_coinstats::CoinStats::default(), )), - noop_tx_index(), - noop_filter_index(), + Some(noop_tx_index()), + Some(noop_filter_index()), Arc::new(RwLock::new(Mempool::new(MempoolLimits::default()))), Arc::new(RwLock::new(Vec::new())), Arc::new(RwLock::new(HashMap::::new())), diff --git a/crates/node/src/config.rs b/crates/node/src/config.rs index 9334289..7738145 100644 --- a/crates/node/src/config.rs +++ b/crates/node/src/config.rs @@ -112,6 +112,8 @@ pub struct Config { pub txindex: bool, /// Whether the compact block filter index is enabled. pub blockfilterindex: bool, + /// Whether P2P sync stops at headers and drops block bodies from the sync channel. + pub headers_only: bool, /// Database cache target in MiB. pub dbcache_mb: u64, /// Tracing filter level used when `RUST_LOG` is unset. @@ -158,6 +160,7 @@ impl fmt::Debug for Config { .field("utreexo_mode", &self.utreexo_mode) .field("txindex", &self.txindex) .field("blockfilterindex", &self.blockfilterindex) + .field("headers_only", &self.headers_only) .field("dbcache_mb", &self.dbcache_mb) .field("log_level", &self.log_level) .field("metrics_bind", &self.metrics_bind) @@ -199,6 +202,7 @@ impl Config { utreexo_mode: false, txindex: false, blockfilterindex: false, + headers_only: false, dbcache_mb: DEFAULT_DBCACHE_MB, log_level: DEFAULT_LOG_LEVEL.to_owned(), metrics_bind: None, @@ -272,6 +276,9 @@ impl Config { if self.electrum_tls_cert.is_some() && self.electrum_bind.is_none() { bail!("electrum_tls_cert requires electrum_bind"); } + if self.electrum_bind.is_some() && !self.txindex { + bail!("electrum_bind requires txindex"); + } match (&self.g2_muhash_samples, self.g2_muhash_tip_height) { (Some(_), Some(0)) => bail!("g2_muhash_tip_height must be greater than zero"), (Some(_), None) => bail!("g2_muhash_samples requires g2_muhash_tip_height"), @@ -405,6 +412,9 @@ impl Config { if let Some(blockfilterindex) = layer.blockfilterindex { self.blockfilterindex = blockfilterindex; } + if let Some(headers_only) = layer.headers_only { + self.headers_only = headers_only; + } if let Some(dbcache_mb) = layer.dbcache_mb { self.dbcache_mb = dbcache_mb; } @@ -493,6 +503,12 @@ pub(crate) struct ConfigLayer { pub(crate) txindex: Option, #[arg(long)] pub(crate) blockfilterindex: Option, + #[arg( + long = "headers-only", + num_args = 0..=1, + default_missing_value = "true" + )] + pub(crate) headers_only: Option, #[arg(long = "dbcache-mb")] pub(crate) dbcache_mb: Option, #[arg(long = "log-level")] @@ -558,6 +574,7 @@ impl ConfigLayer { "BITCOIN_RS_UTREEXO_MODE" => layer.utreexo_mode = Some(parse_bool(value)?), "BITCOIN_RS_TXINDEX" => layer.txindex = Some(parse_bool(value)?), "BITCOIN_RS_BLOCKFILTERINDEX" => layer.blockfilterindex = Some(parse_bool(value)?), + "BITCOIN_RS_HEADERS_ONLY" => layer.headers_only = Some(parse_bool(value)?), "BITCOIN_RS_DBCACHE_MB" => layer.dbcache_mb = Some(value.parse()?), "BITCOIN_RS_LOG_LEVEL" => layer.log_level = Some(value.to_owned()), "BITCOIN_RS_METRICS_BIND" => layer.metrics_bind = Some(value.parse()?), diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index 9f4c8cd..0e29ae8 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -67,9 +67,10 @@ fn spawn_electrum_listener( bitcoin_rs_primitives::Network::Signet => bitcoin::Network::Signet, bitcoin_rs_primitives::Network::Regtest => bitcoin::Network::Regtest, }; + let history_reader = state.electrum_history_reader()?; let index = state - .electrum_index_handle() - .with_history_reader(state.electrum_history_reader()) + .electrum_index_handle()? + .with_history_reader(history_reader) .with_network(network); let mempool = bitcoin_rs_electrum::MempoolHandle::from_arc(state.mempool()); let cfg = bitcoin_rs_electrum::ServerConfig::default(); @@ -378,7 +379,7 @@ pub fn run(mut config: Config) -> Result<()> { Some(state.p2p_outbound_sender()), Arc::clone(&banned), Arc::new(parking_lot::RwLock::new(Vec::new())), - Some(state.tx_index()), + state.tx_index(), ); if let Some(prune_service) = state.prune_service() { rpc_context = rpc_context.with_prune_service(prune_service); diff --git a/crates/node/src/state.rs b/crates/node/src/state.rs index 0e740bd..d3758d5 100644 --- a/crates/node/src/state.rs +++ b/crates/node/src/state.rs @@ -603,9 +603,9 @@ pub struct NodeState { storage: NodeStorage, utxo: Arc, coin_stats: Arc, - tx_index: TxIndexHandle, - tx_index_storage: Arc, - filter_index: FilterIndexHandle, + tx_index: Option, + tx_index_storage: Option>, + filter_index: Option, prune_service: Option>, zmq_publisher: Arc, active_zmq_notifications: Vec, @@ -653,9 +653,17 @@ impl NodeState { .context("open G2 MuHash sample writer")? .map(Arc::new); let storage = NodeStorage::open(&config)?; - let (tx_index, tx_index_storage) = open_tx_index(&config)?; - let tx_index_storage = Arc::new(tx_index_storage); - let filter_index = open_filter_index(&config)?; + let (tx_index, tx_index_storage) = if config.txindex { + let (tx_index, tx_index_storage) = open_tx_index(&config)?; + (Some(tx_index), Some(Arc::new(tx_index_storage))) + } else { + (None, None) + }; + let filter_index = if config.blockfilterindex { + Some(open_filter_index(&config)?) + } else { + None + }; let zmq_publications = config.zmq_publications(); let active_zmq_notifications: Vec<_> = zmq_publications .iter() @@ -707,8 +715,8 @@ impl NodeState { block_tree: Arc::clone(&block_tree), utxo: Arc::clone(&utxo), coin_stats: Arc::clone(&coin_stats), - tx_index: Arc::clone(&tx_index), - filter_index: Arc::clone(&filter_index), + tx_index: tx_index.as_ref().map(Arc::clone), + filter_index: filter_index.as_ref().map(Arc::clone), mempool: Arc::clone(&mempool), blocks: Arc::clone(&blocks), transactions: Arc::clone(&transactions), @@ -720,6 +728,7 @@ impl NodeState { Arc::clone(&peer_outbound), Arc::clone(&inbound_headers_rx), Arc::clone(&inbound_blocks_rx), + config.headers_only, )); let prune_service = if config.prune_target_mb > 0 { Some(storage.prune_service(Arc::clone(&blocks), Arc::clone(&transactions))?) @@ -739,7 +748,7 @@ impl NodeState { utxo, coin_stats, tx_index, - tx_index_storage: Arc::clone(&tx_index_storage), + tx_index_storage, filter_index, prune_service, zmq_publisher, @@ -799,8 +808,8 @@ impl NodeState { /// Returns the shared block indexer handle. #[must_use] - pub fn tx_index(&self) -> Arc>> { - Arc::clone(&self.tx_index) + pub fn tx_index(&self) -> Option>>> { + self.tx_index.as_ref().map(Arc::clone) } /// Builds an Electrum `IndexHandle` backed by the live txindex store. @@ -808,25 +817,29 @@ impl NodeState { /// The handle observes the same `KvStore` the writer side ingests into via /// `apply_block`, so `blockchain.block.headers` returns real data once IBD /// is underway. - #[must_use] - pub fn electrum_index_handle(&self) -> bitcoin_rs_electrum::IndexHandle { - self.tx_index_storage.electrum_index_handle() + pub fn electrum_index_handle(&self) -> Result { + self.tx_index_storage + .as_ref() + .map(|storage| storage.electrum_index_handle()) + .context("electrum requires txindex") } /// Builds an Electrum-side history reader wired through the live txindex store /// and the in-memory block log. The handle can be attached to `IndexHandle` /// via `with_history_reader`. - #[must_use] pub fn electrum_history_reader( &self, - ) -> Arc { - self.tx_index_storage.electrum_history_reader(self.blocks()) + ) -> Result> { + self.tx_index_storage + .as_ref() + .map(|storage| storage.electrum_history_reader(self.blocks())) + .context("electrum requires txindex") } /// Returns the shared compact-filter index handle. #[must_use] - pub fn filter_index(&self) -> FilterIndexHandle { - Arc::clone(&self.filter_index) + pub fn filter_index(&self) -> Option { + self.filter_index.as_ref().map(Arc::clone) } /// Returns the manual pruning service when pruning is enabled. @@ -1013,8 +1026,8 @@ impl NodeState { block_tree: Arc::clone(&self.block_tree), utxo: Arc::clone(&self.utxo), coin_stats: Arc::clone(&self.coin_stats), - tx_index: Arc::clone(&self.tx_index), - filter_index: Arc::clone(&self.filter_index), + tx_index: self.tx_index.as_ref().map(Arc::clone), + filter_index: self.filter_index.as_ref().map(Arc::clone), mempool: Arc::clone(&self.mempool), blocks: Arc::clone(&self.blocks), transactions: Arc::clone(&self.transactions), @@ -1114,9 +1127,14 @@ mod tests { let mut config = crate::Config::default_for_network(crate::Network::Regtest); config.data_dir = dir.path().join("node"); config.p2p_listen.clear(); + config.txindex = true; let state = NodeState::open(config)?; - let a = state.tx_index(); - let b = state.tx_index(); + let a = state + .tx_index() + .ok_or_else(|| anyhow::anyhow!("txindex enabled"))?; + let b = state + .tx_index() + .ok_or_else(|| anyhow::anyhow!("txindex enabled"))?; assert!(Arc::ptr_eq(&a, &b), "tx_index handle stable across calls"); Ok(()) } @@ -1142,9 +1160,14 @@ mod tests { let mut config = crate::Config::default_for_network(crate::Network::Regtest); config.data_dir = dir.path().join("node"); config.p2p_listen.clear(); + config.blockfilterindex = true; let state = NodeState::open(config)?; - let a = state.filter_index(); - let b = state.filter_index(); + let a = state + .filter_index() + .ok_or_else(|| anyhow::anyhow!("blockfilterindex enabled"))?; + let b = state + .filter_index() + .ok_or_else(|| anyhow::anyhow!("blockfilterindex enabled"))?; assert!( Arc::ptr_eq(&a, &b), "filter_index handle stable across calls" diff --git a/crates/node/src/sync.rs b/crates/node/src/sync.rs index 55dd28e..03b79f2 100644 --- a/crates/node/src/sync.rs +++ b/crates/node/src/sync.rs @@ -4,7 +4,7 @@ //! and, when a peer reports a longer chain, sends `getheaders` toward //! that peer. Inbound `headers` batches are drained into the shared //! [`bitcoin_rs_chain::BlockTree`]; inbound full blocks are applied through -//! [`crate::apply::apply_block`]. +//! [`crate::apply::apply_block`] unless headers-only mode is enabled. use alloc::sync::Arc; use alloc::vec::Vec; @@ -25,8 +25,6 @@ use parking_lot::{Mutex, RwLock}; const LOCATOR_MAX_ENTRIES: usize = 32; /// Wire protocol version we advertise on outbound `getheaders`. const PROTOCOL_VERSION: u32 = 70_016; -/// Maximum number of block inventory entries we request per tick. -const GETDATA_BATCH_SIZE: usize = 16; /// Time after which a pending getdata is considered stuck and re-requestable. const PENDING_TIMEOUT: Duration = Duration::from_mins(1); /// Maximum number of in-flight getdata requests we'll track per `BlockSync`. @@ -50,6 +48,7 @@ pub struct BlockSync { inbound_blocks_rx: Arc>>, pending_blocks: Arc>>, received_blocks: Arc>>, + headers_only: bool, } impl BlockSync { @@ -61,6 +60,7 @@ impl BlockSync { peer_outbound: Arc>>>, inbound_headers_rx: Arc>>>, inbound_blocks_rx: Arc>>, + headers_only: bool, ) -> Self { Self { handles, @@ -70,6 +70,7 @@ impl BlockSync { inbound_blocks_rx, pending_blocks: Arc::new(Mutex::new(HashMap::new())), received_blocks: Arc::new(Mutex::new(HashMap::new())), + headers_only, } } @@ -80,18 +81,30 @@ impl BlockSync { self.ensure_genesis_tip(); self.drain_inbound_blocks(); - let applied_height = self - .handles - .applied_tip - .load_full() - .map_or(0, |tip| tip.height); - let Some(target) = self.pick_sync_peer(applied_height) else { - tracing::trace!(applied_height, "block sync: no peer above current height"); + let sync_height = self.sync_height(); + let Some(target) = self.pick_sync_peer(sync_height) else { + tracing::trace!(sync_height, "block sync: no peer above current height"); return; }; - self.send_getdata_for_pending_blocks(target.addr); - self.send_getheaders(target.addr, applied_height, target.start_height); + if !self.headers_only { + self.send_getdata_for_pending_blocks(target.addr); + } + self.send_getheaders(target.addr, sync_height, target.start_height); + } + + fn sync_height(&self) -> u32 { + if self.headers_only { + return self + .handles + .chain_tip + .load_full() + .map_or(0, |tip| tip.height); + } + self.handles + .applied_tip + .load_full() + .map_or(0, |tip| tip.height) } fn drain_inbound_headers(&self) { @@ -124,6 +137,17 @@ impl BlockSync { } fn drain_inbound_blocks(&self) { + if self.headers_only { + let dropped = self.drain_and_drop_inbound_blocks(); + if dropped > 0 { + tracing::debug!( + dropped, + "block sync: dropped inbound blocks in headers-only mode" + ); + } + return; + } + let receiver = self.inbound_blocks_rx.lock(); let mut received = 0_usize; while let Ok(block) = receiver.try_recv() { @@ -148,6 +172,16 @@ impl BlockSync { } } + fn drain_and_drop_inbound_blocks(&self) -> usize { + let receiver = self.inbound_blocks_rx.lock(); + let mut dropped = 0_usize; + while receiver.try_recv().is_ok() { + dropped = dropped.saturating_add(1); + } + self.received_blocks.lock().clear(); + dropped + } + fn buffer_received_block(&self, block: bitcoin::Block) { let now = Instant::now(); let hash = Hash256::from_le_bytes(block.block_hash().as_byte_array()); @@ -250,14 +284,13 @@ impl BlockSync { ); return; } - let batch_cap = remaining_budget.min(GETDATA_BATCH_SIZE); - let mut hashes: Vec = Vec::with_capacity(batch_cap); + let mut hashes: Vec = Vec::with_capacity(remaining_budget); let tree = self.handles.block_tree.read(); let Some(mut height) = applied_height.checked_add(1) else { return; }; - while hashes.len() < batch_cap && height <= chain_tip.height { + while hashes.len() < remaining_budget && height <= chain_tip.height { let Some(node_id) = tree.node_at_height_from(chain_tip.tip_id, height) else { break; }; @@ -364,6 +397,11 @@ impl BlockSync { } fn ensure_genesis_tip(&self) { + if self.headers_only { + self.ensure_genesis_header(); + return; + } + if self.handles.applied_tip.load_full().is_some() { return; } @@ -382,6 +420,24 @@ impl BlockSync { } } } + + fn ensure_genesis_header(&self) { + let genesis_hash = self.handles.network.genesis_block_hash(); + let mut tree = self.handles.block_tree.write(); + if tree.lookup(genesis_hash).is_some() { + return; + } + + let genesis = + bitcoin::blockdata::constants::genesis_block(bitcoin_network(self.handles.network)); + match tree.insert_header( + genesis.header, + bitcoin_rs_chain::node::NodeStatus::HeaderValid, + ) { + Ok(_id) => tracing::debug!(%genesis_hash, "block sync: bootstrapped genesis header"), + Err(error) => tracing::warn!(%error, "block sync: failed to bootstrap genesis header"), + } + } } fn bitcoin_network(network: bitcoin_rs_primitives::Network) -> bitcoin::Network { @@ -473,6 +529,7 @@ mod tests { Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, ); let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); peers.write().push(synthetic_peer(addr, 100)); @@ -504,20 +561,20 @@ mod tests { } #[test] - fn tick_sends_getdata_from_next_applied_height_when_gap_exceeds_batch() + fn tick_fills_pending_budget_from_next_applied_height_when_gap_exceeds_budget() -> Result<(), Box> { let mut tree = BlockTree::new(); let genesis = genesis_header(); let genesis_id = tree.insert_node(None, genesis, NodeStatus::HeaderValid)?; let mut tip_id = genesis_id; let mut expected = Vec::new(); - let batch_size = u32::try_from(super::GETDATA_BATCH_SIZE)?; + let pending_budget = u32::try_from(super::PENDING_BUDGET)?; - for height in 1_u32..=batch_size + 4 { + for height in 1_u32..=pending_budget + 4 { let parent_hash = BlockHash::from_byte_array(tree.node(tip_id)?.hash.to_le_bytes()); let header = test_header(parent_hash, height); tip_id = tree.insert_node(Some(tip_id), header, NodeStatus::HeaderValid)?; - if height <= batch_size { + if height <= pending_budget { expected.push(BlockHash::from_byte_array( tree.node(tip_id)?.hash.to_le_bytes(), )); @@ -544,6 +601,7 @@ mod tests { Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, ); let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); peers.write().push(synthetic_peer(addr, 100)); @@ -565,7 +623,30 @@ mod tests { }) .collect::, _>>()?; assert_eq!(requested, expected); - Ok(()) + + let second = rx.try_recv()?; + if !matches!(second, NetworkMessage::GetHeaders(_)) { + return Err(std::io::Error::other("expected first tick getheaders").into()); + } + + sync.tick(); + + let third = rx.try_recv()?; + if !matches!(third, NetworkMessage::GetHeaders(_)) { + return Err(std::io::Error::other("expected second tick getheaders only").into()); + } + match rx.try_recv() { + Ok(NetworkMessage::GetData(_)) => { + Err(std::io::Error::other("second tick requested beyond pending budget").into()) + } + Ok(_) => { + Err(std::io::Error::other("unexpected extra message after second tick").into()) + } + Err(crossbeam_channel::TryRecvError::Empty) => Ok(()), + Err(crossbeam_channel::TryRecvError::Disconnected) => { + Err(std::io::Error::other("outbound channel disconnected").into()) + } + } } #[test] @@ -602,6 +683,7 @@ mod tests { Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, ); let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); peers.write().push(synthetic_peer(addr, 100)); @@ -714,6 +796,107 @@ mod tests { Ok(()) } + #[test] + fn headers_only_tick_sends_getheaders_without_getdata() -> Result<(), Box> + { + let (sync, peers, peer_outbound, block_tree, applied_tip, _expected) = + sync_with_header_chain_for_mode(3, true)?; + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); + peers.write().push(synthetic_peer(addr, 100)); + let (tx, rx) = unbounded::(); + peer_outbound.write().insert(addr, tx); + + sync.tick(); + + assert_headers_only_no_body_apply(&applied_tip, &block_tree, &sync.handles)?; + assert!(sync.pending_blocks.lock().is_empty()); + let first = rx.try_recv()?; + if !matches!(first, NetworkMessage::GetHeaders(_)) { + return Err(std::io::Error::other("expected headers-only getheaders").into()); + } + match rx.try_recv() { + Ok(NetworkMessage::GetData(_)) => { + Err(std::io::Error::other("headers-only tick sent getdata").into()) + } + Ok(_) => Err(std::io::Error::other("unexpected extra headers-only message").into()), + Err(crossbeam_channel::TryRecvError::Empty) => Ok(()), + Err(crossbeam_channel::TryRecvError::Disconnected) => { + Err(std::io::Error::other("outbound channel disconnected").into()) + } + } + } + + #[test] + fn headers_only_tick_uses_header_tip_height_for_progress() + -> Result<(), Box> { + let (sync, peers, peer_outbound, block_tree, applied_tip, _expected) = + sync_with_header_chain_for_mode(3, true)?; + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8333); + peers.write().push(synthetic_peer(addr, 3)); + let (tx, rx) = unbounded::(); + peer_outbound.write().insert(addr, tx); + + sync.tick(); + + assert_headers_only_no_body_apply(&applied_tip, &block_tree, &sync.handles)?; + assert!(sync.pending_blocks.lock().is_empty()); + if !matches!(rx.try_recv(), Err(crossbeam_channel::TryRecvError::Empty)) { + return Err( + std::io::Error::other("headers-only tick requested current header height").into(), + ); + } + Ok(()) + } + + #[test] + fn headers_only_tick_drains_inbound_blocks_without_buffering() + -> Result<(), Box> { + let genesis_block = bitcoin::blockdata::constants::genesis_block(bitcoin::Network::Regtest); + let genesis_hash = bitcoin_rs_primitives::Hash256::from_le_bytes( + genesis_block.block_hash().as_byte_array(), + ); + let block_tree = Arc::new(RwLock::new(BlockTree::new())); + let chain_tip = block_tree.read().tip_handle(); + let applied_tip = Arc::new(ArcSwapOption::empty()); + let peers = Arc::new(RwLock::new(Vec::new())); + let peer_outbound = Arc::new(RwLock::new(HashMap::new())); + let (_inbound_headers_tx, inbound_headers_rx_raw) = unbounded::>(); + let inbound_headers_rx = Arc::new(Mutex::new(inbound_headers_rx_raw)); + let (inbound_blocks_tx, inbound_blocks_rx_raw) = unbounded::(); + let inbound_blocks_rx = Arc::new(Mutex::new(inbound_blocks_rx_raw)); + let handles = apply_handles( + Arc::clone(&chain_tip), + Arc::clone(&applied_tip), + Arc::clone(&block_tree), + ); + let sync = BlockSync::new( + handles, + Arc::clone(&peers), + Arc::clone(&peer_outbound), + inbound_headers_rx, + inbound_blocks_rx, + true, + ); + inbound_blocks_tx.send(genesis_block.clone())?; + sync.received_blocks.lock().insert( + genesis_hash, + super::ReceivedBlock { + block: genesis_block, + received_at: Instant::now(), + }, + ); + + sync.tick(); + + assert_headers_only_no_body_apply(&applied_tip, &block_tree, &sync.handles)?; + assert!(sync.received_blocks.lock().is_empty()); + assert!(matches!( + sync.inbound_blocks_rx.lock().try_recv(), + Err(crossbeam_channel::TryRecvError::Empty), + )); + Ok(()) + } + #[test] fn drain_inbound_blocks_prunes_stale_received_blocks_without_new_arrivals() -> Result<(), Box> { @@ -780,6 +963,59 @@ mod tests { Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, + ); + + Ok(( + sync, + peers, + peer_outbound, + block_tree, + applied_tip, + expected, + )) + } + + fn sync_with_header_chain_for_mode( + height: u32, + headers_only: bool, + ) -> Result> { + let mut tree = BlockTree::new(); + let genesis = genesis_header(); + let genesis_id = tree.insert_node(None, genesis, NodeStatus::HeaderValid)?; + let mut tip_id = genesis_id; + let mut expected = Vec::new(); + + for height in 1_u32..=height { + let parent_hash = BlockHash::from_byte_array(tree.node(tip_id)?.hash.to_le_bytes()); + let header = test_header(parent_hash, height); + tip_id = tree.insert_node(Some(tip_id), header, NodeStatus::HeaderValid)?; + expected.push(BlockHash::from_byte_array( + tree.node(tip_id)?.hash.to_le_bytes(), + )); + } + + let chain_tip = tree.tip_handle(); + let block_tree = Arc::new(RwLock::new(tree)); + let applied_tip = Arc::new(ArcSwapOption::empty()); + let peers = Arc::new(RwLock::new(Vec::new())); + let peer_outbound = Arc::new(RwLock::new(HashMap::new())); + let (_inbound_headers_tx, inbound_headers_rx_raw) = unbounded::>(); + let inbound_headers_rx = Arc::new(Mutex::new(inbound_headers_rx_raw)); + let (_inbound_blocks_tx, inbound_blocks_rx_raw) = unbounded::(); + let inbound_blocks_rx = Arc::new(Mutex::new(inbound_blocks_rx_raw)); + let handles = apply_handles( + Arc::clone(&chain_tip), + Arc::clone(&applied_tip), + Arc::clone(&block_tree), + ); + let sync = BlockSync::new( + handles, + Arc::clone(&peers), + Arc::clone(&peer_outbound), + inbound_headers_rx, + inbound_blocks_rx, + headers_only, ); Ok(( @@ -819,8 +1055,8 @@ mod tests { Arc::new(bitcoin_rs_coinstats::CoinStatsListener::new( bitcoin_rs_coinstats::CoinStats::default(), )), - noop_tx_index(), - noop_filter_index(), + Some(noop_tx_index()), + Some(noop_filter_index()), Arc::new(RwLock::new(Mempool::new(MempoolLimits::default()))), Arc::new(RwLock::new(Vec::new())), Arc::new(RwLock::new(HashMap::::new())), @@ -912,6 +1148,21 @@ mod tests { Ok(()) } + fn assert_headers_only_no_body_apply( + applied_tip: &Arc>, + block_tree: &Arc>, + handles: &ApplyHandles, + ) -> Result<(), Box> { + let genesis_hash = Network::Regtest.genesis_block_hash(); + if applied_tip.load_full().is_some() { + return Err(std::io::Error::other("headers-only mode applied a block body").into()); + } + assert_eq!(block_tree.read().height_of_hash(genesis_hash), Some(0)); + assert!(handles.blocks.read().is_empty()); + assert_eq!(handles.utxo.len(), 0); + Ok(()) + } + fn synthetic_peer(addr: SocketAddr, start_height: i32) -> PeerInfo { PeerInfo { addr, diff --git a/crates/node/tests/config_layered.rs b/crates/node/tests/config_layered.rs index 1227734..fd83f97 100644 --- a/crates/node/tests/config_layered.rs +++ b/crates/node/tests/config_layered.rs @@ -96,6 +96,41 @@ fn cli_can_override_socket_and_vector_fields() -> Result<()> { Ok(()) } +#[test] +fn headers_only_defaults_false_and_layers_enable() -> Result<()> { + let temp = tempfile::tempdir()?; + let toml_path = temp.path().join("node.toml"); + fs::write(&toml_path, "headers_only = true\n")?; + + let default_config = Config::default_for_network(Network::Regtest); + assert!(!default_config.headers_only); + + let toml_config = Config::from_layered_sources( + Some(&toml_path), + None, + core::iter::empty::(), + ["bitcoin-rs-node"], + )?; + assert!(toml_config.headers_only); + + let env_config = Config::from_layered_sources( + None, + None, + [("BITCOIN_RS_HEADERS_ONLY", "true")], + ["bitcoin-rs-node"], + )?; + assert!(env_config.headers_only); + + let cli_config = Config::from_layered_sources( + None, + None, + core::iter::empty::(), + ["bitcoin-rs-node", "--headers-only"], + )?; + assert!(cli_config.headers_only); + Ok(()) +} + #[test] fn zmq_layers_parse_precedence_and_publication_order() -> Result<()> { let temp = tempfile::tempdir()?; diff --git a/crates/node/tests/rpc_wiring.rs b/crates/node/tests/rpc_wiring.rs index 2957b6c..5a9654d 100644 --- a/crates/node/tests/rpc_wiring.rs +++ b/crates/node/tests/rpc_wiring.rs @@ -21,6 +21,8 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { let dir = tempdir()?; let mut config = Config::default(); config.data_dir = dir.path().join("node"); + config.txindex = true; + config.blockfilterindex = true; config.zmqpubhashblock = vec!["inproc://rpc-wiring-zmq-pubhashblock".to_owned()]; config.zmqpubhashblockhwm = Some(21); let state = NodeState::open(config)?; @@ -51,7 +53,7 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { Arc::clone(&transactions), Arc::clone(&utxo), Arc::clone(&coin_stats), - Arc::clone(&filter_index), + filter_index.clone(), Arc::clone(&network), Arc::clone(&mining_template_id), Arc::clone(&peers), @@ -61,7 +63,7 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { p2p_outbound, Arc::clone(&banned), Arc::clone(&added_nodes), - Some(Arc::clone(&tx_index)), + tx_index.clone(), ) .with_zmq_notifications(state.active_zmq_notifications()); @@ -90,11 +92,28 @@ fn rpc_context_shares_arc_identity_with_node_state() -> Result<()> { Arc::ptr_eq(&ctx.coin_stats, &coin_stats), "coin_stats must share identity" ); + let ctx_filter_index = ctx + .filter_index + .as_ref() + .ok_or_else(|| anyhow::anyhow!("filter index must be wired"))?; + let node_filter_index = filter_index + .as_ref() + .ok_or_else(|| anyhow::anyhow!("node filter index must be wired"))?; assert!( - Arc::ptr_eq(&ctx.filter_index, &filter_index), + Arc::ptr_eq(ctx_filter_index, node_filter_index), "filter_index must share identity" ); - assert!(ctx.indexer.is_some(), "indexer handle must be wired"); + let ctx_indexer = ctx + .indexer + .as_ref() + .ok_or_else(|| anyhow::anyhow!("indexer handle must be wired"))?; + let node_tx_index = tx_index + .as_ref() + .ok_or_else(|| anyhow::anyhow!("node txindex must be wired"))?; + assert!( + Arc::ptr_eq(ctx_indexer, node_tx_index), + "indexer handle must share identity" + ); assert!( Arc::ptr_eq(&ctx.network, &network), "network must share identity" diff --git a/crates/node/tests/state_storage.rs b/crates/node/tests/state_storage.rs index bdd15a7..b1378ff 100644 --- a/crates/node/tests/state_storage.rs +++ b/crates/node/tests/state_storage.rs @@ -2,6 +2,7 @@ use anyhow::Result; use bitcoin_rs_node::{Config, Network, state::NodeState}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; #[test] fn opens_storage_backend() -> Result<()> { @@ -17,6 +18,42 @@ fn opens_storage_backend() -> Result<()> { Ok(()) } +#[test] +fn optional_indexes_are_not_opened_when_disabled() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut config = Config::default_for_network(Network::Regtest); + config.data_dir = temp.path().join("node"); + config.p2p_listen.clear(); + config.txindex = false; + config.blockfilterindex = false; + + let state = NodeState::open(config)?; + + assert!(state.tx_index().is_none()); + assert!(state.filter_index().is_none()); + assert!(!state.data_dir().join("txindex").exists()); + assert!(!state.data_dir().join("filters").exists()); + Ok(()) +} + +#[test] +fn electrum_bind_requires_txindex() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut config = Config::default_for_network(Network::Regtest); + config.data_dir = temp.path().join("node"); + config.p2p_listen.clear(); + config.electrum_bind = Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)); + config.txindex = false; + + let error = match config.validate() { + Ok(()) => anyhow::bail!("electrum without txindex unexpectedly validated"), + Err(error) => error, + }; + + assert!(error.to_string().contains("electrum_bind requires txindex")); + Ok(()) +} + fn assert_backend_opens(backend: &str) -> Result<()> { let temp = tempfile::tempdir()?; let mut config = Config::default_for_network(Network::Regtest); diff --git a/crates/node/tests/sync_smoke.rs b/crates/node/tests/sync_smoke.rs index 0517934..d0ba1f0 100644 --- a/crates/node/tests/sync_smoke.rs +++ b/crates/node/tests/sync_smoke.rs @@ -1,6 +1,10 @@ //! Block sync smoke tests. use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::Arc; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; +use std::time::{Duration, Instant}; use arc_swap::ArcSwapOption; use bitcoin::hashes::Hash as _; @@ -17,7 +21,7 @@ use bitcoin_rs_mempool::{Mempool, MempoolLimits}; use bitcoin_rs_node::{BlockSync, Config, Network, apply::ApplyHandles, state::NodeState}; use bitcoin_rs_p2p::{Message, PeerInfo}; use bitcoin_rs_primitives::{Hash256, OutPoint}; -use bitcoin_rs_utxo::UtxoSet; +use bitcoin_rs_utxo::{UtxoChangeListener, UtxoSet}; use crossbeam_channel::unbounded; use hashbrown::HashMap; use parking_lot::{Mutex, RwLock}; @@ -48,6 +52,7 @@ fn tick_sends_getheaders_to_best_peer_above_our_height() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { let fixture = non_coinbase_spend_chain()?; + let outcome = replay_non_coinbase_spend_chain(&fixture, true)?; + + assert_eq!(outcome.applied_height, 102); + assert_eq!( + outcome.applied_hash, + bitcoin_rs_primitives::Hash256::from_le_bytes( + fixture + .blocks + .last() + .ok_or_else(|| std::io::Error::other("missing final block"))? + .block_hash() + .as_byte_array(), + ) + ); + assert!( + outcome + .utxo + .get(&primitive_outpoint(fixture.mature_coinbase_outpoint)) + .is_none(), + "mature coinbase prevout must be removed by the height-101 spend", + ); + assert!( + outcome + .utxo + .get(&primitive_outpoint(fixture.funding_outpoint)) + .is_none(), + "funding prevout must be removed by the height-102 spend", + ); + assert!( + outcome + .utxo + .get(&primitive_outpoint(fixture.spend_outpoint)) + .is_some(), + "height-102 spend output must remain live", + ); + + let block_refs: Vec<&bitcoin::Block> = fixture.blocks.iter().collect(); + assert_eq!( + outcome.coin_stats.snapshot(), + expected_coin_stats(&block_refs)? + ); + Ok(()) +} + +#[test] +#[cfg(feature = "bitcoinconsensus")] +#[ignore = "bounded local profiling harness; run explicitly with RUST_LOG=bitcoin_rs_node::apply=info"] +fn bounded_apply_profile_replay() -> Result<(), Box> { + let _subscriber_already_set = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .is_err(); + let fixture = non_coinbase_spend_chain()?; + + // This profiles apply path branch overhead, not storage-backed index cost. + for use_noop_index_hooks in [false, true] { + let label = if use_noop_index_hooks { + "index_hooks=noop" + } else { + "index_hooks=disabled" + }; + let outcome = replay_non_coinbase_spend_chain(&fixture, use_noop_index_hooks)?; + println!( + "bounded_apply_profile_replay {label} elapsed_ms={} applied_height={} blocks={}", + elapsed_ms(outcome.elapsed), + outcome.applied_height, + fixture.blocks.len(), + ); + } + + Ok(()) +} + +#[test] +#[cfg(feature = "bitcoinconsensus")] +#[ignore = "bounded local coinstats listener cost harness; run explicitly with RUST_LOG=bitcoin_rs_node::apply=info"] +fn bounded_apply_profile_replay_coinstats_listener_cost() -> Result<(), Box> +{ + let _subscriber_already_set = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .is_err(); + let fixture = non_coinbase_spend_chain()?; + + for coin_stats_listener in [ + CoinStatsListenerMode::Attached, + CoinStatsListenerMode::Detached, + ] { + let outcome = replay_non_coinbase_spend_chain_with_coin_stats_listener( + &fixture, + false, + coin_stats_listener, + )?; + let listener_calls = outcome.listener_calls; + println!( + "bounded_apply_profile_replay_coinstats_listener_cost coin_stats_listener={} elapsed_ms={} applied_height={} blocks={} listener_insert_calls={} listener_remove_calls={} listener_remove_coin_calls={} listener_total_calls={}", + coin_stats_listener.label(), + elapsed_ms(outcome.elapsed), + outcome.applied_height, + fixture.blocks.len(), + listener_calls.insert_calls, + listener_calls.remove_calls, + listener_calls.remove_coin_calls, + listener_calls.total_calls(), + ); + } + + Ok(()) +} + +#[cfg(feature = "redb")] +#[test] +#[ignore = "bounded local storage-backed optional-index cost harness; run explicitly"] +fn optional_index_redb_direct_cost() -> Result<(), Box> { + let temp = tempfile::tempdir()?; + let tx_store = bitcoin_rs_storage::RedbStore::open(temp.path().join("txindex"))?; + let filter_store = bitcoin_rs_storage::RedbStore::open(temp.path().join("filters"))?; + let mut tx_index = bitcoin_rs_index::Indexer::new(Arc::new(tx_store)); + let filter_index = bitcoin_rs_filters::FilterIndex::new(filter_store); + let fixture = non_coinbase_spend_chain()?; + + let mut txids = 0_usize; + let mut funding = 0_usize; + let mut spending = 0_usize; + let mut filter_bytes_len = 0_usize; + let mut txindex_us = 0_u128; + let mut filterindex_us = 0_u128; + let mut prev_header = Hash256::default(); + let mut final_block_hash = None; + + for (height, block) in fixture.blocks.iter().enumerate() { + let height = u32::try_from(height)?; + let block_bytes = bitcoin::consensus::serialize(block); + + let started = Instant::now(); + let counts = tx_index.ingest_block(&block_bytes, height)?; + txindex_us = txindex_us.saturating_add(started.elapsed().as_micros()); + txids = txids.saturating_add(counts.txids); + funding = funding.saturating_add(counts.funding); + spending = spending.saturating_add(counts.spending); + + let filter_bytes = deterministic_filter_bytes(block); + filter_bytes_len = filter_bytes_len.saturating_add(filter_bytes.len()); + let block_hash = Hash256::from_le_bytes(block.block_hash().as_byte_array()); + let started = Instant::now(); + prev_header = filter_index.put_filter(block_hash, prev_header, &filter_bytes)?; + filterindex_us = filterindex_us.saturating_add(started.elapsed().as_micros()); + final_block_hash = Some(block_hash); + } + + if txids == 0 || funding == 0 || spending == 0 || filter_bytes_len == 0 { + return Err(std::io::Error::other(format!( + "optional index direct cost no-op: txids={txids} funding={funding} spending={spending} filter_bytes={filter_bytes_len}", + )) + .into()); + } + let final_block_hash = final_block_hash + .ok_or_else(|| std::io::Error::other("missing final block hash after direct indexing"))?; + let final_header = filter_index + .filter_header(final_block_hash)? + .ok_or_else(|| { + std::io::Error::other("missing final filter header after direct indexing") + })?; + assert_eq!(final_header, prev_header); + + let total_us = txindex_us.saturating_add(filterindex_us); + println!( + "optional_index_redb_direct_cost blocks={} txids={} funding={} spending={} filter_bytes={} txindex_us={} filterindex_us={} total_us={}", + fixture.blocks.len(), + txids, + funding, + spending, + filter_bytes_len, + txindex_us, + filterindex_us, + total_us, + ); + + Ok(()) +} + +struct ReplayOutcome { + elapsed: Duration, + applied_height: u32, + applied_hash: Hash256, + coin_stats: Arc, + listener_calls: ListenerCallCountSnapshot, + utxo: Arc, +} + +#[derive(Default)] +struct ListenerCallCounters { + insert_calls: AtomicU64, + remove_calls: AtomicU64, + remove_coin_calls: AtomicU64, +} + +impl ListenerCallCounters { + fn snapshot(&self) -> ListenerCallCountSnapshot { + ListenerCallCountSnapshot { + insert_calls: self.insert_calls.load(Ordering::Relaxed), + remove_calls: self.remove_calls.load(Ordering::Relaxed), + remove_coin_calls: self.remove_coin_calls.load(Ordering::Relaxed), + } + } +} + +#[derive(Clone, Copy, Default)] +struct ListenerCallCountSnapshot { + insert_calls: u64, + remove_calls: u64, + remove_coin_calls: u64, +} + +impl ListenerCallCountSnapshot { + const fn total_calls(self) -> u64 { + self.insert_calls + .saturating_add(self.remove_calls) + .saturating_add(self.remove_coin_calls) + } +} + +struct CountingCoinStatsListener { + inner: CoinStatsListener, + counters: Arc, +} + +impl UtxoChangeListener for CountingCoinStatsListener { + fn on_insert(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.counters.insert_calls.fetch_add(1, Ordering::Relaxed); + self.inner.on_insert(op, txout, height, coinbase); + } + + fn on_remove(&self, op: &OutPoint, txout: &TxOut, height: u32) { + self.counters.remove_calls.fetch_add(1, Ordering::Relaxed); + self.inner.on_remove(op, txout, height); + } + + fn on_remove_coin(&self, op: &OutPoint, txout: &TxOut, height: u32, coinbase: bool) { + self.counters + .remove_coin_calls + .fetch_add(1, Ordering::Relaxed); + self.inner.on_remove_coin(op, txout, height, coinbase); + } + + fn muhash3072(&self) -> Option<[u8; 384]> { + self.inner.muhash3072() + } +} + +#[derive(Clone, Copy)] +enum CoinStatsListenerMode { + Attached, + Detached, +} +impl CoinStatsListenerMode { + const fn label(self) -> &'static str { + match self { + Self::Attached => "attached", + Self::Detached => "detached", + } + } +} + +fn replay_non_coinbase_spend_chain( + fixture: &SpendChainFixture, + use_noop_index_hooks: bool, +) -> Result> { + replay_non_coinbase_spend_chain_with_coin_stats_listener( + fixture, + use_noop_index_hooks, + CoinStatsListenerMode::Attached, + ) +} + +fn replay_non_coinbase_spend_chain_with_coin_stats_listener( + fixture: &SpendChainFixture, + use_noop_index_hooks: bool, + coin_stats_listener: CoinStatsListenerMode, +) -> Result> { let block_tree = Arc::new(RwLock::new(BlockTree::new())); let chain_tip = block_tree.read().tip_handle(); let applied_tip: Arc> = Arc::new(ArcSwapOption::empty()); @@ -271,18 +560,24 @@ fn tick_applies_non_coinbase_spend_and_updates_utxo_and_coinstats() let inbound_headers_rx = Arc::new(Mutex::new(inbound_headers_rx_raw)); let (inbound_blocks_tx, inbound_blocks_rx_raw) = unbounded::(); let inbound_blocks_rx = Arc::new(Mutex::new(inbound_blocks_rx_raw)); - let (handles, coin_stats, utxo) = apply_handles_with_coin_stats_and_utxo( + let (mut handles, coin_stats, listener_calls, utxo) = apply_handles_with_coin_stats_and_utxo( Network::Regtest, Arc::clone(&chain_tip), Arc::clone(&applied_tip), Arc::clone(&block_tree), + coin_stats_listener, ); + if !use_noop_index_hooks { + handles.tx_index = None; + handles.filter_index = None; + } let sync = BlockSync::new( handles, Arc::clone(&peers), Arc::clone(&peer_outbound), inbound_headers_rx, inbound_blocks_rx, + false, ); inbound_headers_tx.send(fixture.blocks.iter().map(|block| block.header).collect())?; @@ -290,42 +585,56 @@ fn tick_applies_non_coinbase_spend_and_updates_utxo_and_coinstats() inbound_blocks_tx.send(block.clone())?; } + let started = Instant::now(); sync.tick(); + let elapsed = started.elapsed(); let applied = applied_tip .load_full() .ok_or_else(|| std::io::Error::other("missing applied tip"))?; - assert_eq!(applied.height, 102); - assert_eq!( - applied.hash, - bitcoin_rs_primitives::Hash256::from_le_bytes( - fixture - .blocks - .last() - .ok_or_else(|| std::io::Error::other("missing final block"))? - .block_hash() - .as_byte_array(), - ) - ); - assert!( - utxo.get(&primitive_outpoint(fixture.mature_coinbase_outpoint)) - .is_none(), - "mature coinbase prevout must be removed by the height-101 spend", - ); - assert!( - utxo.get(&primitive_outpoint(fixture.funding_outpoint)) - .is_none(), - "funding prevout must be removed by the height-102 spend", - ); - assert!( - utxo.get(&primitive_outpoint(fixture.spend_outpoint)) - .is_some(), - "height-102 spend output must remain live", + let expected_height = u32::try_from( + fixture + .blocks + .len() + .checked_sub(1) + .ok_or_else(|| std::io::Error::other("empty replay fixture"))?, + )?; + let expected_hash = bitcoin_rs_primitives::Hash256::from_le_bytes( + fixture + .blocks + .last() + .ok_or_else(|| std::io::Error::other("missing final block"))? + .block_hash() + .as_byte_array(), ); + if applied.height != expected_height || applied.hash != expected_hash { + return Err(std::io::Error::other(format!( + "replay stopped before final block: applied height/hash {}/{:?}, expected {}/{:?}", + applied.height, applied.hash, expected_height, expected_hash, + )) + .into()); + } + Ok(ReplayOutcome { + elapsed, + applied_height: applied.height, + applied_hash: applied.hash, + coin_stats, + listener_calls: listener_calls.snapshot(), + utxo, + }) +} - let block_refs: Vec<&bitcoin::Block> = fixture.blocks.iter().collect(); - assert_eq!(coin_stats.snapshot(), expected_coin_stats(&block_refs)?); - Ok(()) +fn elapsed_ms(duration: Duration) -> u128 { + duration.as_millis() +} + +#[cfg(feature = "redb")] +fn deterministic_filter_bytes(block: &bitcoin::Block) -> Vec { + let mut filter_bytes = Vec::with_capacity(block.txdata.len().saturating_mul(32)); + for tx in &block.txdata { + filter_bytes.extend_from_slice(tx.compute_txid().as_byte_array()); + } + filter_bytes } struct SpendChainFixture { @@ -446,8 +755,13 @@ fn apply_handles_with_coin_stats( applied_tip: Arc>, block_tree: Arc>, ) -> (ApplyHandles, Arc) { - let (handles, coin_stats, _utxo) = - apply_handles_with_coin_stats_and_utxo(network, chain_tip, applied_tip, block_tree); + let (handles, coin_stats, _listener_calls, _utxo) = apply_handles_with_coin_stats_and_utxo( + network, + chain_tip, + applied_tip, + block_tree, + CoinStatsListenerMode::Attached, + ); (handles, coin_stats) } @@ -457,10 +771,22 @@ fn apply_handles_with_coin_stats_and_utxo( chain_tip: Arc>, applied_tip: Arc>, block_tree: Arc>, -) -> (ApplyHandles, Arc, Arc) { + coin_stats_listener: CoinStatsListenerMode, +) -> ( + ApplyHandles, + Arc, + Arc, + Arc, +) { let coin_stats = Arc::new(CoinStatsListener::new(CoinStats::default())); + let listener_calls = Arc::new(ListenerCallCounters::default()); let mut utxo = UtxoSet::new(); - utxo.set_listener(Box::new((*coin_stats).clone())); + if matches!(coin_stats_listener, CoinStatsListenerMode::Attached) { + utxo.set_listener(Box::new(CountingCoinStatsListener { + inner: (*coin_stats).clone(), + counters: Arc::clone(&listener_calls), + })); + } let utxo = Arc::new(utxo); let handles = ApplyHandles::new( network, @@ -469,14 +795,14 @@ fn apply_handles_with_coin_stats_and_utxo( block_tree, Arc::clone(&utxo), Arc::clone(&coin_stats), - noop_tx_index(), - noop_filter_index(), + Some(noop_tx_index()), + Some(noop_filter_index()), Arc::new(RwLock::new(Mempool::new(MempoolLimits::default()))), Arc::new(RwLock::new(Vec::new())), Arc::new(RwLock::new(HashMap::::new())), Arc::new(bitcoin_rs_node::NoOpZmqPublisher), ); - (handles, coin_stats, utxo) + (handles, coin_stats, listener_calls, utxo) } struct NoopIndexer; diff --git a/crates/rpc/src/context.rs b/crates/rpc/src/context.rs index 714482e..884a63e 100644 --- a/crates/rpc/src/context.rs +++ b/crates/rpc/src/context.rs @@ -151,39 +151,6 @@ pub trait PruneService: Send + Sync { /// Reports whether pruning is enabled and the highest completed prune height. fn status(&self) -> PruneStatus; } -#[derive(Debug, Default)] -struct NoopFilterIndex; - -impl bitcoin_rs_filters::FilterIndexLike for NoopFilterIndex { - fn put_filter( - &self, - _block_hash: bitcoin_rs_primitives::Hash256, - _prev_header: bitcoin_rs_primitives::Hash256, - _filter_bytes: &[u8], - ) -> Result { - Ok(bitcoin_rs_primitives::Hash256::default()) - } - - fn filter_header( - &self, - _block_hash: bitcoin_rs_primitives::Hash256, - ) -> Result, bitcoin_rs_filters::FilterIndexError> { - Ok(None) - } - - fn filter( - &self, - _block_hash: bitcoin_rs_primitives::Hash256, - ) -> Result>, bitcoin_rs_filters::FilterIndexError> { - Ok(None) - } -} - -fn noop_filter_index() -> Arc> { - let filter_index: Box = Box::new(NoopFilterIndex); - Arc::new(filter_index) -} - /// Shared state consumed by JSON-RPC handlers. pub struct Context { /// Best-chain tip snapshot published by chain validation. @@ -200,8 +167,8 @@ pub struct Context { pub utxo: Arc, /// Incremental UTXO-set statistics. pub coin_stats: Arc, - /// BIP157/158 compact-filter index used by filter RPCs. - pub filter_index: Arc>, + /// BIP157/158 compact-filter index used by filter RPCs when enabled. + pub filter_index: Option>>, /// Optional storage pruning mutator. pub prune_service: Option>, /// Optional shared confirmed-block indexer used to resolve prevout values for fee statistics. @@ -278,7 +245,7 @@ impl Context { transactions: Arc::new(RwLock::new(HashMap::new())), utxo: Arc::new(utxo), coin_stats, - filter_index: noop_filter_index(), + filter_index: None, indexer: None, prune_service: None, network: Arc::new(RwLock::new(NetworkState::default())), @@ -311,7 +278,7 @@ impl Context { transactions: Arc>>, utxo: Arc, coin_stats: Arc, - filter_index: Arc>, + filter_index: Option>>, network: Arc>, mining_template_id: Arc>, peers: Arc>>, @@ -466,6 +433,12 @@ impl Context { /// Returns the block hash for `height` when known without blocking I/O. #[must_use] pub fn block_hash_at_height(&self, height: u32) -> Option { + let tree = self.block_tree.read(); + if tree.tip().is_some() { + return tree.active_node_at_height(height).map(|node| node.hash); + } + drop(tree); + self.blocks .read() .iter() @@ -585,6 +558,31 @@ fn target_to_f64(target: bitcoin::pow::Target) -> f64 { mod tests { use super::*; + struct NoopFilterIndex; + + impl bitcoin_rs_filters::FilterIndexLike for NoopFilterIndex { + fn put_filter( + &self, + _block_hash: bitcoin_rs_primitives::Hash256, + prev_header: bitcoin_rs_primitives::Hash256, + _filter_bytes: &[u8], + ) -> Result { + Ok(prev_header) + } + + fn filter_header( + &self, + _block_hash: bitcoin_rs_primitives::Hash256, + ) -> Result, bitcoin_rs_filters::FilterIndexError> + { + Ok(None) + } + } + + fn noop_filter_index() -> Arc> { + Arc::new(Box::new(NoopFilterIndex)) + } + #[test] #[allow(clippy::arc_with_non_send_sync)] fn from_handles_shares_tip_handles_with_caller() { @@ -608,7 +606,7 @@ mod tests { Arc::new(RwLock::new(HashMap::new())), Arc::clone(&utxo), Arc::clone(&coin_stats), - Arc::clone(&filter_index), + Some(Arc::clone(&filter_index)), Arc::new(RwLock::new(NetworkState::default())), Arc::new(ArcSwap::from_pointee(CompactString::new("0"))), Arc::new(RwLock::new(Vec::new())), @@ -637,7 +635,12 @@ mod tests { "coin_stats must be shared with caller" ); assert!( - Arc::ptr_eq(&ctx.filter_index, &filter_index), + Arc::ptr_eq( + ctx.filter_index + .as_ref() + .expect("filter_index must be wired"), + &filter_index + ), "filter_index must be shared with caller" ); assert!( diff --git a/crates/rpc/src/error.rs b/crates/rpc/src/error.rs index 2fa99ec..6072354 100644 --- a/crates/rpc/src/error.rs +++ b/crates/rpc/src/error.rs @@ -28,6 +28,9 @@ pub enum RpcError { /// A method is intentionally disabled by policy. #[error("{0}")] MethodDisabled(&'static str), + /// A requested optional index is disabled. + #[error("{0}")] + IndexDisabled(&'static str), /// Internal server failure. #[error("internal error: {0}")] Internal(String), @@ -67,6 +70,7 @@ impl RpcError { Self::InvalidParams(_) => Self::INVALID_PARAMS, Self::InvalidType(_) => Self::CORE_INVALID_TYPE, Self::NotFound(_) => Self::CORE_NOT_FOUND, + Self::IndexDisabled(_) => -1, Self::MethodDisabled(_) | Self::Internal(_) => Self::INTERNAL_ERROR, } } diff --git a/crates/rpc/src/handlers/chain.rs b/crates/rpc/src/handlers/chain.rs index a986b35..0a9f0a2 100644 --- a/crates/rpc/src/handlers/chain.rs +++ b/crates/rpc/src/handlers/chain.rs @@ -652,13 +652,14 @@ pub(crate) fn gettxoutsetinfo(ctx: &Arc, params: &Value) -> Result, params: &Value) -> Result { let hash = required_str(params, 0, "block hash is required")?; let hash = parse_hash(hash)?; - let filter_bytes = ctx - .filter_index + let filter_index = ctx.filter_index.as_ref().ok_or(RpcError::IndexDisabled( + "Index is not enabled for filtertype basic", + ))?; + let filter_bytes = filter_index .filter(hash) .map_err(|error| RpcError::Internal(error.to_string()))? .ok_or(RpcError::NotFound("block filter not found"))?; - let header = ctx - .filter_index + let header = filter_index .filter_header(hash) .map_err(|error| RpcError::Internal(error.to_string()))? .ok_or(RpcError::NotFound("block filter header not found"))?; @@ -691,13 +692,20 @@ pub(crate) fn getindexinfo(ctx: &Arc, params: &Value) -> Result Ok(json!({ - "txindex": entry(), - "basicblockfilterindex": entry(), - })), - Some("txindex") => Ok(json!({ "txindex": entry() })), - Some("basicblockfilterindex") => Ok(json!({ "basicblockfilterindex": entry() })), + None => Ok(Value::from(indexes)), + Some("txindex") if ctx.indexer.is_some() => Ok(json!({ "txindex": entry() })), + Some("basic block filter index") if ctx.filter_index.is_some() => { + Ok(json!({ "basic block filter index": entry() })) + } Some(_) => Ok(json!({})), } } diff --git a/crates/rpc/src/handlers/tx.rs b/crates/rpc/src/handlers/tx.rs index 0ec8504..0590180 100644 --- a/crates/rpc/src/handlers/tx.rs +++ b/crates/rpc/src/handlers/tx.rs @@ -42,7 +42,7 @@ pub(crate) fn getrawtransaction(ctx: &Arc, params: &Value) -> Result Result<(), Box Result<(), Box> { + let ctx = active_regtest_header_context()?; + let genesis_hash = bitcoin_rs_primitives::Network::Regtest.genesis_block_hash(); + assert!(ctx.blocks.read().is_empty()); + let handler = Handler::new(Arc::new(ctx)); + + let result = handler.dispatch("getblockhash", &json!([0]))?; + + assert_eq!(result.as_str(), Some(genesis_hash.to_string_be().as_str())); + Ok(()) +} + +#[test] +fn getblockhash_prefers_active_header_over_stale_block_record() +-> Result<(), Box> { + let ctx = active_regtest_header_context()?; + let genesis_hash = bitcoin_rs_primitives::Network::Regtest.genesis_block_hash(); + ctx.add_block(BlockRecord::synthetic( + 0, + Hash256::from_le_bytes(&[0x99; 32]), + )); + let handler = Handler::new(Arc::new(ctx)); + + let result = handler.dispatch("getblockhash", &json!([0]))?; + + assert_eq!(result.as_str(), Some(genesis_hash.to_string_be().as_str())); + Ok(()) +} + +#[test] +fn getblockhash_rejects_stale_block_record_above_active_tip() +-> Result<(), Box> { + let ctx = active_regtest_header_context()?; + ctx.add_block(BlockRecord::synthetic( + 2, + Hash256::from_le_bytes(&[0x77; 32]), + )); + let handler = Handler::new(Arc::new(ctx)); + + let error = handler + .dispatch("getblockhash", &json!([2])) + .expect_err("stale block record above active tip unexpectedly resolved"); + + assert_eq!(error.code(), RpcError::CORE_NOT_FOUND); + Ok(()) +} + #[test] fn getblockfilter_reads_filter_index() -> Result<(), Box> { let block_hash = Hash256::from_le_bytes(&[9_u8; 32]); @@ -260,7 +309,7 @@ fn getblockfilter_reads_filter_index() -> Result<(), Box> filter: vec![0xab, 0xcd], header, }); - ctx.filter_index = Arc::new(filter_index); + ctx.filter_index = Some(Arc::new(filter_index)); let handler = Handler::new(Arc::new(ctx)); let block_hash_hex = block_hash.to_string_be(); @@ -274,6 +323,23 @@ fn getblockfilter_reads_filter_index() -> Result<(), Box> Ok(()) } +#[test] +fn getblockfilter_reports_disabled_basic_filter_index() { + let handler = Handler::new(Arc::new(Context::new())); + let block_hash = Hash256::from_le_bytes(&[9_u8; 32]); + let block_hash_hex = block_hash.to_string_be(); + + let error = handler + .dispatch("getblockfilter", &json!([block_hash_hex.as_str()])) + .expect_err("disabled block filter index unexpectedly succeeded"); + + assert_eq!(error.code(), -1); + assert_eq!( + error.to_string(), + "Index is not enabled for filtertype basic" + ); +} + #[test] fn getblockfilter_returns_not_found_for_missing_filter_row() -> Result<(), Box> { @@ -285,7 +351,7 @@ fn getblockfilter_returns_not_found_for_missing_filter_row() filter: vec![0xab, 0xcd], header, }); - ctx.filter_index = Arc::new(filter_index); + ctx.filter_index = Some(Arc::new(filter_index)); let handler = Handler::new(Arc::new(ctx)); let missing_hash = Hash256::from_le_bytes(&[7_u8; 32]); let missing_hash_hex = missing_hash.to_string_be(); @@ -301,9 +367,27 @@ fn getblockfilter_returns_not_found_for_missing_filter_row() } #[test] -fn getindexinfo_returns_both_indexes() -> Result<(), Box> { - let ctx = Arc::new(Context::new()); - let handler = Handler::new(Arc::clone(&ctx)); +fn getindexinfo_omits_disabled_indexes() -> Result<(), Box> { + let handler = Handler::new(Arc::new(Context::new())); + + let result = handler.dispatch("getindexinfo", &json!([]))?; + + assert_eq!(result.as_object().map(sonic_rs::Object::len), Some(0)); + Ok(()) +} + +#[test] +fn getindexinfo_returns_enabled_core_index_names() -> Result<(), Box> { + let mut ctx = Context::new(); + ctx.indexer = Some(Arc::new(Mutex::new(Box::new(FakeIndexer { + values: HashMap::new(), + })))); + ctx.filter_index = Some(Arc::new(Box::new(StaticFilterIndex { + block_hash: Hash256::from_le_bytes(&[9_u8; 32]), + filter: vec![0xab, 0xcd], + header: Hash256::from_le_bytes(&[8_u8; 32]), + }))); + let handler = Handler::new(Arc::new(ctx)); let result = handler.dispatch("getindexinfo", &json!([]))?; @@ -312,17 +396,51 @@ fn getindexinfo_returns_both_indexes() -> Result<(), Box> assert_eq!(txindex.get("synced").as_bool(), Some(false)); assert_eq!(txindex.get("best_block_height").as_u64(), Some(0)); - let filter_index = result.get("basicblockfilterindex"); + let filter_index = result.get("basic block filter index"); assert!( filter_index.is_some(), - "basicblockfilterindex entry missing: {result:?}" + "basic block filter index entry missing: {result:?}" ); assert_eq!(filter_index.get("synced").as_bool(), Some(false)); assert_eq!(filter_index.get("best_block_height").as_u64(), Some(0)); + assert!(result.get("basicblockfilterindex").is_none()); Ok(()) } +#[test] +fn getrawtransaction_does_not_use_confirmed_map_when_txindex_disabled() +-> Result<(), Box> { + let ctx = Context::new(); + let tx = tx(42, ScriptBuf::from_bytes(vec![0x51])); + let txid = ctx.add_transaction(tx); + let handler = Handler::new(Arc::new(ctx)); + + let error = handler + .dispatch("getrawtransaction", &json!([txid.to_string()])) + .expect_err("confirmed map lookup unexpectedly succeeded with txindex disabled"); + + assert_eq!(error.code(), RpcError::CORE_NOT_FOUND); + Ok(()) +} + +#[test] +fn getrawtransaction_uses_confirmed_map_when_txindex_enabled() +-> Result<(), Box> { + let mut ctx = Context::new(); + ctx.indexer = Some(Arc::new(Mutex::new(Box::new(FakeIndexer { + values: HashMap::new(), + })))); + let tx = tx(43, ScriptBuf::from_bytes(vec![0x51])); + let txid = ctx.add_transaction(tx); + let handler = Handler::new(Arc::new(ctx)); + + let result = handler.dispatch("getrawtransaction", &json!([txid.to_string()]))?; + + assert!(result.as_str().is_some()); + Ok(()) +} + #[test] fn getblockstats_fee_fields_are_zero_without_indexer() -> Result<(), Box> { let (ctx, _low_tx, _high_tx) = fee_stats_context(None); @@ -662,11 +780,11 @@ impl Fixture { }; let block_hash_bytes = block.block_hash(); let block_hash = Hash256::from_le_bytes(block_hash_bytes.as_byte_array()); - ctx.filter_index = Arc::new(Box::new(StaticFilterIndex { + ctx.filter_index = Some(Arc::new(Box::new(StaticFilterIndex { block_hash, filter: vec![0x00], header: Hash256::from_le_bytes(&[0x08; 32]), - })); + }))); ctx.set_chain_tip(TipSnapshot { tip_id: NodeId::new(0), height: 7, @@ -699,6 +817,32 @@ fn context_with_peers(peers: Arc>>) -> Arc { Arc::new(ctx) } +fn active_regtest_header_context() -> Result> { + let ctx = Context::new(); + let genesis = bitcoin::blockdata::constants::genesis_block(bitcoin::Network::Regtest); + let child_header = bitcoin::block::Header { + version: bitcoin::block::Version::ONE, + prev_blockhash: genesis.block_hash(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: genesis.header.time.saturating_add(1), + bits: genesis.header.bits, + nonce: genesis.header.nonce.saturating_add(1), + }; + let child_hash = Hash256::from_le_bytes(child_header.block_hash().as_byte_array()); + let child_id = { + let mut tree = ctx.block_tree.write(); + tree.insert_header(genesis.header, NodeStatus::HeaderValid)?; + tree.insert_header(child_header, NodeStatus::HeaderValid)? + }; + ctx.set_chain_tip(TipSnapshot { + tip_id: child_id, + height: 1, + chainwork: ChainWork::ZERO, + hash: child_hash, + }); + Ok(ctx) +} + fn tx(label: u8, script_pubkey: ScriptBuf) -> Transaction { Transaction { version: bitcoin::transaction::Version::TWO, diff --git a/crates/script/Cargo.toml b/crates/script/Cargo.toml index eb5107d..74e7f3a 100644 --- a/crates/script/Cargo.toml +++ b/crates/script/Cargo.toml @@ -32,5 +32,11 @@ bytemuck.workspace = true serde.workspace = true [dev-dependencies] +criterion.workspace = true serde_json.workspace = true proptest.workspace = true + +[[bench]] +name = "interpreter_execute_profile" +harness = false +required-features = ["bitcoinconsensus"] diff --git a/crates/script/benches/interpreter_execute_profile.rs b/crates/script/benches/interpreter_execute_profile.rs new file mode 100644 index 0000000..f5356bf --- /dev/null +++ b/crates/script/benches/interpreter_execute_profile.rs @@ -0,0 +1,221 @@ +//! Diagnostic benchmark for `Interpreter::execute` boundary costs. +// PERF: Criterion emits public harness items whose docs are irrelevant to the benchmark report. +#![allow(missing_docs)] + +use std::{hint::black_box, time::Duration}; + +use bitcoin::consensus::encode; +use bitcoin::hashes::Hash as _; +use bitcoin::script::Builder; +use bitcoin::{ + Amount, OutPoint, Script, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness, + absolute, transaction, +}; +use bitcoin_rs_script::{Interpreter, VerifyFlags}; +use criterion::{ + BatchSize, BenchmarkGroup, Criterion, criterion_group, criterion_main, measurement::WallTime, +}; + +const INPUT_COUNT: usize = 400; +const SELECTED_INPUT: usize = INPUT_COUNT - 1; +const PREVOUT_VALUE: u64 = 50_000; +const OUTPUT_VALUE: u64 = 1_000; +const SAMPLE_SIZE: usize = 20; +const MEASUREMENT_SECONDS: u64 = 3; +const WITNESS_ITEM_COUNT: usize = 4; +const WITNESS_ITEM_LEN: usize = 72; + +struct SpendCase { + tx: Transaction, + prevout: TxOut, + script_pubkey: ScriptBuf, + script_sig: ScriptBuf, + witness_vec: Vec>, + flags: VerifyFlags, +} + +fn interpreter_execute_profile(c: &mut Criterion) { + let case = spend_case(); + let mut group = c.benchmark_group("interpreter_execute_profile"); + group.sample_size(SAMPLE_SIZE); + group.measurement_time(Duration::from_secs(MEASUREMENT_SECONDS)); + + bench_witness_to_vec(&mut group, &case); + bench_clone_mutate(&mut group, &case); + bench_serialize_mutated(&mut group, &case); + bench_bitcoinconsensus_verify_serialized(&mut group, &case); + bench_interpreter_execute(&mut group, &case); + + group.finish(); +} + +fn bench_witness_to_vec(group: &mut BenchmarkGroup<'_, WallTime>, case: &SpendCase) { + group.bench_function("witness_to_vec_400_input", |b| { + b.iter(|| black_box(selected_input(&case.tx).witness.to_vec())); + }); +} + +fn bench_clone_mutate(group: &mut BenchmarkGroup<'_, WallTime>, case: &SpendCase) { + group.bench_function("clone_mutate_400_input", |b| { + b.iter(|| black_box(cloned_spending(black_box(case)))); + }); +} + +fn bench_serialize_mutated(group: &mut BenchmarkGroup<'_, WallTime>, case: &SpendCase) { + group.bench_function("serialize_mutated_400_input", |b| { + b.iter_batched( + || cloned_spending(case), + |spending| black_box(encode::serialize(&spending)), + BatchSize::SmallInput, + ); + }); +} + +fn bench_bitcoinconsensus_verify_serialized( + group: &mut BenchmarkGroup<'_, WallTime>, + case: &SpendCase, +) { + let spending = cloned_spending(case); + let serialized = encode::serialize(&spending); + let script = Script::from_bytes(case.script_pubkey.as_bytes()); + + group.bench_function("bitcoinconsensus_verify_serialized_400_input", |b| { + b.iter(|| { + let result = script.verify_with_flags( + SELECTED_INPUT, + case.prevout.value, + black_box(serialized.as_slice()), + case.flags.consensus_bits(), + ); + match result { + Ok(()) => black_box(true), + Err(error) => panic!("bitcoinconsensus verification failed: {error}"), + } + }); + }); +} + +fn bench_interpreter_execute(group: &mut BenchmarkGroup<'_, WallTime>, case: &SpendCase) { + let interpreter = Interpreter; + group.bench_function("interpreter_execute_400_input", |b| { + b.iter(|| { + let result = interpreter.execute( + case.script_pubkey.as_bytes(), + case.script_sig.as_bytes(), + black_box(case.witness_vec.as_slice()), + case.flags, + &case.prevout, + &case.tx, + SELECTED_INPUT, + ); + match result { + Ok(value) => black_box(value), + Err(error) => panic!("interpreter execution failed: {error}"), + } + }); + }); +} + +fn spend_case() -> SpendCase { + let script_pubkey = op_true_script(); + let script_sig = ScriptBuf::new(); + let witness_vec = witness_vec(); + let mut tx = Transaction { + version: transaction::Version(1), + lock_time: absolute::LockTime::ZERO, + input: (0..INPUT_COUNT) + .map(|index| TxIn { + previous_output: OutPoint { + txid: txid(usize_to_u64(index)), + vout: 0, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }) + .collect(), + output: vec![TxOut { + value: Amount::from_sat(OUTPUT_VALUE), + script_pubkey: ScriptBuf::new(), + }], + }; + selected_input_mut(&mut tx).witness = Witness::from_slice(&witness_vec); + + SpendCase { + tx, + prevout: TxOut { + value: Amount::from_sat(PREVOUT_VALUE), + script_pubkey: script_pubkey.clone(), + }, + script_pubkey, + script_sig, + witness_vec, + flags: VerifyFlags::NONE, + } +} + +fn cloned_spending(case: &SpendCase) -> Transaction { + let mut spending = case.tx.clone(); + let inputs = spending.input.len(); + let input = match spending.input.get_mut(SELECTED_INPUT) { + Some(input) => input, + None => panic!("selected input {SELECTED_INPUT} out of range for {inputs} inputs"), + }; + input.script_sig = case.script_sig.clone(); + input.witness = Witness::from_slice(&case.witness_vec); + spending +} + +fn selected_input(tx: &Transaction) -> &TxIn { + match tx.input.get(SELECTED_INPUT) { + Some(input) => input, + None => panic!( + "selected input {SELECTED_INPUT} out of range for {} inputs", + tx.input.len() + ), + } +} + +fn selected_input_mut(tx: &mut Transaction) -> &mut TxIn { + let inputs = tx.input.len(); + match tx.input.get_mut(SELECTED_INPUT) { + Some(input) => input, + None => panic!("selected input {SELECTED_INPUT} out of range for {inputs} inputs"), + } +} + +fn op_true_script() -> ScriptBuf { + Builder::new().push_int(1).into_script() +} + +fn witness_vec() -> Vec> { + (0..WITNESS_ITEM_COUNT) + .map(|index| vec![usize_to_u8(index); WITNESS_ITEM_LEN]) + .collect() +} + +fn txid(seed: u64) -> Txid { + let mut bytes = [0_u8; 32]; + bytes[..8].copy_from_slice(&seed.to_le_bytes()); + bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); + bytes[16..24].copy_from_slice(&seed.wrapping_mul(0x9e37_79b9_7f4a_7c15).to_le_bytes()); + bytes[24..32].copy_from_slice(&seed.wrapping_add(0xd1b5_4a32_d192_ed03).to_le_bytes()); + Txid::from_byte_array(bytes) +} + +fn usize_to_u64(value: usize) -> u64 { + match u64::try_from(value) { + Ok(value) => value, + Err(error) => panic!("usize to u64 conversion failed: {error}"), + } +} + +fn usize_to_u8(value: usize) -> u8 { + match u8::try_from(value) { + Ok(value) => value, + Err(error) => panic!("usize to u8 conversion failed: {error}"), + } +} + +criterion_group!(benches, interpreter_execute_profile); +criterion_main!(benches); diff --git a/crates/utxo/benches/utxo_commit.rs b/crates/utxo/benches/utxo_commit.rs index 4dccf5f..c3ab56d 100644 --- a/crates/utxo/benches/utxo_commit.rs +++ b/crates/utxo/benches/utxo_commit.rs @@ -19,7 +19,15 @@ const fn next_u64(state: &mut u64) -> u64 { *state } +fn mix_synthetic_seed(seed: u64) -> u64 { + let mut value = seed.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} + fn txid(seed: u64) -> Hash256 { + let seed = mix_synthetic_seed(seed); let mut bytes = [0_u8; 32]; bytes[..8].copy_from_slice(&seed.to_le_bytes()); bytes[8..16].copy_from_slice(&seed.rotate_left(11).to_le_bytes()); @@ -79,6 +87,10 @@ const fn percentile(samples: &[Duration], numerator: usize, denominator: usize) fn print_synthetic_summary() { let mut samples = Vec::with_capacity(9); let (_, _, distribution) = synthetic_case(0x5555_aaaa_ffff_0000); + assert!( + distribution.iter().all(|entries| *entries > 0), + "synthetic txids must exercise every UTXO shard" + ); for seed in 0_u64..9 { let (set, changes, _) = synthetic_case(seed + 1); let start = Instant::now(); diff --git a/crates/utxo/src/set.rs b/crates/utxo/src/set.rs index beabb98..f96e0f3 100644 --- a/crates/utxo/src/set.rs +++ b/crates/utxo/src/set.rs @@ -427,15 +427,21 @@ impl UtxoSet { ) -> Result<(), UtxoError> { let mut adds_by_shard = empty_add_buckets(); let mut removes_by_shard = empty_remove_buckets(); + let mut dirty = [false; UtxoKey::SHARD_COUNT]; + let mut dirty_shards = Vec::new(); for add in adds { validate_add(add)?; let key = UtxoKey::from_txid(&add.outpoint.txid); - adds_by_shard[usize::from(key.shard())].push((key, add.outpoint.txid, add.payload())); + let shard_idx = usize::from(key.shard()); + mark_dirty(&mut dirty, &mut dirty_shards, shard_idx); + adds_by_shard[shard_idx].push((key, add.outpoint.txid, add.payload())); } for remove in removes { let key = UtxoKey::from_txid(&remove.txid); - removes_by_shard[usize::from(key.shard())].push(SpendPayload { + let shard_idx = usize::from(key.shard()); + mark_dirty(&mut dirty, &mut dirty_shards, shard_idx); + removes_by_shard[shard_idx].push(SpendPayload { op: remove, key, vout: remove.vout, @@ -444,16 +450,21 @@ impl UtxoSet { } let _stable_commit = self.stable_view_lock.write(); + let listener = self.listener.as_deref(); + + if let [shard_idx] = dirty_shards.as_slice() { + return self.shards[*shard_idx].commit_batch( + &adds_by_shard[*shard_idx], + &removes_by_shard[*shard_idx], + listener, + ); + } let errors = Mutex::new(Vec::new()); - let listener = self.listener.as_deref(); rayon::scope(|scope| { - for shard_idx in 0..UtxoKey::SHARD_COUNT { + for shard_idx in dirty_shards.iter().copied() { let shard_adds = &adds_by_shard[shard_idx]; let shard_removes = &removes_by_shard[shard_idx]; - if shard_adds.is_empty() && shard_removes.is_empty() { - continue; - } let shard = &self.shards[shard_idx]; let errors = &errors; scope.spawn(move |_| { @@ -486,12 +497,23 @@ fn validate_add(add: &UtxoAdd) -> Result<(), UtxoError> { Ok(()) } -fn empty_add_buckets<'a>() -> Vec)>> { - (0..UtxoKey::SHARD_COUNT).map(|_| Vec::new()).collect() +fn empty_add_buckets<'a>() -> [Vec<(UtxoKey, Hash256, BuildPayload<'a>)>; UtxoKey::SHARD_COUNT] { + core::array::from_fn(|_| Vec::new()) } -fn empty_remove_buckets<'a>() -> Vec>> { - (0..UtxoKey::SHARD_COUNT).map(|_| Vec::new()).collect() +fn empty_remove_buckets<'a>() -> [Vec>; UtxoKey::SHARD_COUNT] { + core::array::from_fn(|_| Vec::new()) +} + +fn mark_dirty( + dirty: &mut [bool; UtxoKey::SHARD_COUNT], + dirty_shards: &mut Vec, + shard_idx: usize, +) { + if !dirty[shard_idx] { + dirty[shard_idx] = true; + dirty_shards.push(shard_idx); + } } fn stable_view_len(view: &UtxoSetView<'_>) -> usize { view.len()