diff --git a/crates/iota-analytics-indexer/src/handlers/transaction_handler.rs b/crates/iota-analytics-indexer/src/handlers/transaction_handler.rs index 13c598df9054..10ddabea8870 100644 --- a/crates/iota-analytics-indexer/src/handlers/transaction_handler.rs +++ b/crates/iota-analytics-indexer/src/handlers/transaction_handler.rs @@ -223,11 +223,13 @@ mod tests { // Create a checkpoint which should include the transaction we executed. let checkpoint = sim.create_checkpoint(); - let checkpoint_data = sim.get_checkpoint_data( - checkpoint.clone(), - sim.get_checkpoint_contents_by_digest(&checkpoint.contents_digest) - .unwrap(), - ); + let checkpoint_data: iota_types::full_checkpoint_content::CheckpointData = sim + .get_checkpoint_data( + checkpoint.clone(), + sim.get_checkpoint_contents_by_digest(&checkpoint.contents_digest) + .unwrap(), + ) + .into(); let shared_checkpoint_data = Arc::new(checkpoint_data); let txn_handler = TransactionHandler::new(); txn_handler diff --git a/crates/iota-core/src/authority.rs b/crates/iota-core/src/authority.rs index c17dcf16e8ba..285d8ec8a642 100644 --- a/crates/iota-core/src/authority.rs +++ b/crates/iota-core/src/authority.rs @@ -106,7 +106,8 @@ use iota_types::{ move_authenticator::MoveAuthenticatorExt, object::{Object, ObjectRead, PastObjectRead, bounded_visitor::BoundedVisitor}, storage::{ - BackingPackageStore, BackingStore, ObjectKey, ObjectOrTombstone, ObjectStore, WriteKind, + BackingPackageStore, BackingStore, ObjectKey, ObjectOrTombstone, ObjectStore, + TrackingBackingStore, WriteKind, }, supported_protocol_versions::{ ProtocolConfig, SupportedProtocolVersions, SupportedProtocolVersionsWithHashes, @@ -1649,20 +1650,21 @@ impl AuthorityState { // errors). However, all errors from this function occur before we have // written anything to the db, so we commit the tx guard and rely on the // client to retry the tx (if it was transient). - let (inner_temporary_store, effects, execution_error_opt) = match self.execute_transaction( - &execution_guard, - transaction, - tx_input_objects, - per_authenticator_inputs, - epoch_store, - ) { - Err(e) => { - info!(name = ?self.name, ?digest, "Error preparing transaction: {e}"); - tx_guard.release(); - return Err(e); - } - Ok(res) => res, - }; + let (inner_temporary_store, effects, execution_error_opt, unchanged_loaded_runtime_objects) = + match self.execute_transaction( + &execution_guard, + transaction, + tx_input_objects, + per_authenticator_inputs, + epoch_store, + ) { + Err(e) => { + info!(name = ?self.name, ?digest, "Error preparing transaction: {e}"); + tx_guard.release(); + return Err(e); + } + Ok(res) => res, + }; if let Some(expected_effects_digest) = expected_effects_digest { if effects.digest() != expected_effects_digest { @@ -1707,6 +1709,7 @@ impl AuthorityState { transaction, inner_temporary_store, &effects, + unchanged_loaded_runtime_objects, tx_guard, execution_guard, epoch_store, @@ -1740,6 +1743,7 @@ impl AuthorityState { transaction: &VerifiedExecutableTransaction, inner_temporary_store: InnerTemporaryStore, effects: &TransactionEffects, + unchanged_loaded_runtime_objects: Vec, tx_guard: TxGuard, _execution_guard: ExecutionLockReadGuard<'_>, epoch_store: &Arc, @@ -1775,6 +1779,7 @@ impl AuthorityState { transaction.clone().into_unsigned(), effects.clone(), inner_temporary_store, + unchanged_loaded_runtime_objects, ); self.get_cache_writer() .try_write_transaction_outputs(epoch_store.epoch(), transaction_outputs.into())?; @@ -1860,6 +1865,7 @@ impl AuthorityState { InnerTemporaryStore, TransactionEffects, Option, + Vec, )> { let _scope = monitored_scope("Execution::execute_certificate"); let _metrics_guard = self.metrics.prepare_certificate_latency.start_timer(); @@ -1875,7 +1881,8 @@ impl AuthorityState { .epoch_data() .epoch_start_timestamp(); - let backing_store = self.get_backing_store().as_ref(); + let tracking_store = TrackingBackingStore::new(self.get_backing_store().as_ref()); + let backing_store = &tracking_store; let tx_digest = *transaction.digest(); @@ -2082,7 +2089,19 @@ impl AuthorityState { .observe(effects.gas_cost_summary().computation_cost as f64 / elapsed); } - Ok((inner_temp_store, effects, execution_error_opt.err())) + let unchanged_loaded_runtime_objects = + iota_types::storage::unchanged_loaded_runtime_objects( + tx, + &effects, + &tracking_store.into_read_objects(), + ); + + Ok(( + inner_temp_store, + effects, + execution_error_opt.err(), + unchanged_loaded_runtime_objects, + )) } pub fn prepare_transaction_for_benchmark( @@ -2105,6 +2124,9 @@ impl AuthorityState { vec![], epoch_store, ) + .map(|(inner_temp_store, effects, execution_error, _)| { + (inner_temp_store, effects, execution_error) + }) } /// Simulate a transaction without committing it. @@ -2305,6 +2327,7 @@ impl AuthorityState { // Execute the simulation let (kind, signer, gas_data) = transaction.execution_parts(); + let (inner_temp_store, _, effects, execution_result) = executor.dev_inspect_transaction( self.get_backing_store().as_ref(), protocol_config, @@ -2325,8 +2348,16 @@ impl AuthorityState { checks.disabled(), ); + let mut input_objects = inner_temp_store.input_objects; + iota_types::storage::extend_input_objects_with_loaded_runtime_objects( + &mut input_objects, + &effects, + &inner_temp_store.loaded_runtime_objects, + self.get_backing_store().as_object_store(), + ); + Ok(SimulateTransactionResult { - input_objects: inner_temp_store.input_objects, + input_objects, output_objects: inner_temp_store.written, events: effects.events_digest().map(|_| inner_temp_store.events), effects, @@ -5265,7 +5296,7 @@ impl AuthorityState { let (input_objects, _) = self.read_objects_for_execution(&tx_lock, &executable_tx, epoch_store)?; - let (temporary_store, effects, _execution_error_opt) = self.execute_transaction( + let (temporary_store, effects, _execution_error_opt, _) = self.execute_transaction( &execution_guard, &executable_tx, input_objects, diff --git a/crates/iota-core/src/authority/authority_store.rs b/crates/iota-core/src/authority/authority_store.rs index 94597a6aeeb3..bb649444e121 100644 --- a/crates/iota-core/src/authority/authority_store.rs +++ b/crates/iota-core/src/authority/authority_store.rs @@ -406,6 +406,15 @@ impl AuthorityStore { .collect::, _>>()?) } + pub fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Result>, TypedStoreError> { + self.perpetual_tables + .unchanged_loaded_runtime_objects + .get(digest) + } + pub fn multi_get_effects<'a>( &self, effects_digests: impl Iterator, @@ -835,6 +844,7 @@ impl AuthorityStore { deleted, written, events, + unchanged_loaded_runtime_objects, live_object_markers_to_delete, new_live_object_markers_to_init, .. @@ -885,6 +895,14 @@ impl AuthorityStore { )?; } + // Write unchanged_loaded_runtime_objects + if !unchanged_loaded_runtime_objects.is_empty() { + write_batch.insert_batch( + &self.perpetual_tables.unchanged_loaded_runtime_objects, + [(transaction_digest, unchanged_loaded_runtime_objects)], + )?; + } + self.initialize_live_object_markers_impl(write_batch, new_live_object_markers_to_init)?; // Note: deletes live object markers for received objects as well (but not for diff --git a/crates/iota-core/src/authority/authority_store_pruner.rs b/crates/iota-core/src/authority/authority_store_pruner.rs index a244442e650c..7289dbd00937 100644 --- a/crates/iota-core/src/authority/authority_store_pruner.rs +++ b/crates/iota-core/src/authority/authority_store_pruner.rs @@ -347,7 +347,7 @@ impl AuthorityStorePruner { perpetual_batch.delete_batch(&perpetual_db.executed_effects, transactions.iter())?; perpetual_batch.delete_batch( &perpetual_db.executed_transactions_to_checkpoint, - transactions, + transactions.iter(), )?; let mut effect_digests = vec![]; @@ -361,6 +361,10 @@ impl AuthorityStorePruner { .delete_batch(&perpetual_db.events_2, [effects.transaction_digest()])?; } } + perpetual_batch.delete_batch( + &perpetual_db.unchanged_loaded_runtime_objects, + transactions.iter(), + )?; perpetual_batch.delete_batch(&perpetual_db.effects, effect_digests)?; let mut checkpoints_batch = checkpoint_db.tables.certified_checkpoints.batch(); diff --git a/crates/iota-core/src/authority/authority_store_tables.rs b/crates/iota-core/src/authority/authority_store_tables.rs index d9fe86a78350..cb6e9803456a 100644 --- a/crates/iota-core/src/authority/authority_store_tables.rs +++ b/crates/iota-core/src/authority/authority_store_tables.rs @@ -109,6 +109,9 @@ pub struct AuthorityPerpetualTables { // Events keyed by the digest of the transaction that produced them. pub(crate) events_2: DBMap, + // Loaded (and unchanged) runtime object references. + pub(crate) unchanged_loaded_runtime_objects: DBMap>, + /// Epoch and checkpoint of transactions finalized by checkpoint /// executor. Currently, mainly used to implement JSON RPC `ReadApi`. /// Note, there is a table with the same name in diff --git a/crates/iota-core/src/checkpoints/checkpoint_executor/data_ingestion_handler.rs b/crates/iota-core/src/checkpoints/checkpoint_executor/data_ingestion_handler.rs index d6cb2fa49cd0..a4db439f40cb 100644 --- a/crates/iota-core/src/checkpoints/checkpoint_executor/data_ingestion_handler.rs +++ b/crates/iota-core/src/checkpoints/checkpoint_executor/data_ingestion_handler.rs @@ -2,13 +2,17 @@ // Modifications Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{collections::HashMap, path::Path}; +use std::{ + collections::{BTreeSet, HashMap}, + path::Path, +}; use iota_storage::blob::{Blob, BlobEncoding}; use iota_types::{ effects::TransactionEffectsAPI, error::{IotaError, IotaResult}, - full_checkpoint_content::{CheckpointData, CheckpointTransaction}, + full_checkpoint_content::{Checkpoint, CheckpointData, ExecutedTransaction}, + object::ObjectSet, storage::ObjectStore, }; @@ -17,19 +21,19 @@ use crate::{ execution_cache::TransactionCacheRead, }; -pub(crate) fn load_checkpoint_data( +pub(crate) fn load_checkpoint( checkpoint_exec_data: &CheckpointExecutionData, checkpoint_tx_data: &CheckpointTransactionData, object_store: &dyn ObjectStore, transaction_cache_reader: &dyn TransactionCacheRead, -) -> IotaResult { +) -> IotaResult { let event_tx_digests = checkpoint_tx_data .effects .iter() .flat_map(|fx| fx.events_digest().map(|_| fx.transaction_digest()).copied()) .collect::>(); - let events = transaction_cache_reader + let mut events = transaction_cache_reader .try_multi_get_events(&event_tx_digests)? .into_iter() .zip(event_tx_digests) @@ -40,7 +44,7 @@ pub(crate) fn load_checkpoint_data( }) .collect::>>()?; - let mut full_transactions = Vec::with_capacity(checkpoint_tx_data.transactions.len()); + let mut transactions = Vec::with_capacity(checkpoint_tx_data.transactions.len()); for (tx, fx) in checkpoint_tx_data .transactions .iter() @@ -48,31 +52,58 @@ pub(crate) fn load_checkpoint_data( { let events = fx.events_digest().map(|_event_digest| { events - .get(fx.transaction_digest()) - .cloned() + .remove(fx.transaction_digest()) .expect("event was already checked to be present") }); - let input_objects = iota_types::storage::get_transaction_input_objects(object_store, fx) - .map_err(|e| IotaError::Unknown(e.to_string()))?; - let output_objects = iota_types::storage::get_transaction_output_objects(object_store, fx) - .map_err(|e| IotaError::Unknown(e.to_string()))?; - - let full_transaction = CheckpointTransaction { - transaction: (*tx).clone().into_unsigned().into(), + let transaction = ExecutedTransaction { + transaction: tx.data().transaction().clone(), + signatures: tx.data().signatures().to_vec(), effects: fx.clone(), events, - input_objects, - output_objects, + unchanged_loaded_runtime_objects: transaction_cache_reader + .get_unchanged_loaded_runtime_objects(tx.digest()) + // We don't write empty sets to the DB to save space, so if this load went + // through the writeback cache to the DB itself it wouldn't find an entry. + .unwrap_or_default(), }; - full_transactions.push(full_transaction); + transactions.push(transaction); } - let checkpoint_data = CheckpointData { - checkpoint_summary: checkpoint_exec_data.checkpoint.clone().into(), - checkpoint_contents: checkpoint_exec_data.checkpoint_contents.clone(), - transactions: full_transactions, + + let object_set = { + let refs = transactions + .iter() + .flat_map(|tx| { + iota_types::storage::get_transaction_object_set( + &tx.transaction, + &tx.effects, + &tx.unchanged_loaded_runtime_objects, + ) + }) + .collect::>() + .into_iter() + .collect::>(); + + let objects = object_store.multi_get_objects_by_key(&refs); + + let mut object_set = ObjectSet::default(); + for (idx, object) in objects.into_iter().enumerate() { + object_set.insert(object.ok_or_else(|| { + iota_types::storage::error::Error::custom(format!( + "unable to load object {:?}", + refs[idx] + )) + })?); + } + object_set + }; + let checkpoint = Checkpoint { + summary: checkpoint_exec_data.checkpoint.clone().into(), + contents: checkpoint_exec_data.checkpoint_contents.clone(), + transactions, + object_set, }; - Ok(checkpoint_data) + Ok(checkpoint) } pub(crate) fn store_checkpoint_locally( diff --git a/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs b/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs index b7350662fde3..45f99280d631 100644 --- a/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs +++ b/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs @@ -35,7 +35,7 @@ use iota_types::{ base_types::ExecutionData, effects::TransactionEffectsAPI, executable_transaction::VerifiedExecutableTransaction, - full_checkpoint_content::CheckpointData, + full_checkpoint_content::{Checkpoint, CheckpointData}, global_state_hash::GlobalStateHash, messages_checkpoint::{ CheckpointContentsExt, CheckpointSequenceNumber, CheckpointSummaryExt, @@ -68,7 +68,7 @@ pub(crate) mod utils; #[cfg(test)] pub(crate) mod tests; -use data_ingestion_handler::{load_checkpoint_data, store_checkpoint_locally}; +use data_ingestion_handler::{load_checkpoint, store_checkpoint_locally}; use metrics::CheckpointExecutorMetrics; use utils::*; @@ -96,7 +96,7 @@ pub(crate) struct CheckpointTransactionData { pub(crate) struct CheckpointExecutionState { pub data: CheckpointExecutionData, state_hash: Option, - full_data: Option, + full_data: Option, } impl CheckpointExecutionState { @@ -660,7 +660,7 @@ impl CheckpointExecutor { &self, ckpt_data: &CheckpointExecutionData, tx_data: &CheckpointTransactionData, - ) -> Option { + ) -> Option { let is_checkpoint_data_enabled = self.checkpoint_data_enabled(); // Boundaries always need full `CheckpointData` to persist `epoch_info`, // even when no other consumer is configured. @@ -669,13 +669,15 @@ impl CheckpointExecutor { return None; } - let checkpoint_data = load_checkpoint_data( + let checkpoint = load_checkpoint( ckpt_data, tx_data, self.state.get_object_store(), &*self.transaction_cache_reader, ) .expect("failed to load checkpoint data"); + // The consumers below still take `CheckpointData`. + let checkpoint_data: CheckpointData = (&checkpoint).into(); // Persist the boundary's `epoch_info` row eagerly. Two properties make // that safe. Boundaries run in epoch order: the boundary waits for every @@ -711,7 +713,7 @@ impl CheckpointExecutor { .expect("failed to store checkpoint locally"); } - Some(checkpoint_data) + Some(checkpoint) } // Load all required transaction and effects data for the checkpoint. @@ -1007,23 +1009,24 @@ impl CheckpointExecutor { fn broadcast_checkpoint( &self, checkpoint_exec_data: &CheckpointExecutionData, - checkpoint_data: Option<&CheckpointData>, + checkpoint: Option<&Checkpoint>, ) { if let Some(data_sender) = &self.data_sender { - let checkpoint_data = if let Some(data) = checkpoint_data { - data.clone() + let checkpoint_data: CheckpointData = if let Some(checkpoint) = checkpoint { + checkpoint.into() } else { // Reconstruct checkpoint data if needed (rare case: data_sender configured but // checkpoint_data_enabled is false) let (_, tx_data) = self.load_checkpoint_transactions(checkpoint_exec_data.checkpoint.clone()); - load_checkpoint_data( + load_checkpoint( checkpoint_exec_data, &tx_data, self.state.get_object_store(), self.transaction_cache_reader.as_ref(), ) .expect("Failed to load full CheckpointData") + .into() }; data_sender(&checkpoint_data); } @@ -1037,10 +1040,10 @@ impl CheckpointExecutor { /// If configured, commit the pending index updates for the provided /// checkpoint #[instrument(level = "info", skip_all)] - fn commit_index_updates(&self, checkpoint: CheckpointData) { + fn commit_index_updates(&self, checkpoint: Checkpoint) { if let Some(grpc_indexes_store) = &self.state.grpc_indexes_store { grpc_indexes_store - .commit_update_for_checkpoint(checkpoint.checkpoint_summary.sequence_number) + .commit_update_for_checkpoint(checkpoint.summary.sequence_number) .expect("failed to update gRPC indexes"); } } diff --git a/crates/iota-core/src/checkpoints/mod.rs b/crates/iota-core/src/checkpoints/mod.rs index cc00f8147859..dab16b97995d 100644 --- a/crates/iota-core/src/checkpoints/mod.rs +++ b/crates/iota-core/src/checkpoints/mod.rs @@ -3664,6 +3664,13 @@ mod tests { } impl TransactionCacheRead for HashMap { + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + unimplemented!() + } + fn try_notify_read_executed_effects( &self, _: &str, diff --git a/crates/iota-core/src/execution_cache.rs b/crates/iota-core/src/execution_cache.rs index f6e6f529df91..9a5bb2e88cc5 100644 --- a/crates/iota-core/src/execution_cache.rs +++ b/crates/iota-core/src/execution_cache.rs @@ -934,6 +934,11 @@ pub trait TransactionCacheRead: Send + Sync { self.try_get_events(digest).expect("storage access failed") } + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option>; + fn try_notify_read_executed_effects_digests<'a>( &'a self, task_name: &'static str, diff --git a/crates/iota-core/src/execution_cache/unit_tests/writeback_cache_tests.rs b/crates/iota-core/src/execution_cache/unit_tests/writeback_cache_tests.rs index abab27156fdf..40f4c36a6e42 100644 --- a/crates/iota-core/src/execution_cache/unit_tests/writeback_cache_tests.rs +++ b/crates/iota-core/src/execution_cache/unit_tests/writeback_cache_tests.rs @@ -175,6 +175,7 @@ impl Scenario { transaction: Arc::new(tx), effects, events, + unchanged_loaded_runtime_objects: Default::default(), markers: Default::default(), wrapped: Default::default(), deleted: Default::default(), diff --git a/crates/iota-core/src/execution_cache/writeback_cache.rs b/crates/iota-core/src/execution_cache/writeback_cache.rs index d16dac763a25..dfbd5ee44b4a 100644 --- a/crates/iota-core/src/execution_cache/writeback_cache.rs +++ b/crates/iota-core/src/execution_cache/writeback_cache.rs @@ -234,6 +234,8 @@ struct UncommittedData { transaction_events: DashMap, + unchanged_loaded_runtime_objects: DashMap>, + executed_effects_digests: DashMap, // Transaction outputs that have not yet been written to the DB. Items are removed from this @@ -253,6 +255,7 @@ impl UncommittedData { executed_effects_digests: DashMap::with_shard_amount(2048), pending_transaction_writes: DashMap::with_shard_amount(2048), transaction_events: DashMap::with_shard_amount(2048), + unchanged_loaded_runtime_objects: DashMap::with_shard_amount(2048), total_transaction_inserts: AtomicU64::new(0), total_transaction_commits: AtomicU64::new(0), } @@ -265,6 +268,7 @@ impl UncommittedData { self.executed_effects_digests.clear(); self.pending_transaction_writes.clear(); self.transaction_events.clear(); + self.unchanged_loaded_runtime_objects.clear(); self.total_transaction_inserts .store(0, std::sync::atomic::Ordering::Relaxed); self.total_transaction_commits @@ -280,6 +284,7 @@ impl UncommittedData { && self.transaction_effects.is_empty() && self.executed_effects_digests.is_empty() && self.transaction_events.is_empty() + && self.unchanged_loaded_runtime_objects.is_empty() && self .total_transaction_inserts .load(std::sync::atomic::Ordering::Relaxed) @@ -881,6 +886,7 @@ impl WritebackCache { deleted, wrapped, events, + unchanged_loaded_runtime_objects, .. } = &*tx_outputs; @@ -944,6 +950,12 @@ impl WritebackCache { .transaction_events .insert(tx_digest, events.clone()); + self.metrics + .record_cache_write("unchanged_loaded_runtime_objects"); + self.dirty + .unchanged_loaded_runtime_objects + .insert(tx_digest, unchanged_loaded_runtime_objects.clone()); + self.metrics.record_cache_write("executed_effects_digests"); self.dirty .executed_effects_digests @@ -1146,6 +1158,11 @@ impl WritebackCache { .remove(&tx_digest) .expect("events must exist"); + self.dirty + .unchanged_loaded_runtime_objects + .remove(&tx_digest) + .expect("unchanged_loaded_runtime_objects must exist"); + self.dirty .executed_effects_digests .remove(&tx_digest) @@ -2162,6 +2179,21 @@ impl TransactionCacheRead for WritebackCache { }, ) } + + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option> { + self.dirty + .unchanged_loaded_runtime_objects + .get(digest) + .map(|b| b.clone()) + .or_else(|| { + self.store + .get_unchanged_loaded_runtime_objects(digest) + .expect("db error") + }) + } } impl ExecutionCacheWrite for WritebackCache { diff --git a/crates/iota-core/src/storage.rs b/crates/iota-core/src/storage.rs index f8937fadc50a..28d940abd8ea 100644 --- a/crates/iota-core/src/storage.rs +++ b/crates/iota-core/src/storage.rs @@ -220,6 +220,15 @@ impl ReadStore for RocksDbStore { .map_err(StorageError::custom) } + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option> { + self.cache_traits + .transaction_cache_reader + .get_unchanged_loaded_runtime_objects(digest) + } + fn try_get_latest_checkpoint(&self) -> iota_types::storage::error::Result { self.checkpoint_store .get_highest_executed_checkpoint() @@ -478,6 +487,13 @@ impl ReadStore for GrpcReadStore { ) -> iota_types::storage::error::Result> { self.rocks.try_get_full_checkpoint_contents(digest) } + + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option> { + self.rocks.get_unchanged_loaded_runtime_objects(digest) + } } impl GrpcStateReader for GrpcReadStore { diff --git a/crates/iota-core/src/transaction_outputs.rs b/crates/iota-core/src/transaction_outputs.rs index 9f5576aacd1f..88a833fa86e3 100644 --- a/crates/iota-core/src/transaction_outputs.rs +++ b/crates/iota-core/src/transaction_outputs.rs @@ -20,6 +20,7 @@ pub struct TransactionOutputs { pub transaction: Arc, pub effects: TransactionEffects, pub events: TransactionEvents, + pub unchanged_loaded_runtime_objects: Vec, pub markers: Vec<(ObjectKey, MarkerValue)>, pub wrapped: Vec, @@ -36,6 +37,7 @@ impl TransactionOutputs { transaction: VerifiedTransaction, effects: TransactionEffects, inner_temporary_store: InnerTemporaryStore, + unchanged_loaded_runtime_objects: Vec, ) -> TransactionOutputs { let InnerTemporaryStore { input_objects, @@ -130,6 +132,7 @@ impl TransactionOutputs { transaction: Arc::new(transaction), effects, events, + unchanged_loaded_runtime_objects, markers, wrapped, deleted, diff --git a/crates/iota-core/src/unit_tests/authority_tests.rs b/crates/iota-core/src/unit_tests/authority_tests.rs index 16e673d17fd3..11e5fbad77a0 100644 --- a/crates/iota-core/src/unit_tests/authority_tests.rs +++ b/crates/iota-core/src/unit_tests/authority_tests.rs @@ -1201,6 +1201,37 @@ async fn test_dry_run_dev_inspect_dynamic_field_too_new() { .unwrap(); assert_eq!(result.effects.deleted().len(), 0); assert_eq!(execution_error_source(&result), Some("VMError with status ABORTED with sub status 1 at location Module ModuleId { address: 0000000000000000000000000000000000000000000000000000000000000002, name: Identifier(\"dynamic_field\") } at code offset 0 in function definition 13".to_string())); + + // dry run against the current parent: the field object is loaded at + // runtime and removed, so it must appear in the returned input objects at + // its pre-state version even though it is not a declared input + let field = effects.created()[0].0; + let pt = ProgrammableTransaction { + inputs: vec![CallArg::ImmutableOrOwned(new_parent.object_ref())], + commands: vec![Command::new_move_call( + object_basics.object_id, + Identifier::from_static("object_basics"), + Identifier::from_static("remove_field"), + vec![], + vec![Argument::Input(0)], + )], + }; + let tx = Transaction::new_programmable( + sender, + vec![gas_object_ref], + pt, + rgp * TEST_ONLY_GAS_UNIT_FOR_OBJECT_BASICS, + rgp, + ); + let result = fullnode + .simulate_transaction(tx, VmChecks::Enabled) + .unwrap(); + assert_eq!(result.effects.status(), &ExecutionStatus::Success); + let field_input = result + .input_objects + .get(&field.object_id) + .expect("runtime-loaded field object should be in the input objects"); + assert_eq!(field_input.version(), field.version); } /// A gas payment that names an object which is not a gas coin is rejected, in diff --git a/crates/iota-grpc-server/src/transaction_execution_service/simulate.rs b/crates/iota-grpc-server/src/transaction_execution_service/simulate.rs index e2e766617f3a..ba0b59c78ae9 100644 --- a/crates/iota-grpc-server/src/transaction_execution_service/simulate.rs +++ b/crates/iota-grpc-server/src/transaction_execution_service/simulate.rs @@ -207,16 +207,7 @@ async fn simulate_single_transaction( }; // Simulate the transaction - let InternalSimulateResult { - effects, - events, - input_objects, - output_objects, - execution_result, - mock_gas_id, - suggested_gas_price, - gas_data, - } = executor + let simulation = executor .simulate_transaction(transaction_data.clone(), vm_checks) .map_err(|e| { RpcError::new( @@ -224,6 +215,16 @@ async fn simulate_single_transaction( format!("transaction simulation failed: {e}"), ) })?; + let InternalSimulateResult { + effects, + events, + execution_result, + mock_gas_id, + suggested_gas_price, + gas_data, + input_objects, + output_objects, + } = simulation; // Build the response let mut response = SimulatedTransaction::default(); diff --git a/crates/iota-grpc-server/tests/common/mod.rs b/crates/iota-grpc-server/tests/common/mod.rs index 84aa76488e6a..ba974e9990e2 100644 --- a/crates/iota-grpc-server/tests/common/mod.rs +++ b/crates/iota-grpc-server/tests/common/mod.rs @@ -20,13 +20,13 @@ use iota_sdk_types::{ }; use iota_types::{ crypto::AuthorityStrongQuorumSignInfo, - full_checkpoint_content::{CheckpointData, CheckpointTransaction}, + full_checkpoint_content::{Checkpoint, CheckpointTransaction, ExecutedTransaction}, gas_coin::GasCoin, messages_checkpoint::{ CertifiedCheckpointSummary, CheckpointContentsExt, CheckpointSequenceNumber, VerifiedCheckpoint, }, - object::{MoveStructExt, OBJECT_START_VERSION, Object}, + object::{MoveStructExt, OBJECT_START_VERSION, Object, ObjectSet}, storage::error::Result as StorageResult, transaction::VerifiedTransaction, }; @@ -332,24 +332,47 @@ impl iota_types::storage::ReadStore for MockGrpcStateReader { unimplemented!() } + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + None + } + fn get_checkpoint_data( &self, checkpoint: VerifiedCheckpoint, checkpoint_contents: CheckpointContents, - ) -> CheckpointData { + ) -> Checkpoint { let seq = checkpoint.sequence_number; - if self.is_large_checkpoint(seq) { - CheckpointData { - checkpoint_summary: checkpoint.into_inner(), - checkpoint_contents, - transactions: self.large_checkpoint_transactions.clone(), - } + let checkpoint_transactions = if self.is_large_checkpoint(seq) { + self.large_checkpoint_transactions.clone() } else { - CheckpointData { - checkpoint_summary: checkpoint.into_inner(), - checkpoint_contents, - transactions: self.checkpoint_transactions.clone(), - } + self.checkpoint_transactions.clone() + }; + + let mut object_set = ObjectSet::default(); + let transactions = checkpoint_transactions + .into_iter() + .map(|tx| { + for o in tx.input_objects.into_iter().chain(tx.output_objects) { + object_set.insert(o); + } + ExecutedTransaction { + transaction: tx.transaction.data().transaction().clone(), + signatures: tx.transaction.data().signatures().to_vec(), + effects: tx.effects, + events: tx.events, + unchanged_loaded_runtime_objects: vec![], + } + }) + .collect(); + + Checkpoint { + summary: checkpoint.into_inner(), + contents: checkpoint_contents, + transactions, + object_set, } } diff --git a/crates/iota-json-rpc/src/transaction_execution_api.rs b/crates/iota-json-rpc/src/transaction_execution_api.rs index d78d138e2c15..45434a7424e3 100644 --- a/crates/iota-json-rpc/src/transaction_execution_api.rs +++ b/crates/iota-json-rpc/src/transaction_execution_api.rs @@ -378,14 +378,15 @@ impl TransactionExecutionApi { // Resolve types against the objects the simulation wrote before falling back to // the store, so that packages published by the transaction itself are visible. let (input, events) = { + let output_objects = simulation.output_objects.clone(); let mut layout_resolver = epoch_store.executor().type_layout_resolver(Box::new( PackageStoreWithFallback::new( - ObjectMapPackageStore(&simulation.output_objects), + ObjectMapPackageStore(&output_objects), self.state.get_backing_package_store(), ), )); let module_cache = TemporaryModuleResolver::new( - &simulation.output_objects, + &output_objects, to_binary_config(epoch_store.protocol_config()), epoch_store.module_cache().clone(), ); @@ -540,11 +541,12 @@ impl TransactionExecutionApi { // Resolve types against the objects the simulation wrote before falling back to // the store, so that packages published by the transaction itself are visible. + let output_objects = &simulation.output_objects; let mut layout_resolver = epoch_store .executor() .type_layout_resolver(Box::new(PackageStoreWithFallback::new( - ObjectMapPackageStore(&simulation.output_objects), + ObjectMapPackageStore(output_objects), self.state.get_backing_package_store(), ))); diff --git a/crates/iota-transactional-test-runner/src/lib.rs b/crates/iota-transactional-test-runner/src/lib.rs index ef5914c2137c..a6fb2636016d 100644 --- a/crates/iota-transactional-test-runner/src/lib.rs +++ b/crates/iota-transactional-test-runner/src/lib.rs @@ -367,6 +367,13 @@ impl ReadStore for ValidatorWithFullnode { > { todo!() } + + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + None + } } impl ObjectStore for ValidatorWithFullnode { diff --git a/crates/iota-transactional-test-runner/src/simulator_persisted_store.rs b/crates/iota-transactional-test-runner/src/simulator_persisted_store.rs index 852ce3e8a028..6c05ffa7205b 100644 --- a/crates/iota-transactional-test-runner/src/simulator_persisted_store.rs +++ b/crates/iota-transactional-test-runner/src/simulator_persisted_store.rs @@ -598,6 +598,13 @@ impl ReadStore for PersistedStore { > { unimplemented!() } + + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + None + } } impl ObjectStore for PersistedStoreInnerReadOnlyWrapper { @@ -774,6 +781,13 @@ impl ReadStore for PersistedStoreInnerReadOnlyWrapper { > { todo!() } + + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + None + } } impl GrpcStateReader for PersistedStoreInnerReadOnlyWrapper { diff --git a/crates/iota-transactional-test-runner/src/test_adapter.rs b/crates/iota-transactional-test-runner/src/test_adapter.rs index 95a17a564943..5f9c94b6540e 100644 --- a/crates/iota-transactional-test-runner/src/test_adapter.rs +++ b/crates/iota-transactional-test-runner/src/test_adapter.rs @@ -2941,6 +2941,13 @@ impl ReadStore for IotaTestAdapter { > { self.executor.try_get_full_checkpoint_contents(digest) } + + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option> { + self.executor.get_unchanged_loaded_runtime_objects(digest) + } } fn find_iota_root_dir() -> PathBuf { diff --git a/crates/iota-types/src/full_checkpoint_content.rs b/crates/iota-types/src/full_checkpoint_content.rs index 8a6cd9ef9c80..200541a12def 100644 --- a/crates/iota-types/src/full_checkpoint_content.rs +++ b/crates/iota-types/src/full_checkpoint_content.rs @@ -5,8 +5,8 @@ use std::collections::BTreeMap; use iota_sdk_types::{ - ObjectId, ObjectReference, TransactionEffects, TransactionEvents, TransactionKind, - checkpoint::CheckpointContents, + ObjectId, ObjectReference, Transaction, TransactionEffects, TransactionEvents, TransactionKind, + UserSignature, checkpoint::CheckpointContents, }; use serde::{Deserialize, Serialize}; use tap::Pipe; @@ -16,8 +16,8 @@ use crate::{ effects::{TransactionEffectsAPI, TransactionEffectsExt}, iota_system_state::{IotaSystemStateTrait, get_iota_system_state}, messages_checkpoint::CertifiedCheckpointSummary, - object::Object, - storage::{BackingPackageStore, EpochInfo, error::Error as StorageError}, + object::{Object, ObjectSet}, + storage::{BackingPackageStore, EpochInfo, ObjectKey, error::Error as StorageError}, transaction::{TransactionAPI, TransactionEnvelope}, }; @@ -242,3 +242,86 @@ impl BackingPackageStore for CheckpointData { .pipe(Ok) } } + +// Never remove these asserts! +// These data structures are meant to be used in-memory, for structures that can +// be persisted in storage you should look at the protobuf versions. +static_assertions::assert_not_impl_any!(Checkpoint: serde::Serialize, serde::de::DeserializeOwned); +static_assertions::assert_not_impl_any!(ExecutedTransaction: serde::Serialize, serde::de::DeserializeOwned); + +#[derive(Clone, Debug)] +pub struct Checkpoint { + pub summary: CertifiedCheckpointSummary, + pub contents: CheckpointContents, + pub transactions: Vec, + pub object_set: ObjectSet, +} + +#[derive(Clone, Debug)] +pub struct ExecutedTransaction { + /// The input Transaction + pub transaction: Transaction, + pub signatures: Vec, + /// The effects produced by executing this transaction + pub effects: TransactionEffects, + /// The events, if any, emitted by this transactions during execution + pub events: Option, + pub unchanged_loaded_runtime_objects: Vec, +} + +impl From<&Checkpoint> for CheckpointData { + fn from(value: &Checkpoint) -> Self { + let get_object = |key: ObjectKey| { + let object = value.object_set.get(&key).cloned(); + if object.is_none() { + let msg = format!( + "object {key:?} missing from the object set of checkpoint {}", + value.summary.sequence_number + ); + debug_assert!(false, "{msg}"); + tracing::error!("{msg}"); + } + object + }; + let transactions = value + .transactions + .iter() + .map(|tx| { + let input_objects = tx + .effects + .modified_at_versions() + .into_iter() + .filter_map(|(object_id, version)| get_object(ObjectKey(object_id, version))) + .collect::>(); + let output_objects = tx + .effects + .all_changed_objects() + .into_iter() + .filter_map(|(object_ref, _owner, _kind)| get_object(object_ref.into())) + .collect::>(); + + CheckpointTransaction { + transaction: TransactionEnvelope::from_user_sig_data( + tx.transaction.clone(), + tx.signatures.clone(), + ), + effects: tx.effects.clone(), + events: tx.events.clone(), + input_objects, + output_objects, + } + }) + .collect(); + Self { + checkpoint_summary: value.summary.clone(), + checkpoint_contents: value.contents.clone(), + transactions, + } + } +} + +impl From for CheckpointData { + fn from(value: Checkpoint) -> Self { + (&value).into() + } +} diff --git a/crates/iota-types/src/object.rs b/crates/iota-types/src/object.rs index e4d3dfa8b1f5..5a85ba9382b9 100644 --- a/crates/iota-types/src/object.rs +++ b/crates/iota-types/src/object.rs @@ -32,6 +32,7 @@ use crate::{ iota_sdk_types_conversions::type_tag_sdk_to_core, layout_resolver::LayoutResolver, move_package::MovePackageExt, + storage::ObjectKey, timelock::timelock::TimeLock, }; @@ -891,6 +892,31 @@ impl Display for PastObjectRead { } } +/// A collection of objects keyed by their `(id, version)`, so both the input +/// and output versions of a mutated object can coexist. +// Never remove this assert! +// This data structure is meant to be used in-memory; for structures that can +// be persisted in storage you should look at the protobuf versions. +#[derive(Default, Clone, Debug)] +pub struct ObjectSet(BTreeMap); + +static_assertions::assert_not_impl_any!(ObjectSet: Serialize, serde::de::DeserializeOwned); + +impl ObjectSet { + pub fn get(&self, key: &ObjectKey) -> Option<&Object> { + self.0.get(key) + } + + pub fn insert(&mut self, object: Object) { + self.0 + .insert(ObjectKey(object.id(), object.version()), object); + } + + pub fn iter(&self) -> impl Iterator { + self.0.values() + } +} + #[cfg(test)] mod tests { use iota_sdk_types::{Address, ObjectId, TransactionDigest}; diff --git a/crates/iota-types/src/storage/mod.rs b/crates/iota-types/src/storage/mod.rs index 24ee74d091ee..51c244e27bbc 100644 --- a/crates/iota-types/src/storage/mod.rs +++ b/crates/iota-types/src/storage/mod.rs @@ -10,15 +10,15 @@ mod write_store; use std::{ cell::RefCell, - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, fmt::{Display, Formatter}, rc::Rc, sync::Arc, }; use iota_sdk_types::{ - ObjectId, ObjectReference, SenderSignedTransaction, TransactionDigest, TransactionEffects, - Version, move_package::MovePackage, + ObjectId, ObjectReference, SenderSignedTransaction, Transaction, TransactionDigest, + TransactionEffects, UnchangedSharedKind, Version, move_package::MovePackage, }; use itertools::Itertools; use move_binary_format::CompiledModule; @@ -42,7 +42,7 @@ use crate::{ error::{ExecutionError, IotaError, IotaResult}, execution::{DynamicallyLoadedObjectMetadata, ExecutionResults}, iota_sdk_types_conversions::identifier_core_to_sdk, - object::Object, + object::{Object, ObjectSet}, storage::error::Error as StorageError, transaction::{SenderSignedTransactionAPI, TransactionAPI}, }; @@ -628,3 +628,197 @@ pub fn get_transaction_output_objects( .collect::, _>>()?; Ok(output_objects) } + +// Returns an iterator over the ObjectKey's of objects read or written by this +// transaction +pub fn get_transaction_object_set( + transaction: &Transaction, + effects: &TransactionEffects, + unchanged_loaded_runtime_objects: &[ObjectKey], +) -> BTreeSet { + // enumerate the full set of input objects in order to properly capture + // immutable objects that may not appear in the effects. + // + // This excludes packages + let input_objects = transaction + .input_objects() + .expect("txn was executed and must have valid input objects") + .into_iter() + .filter_map(|input| { + input + .version() + .map(|version| ObjectKey(input.object_id(), version)) + }); + + // The full set of output/written objects as well as any of their initial + // versions + let modified_set = effects + .object_changes() + .into_iter() + .flat_map(|change| { + [ + change + .input_version + .map(|version| ObjectKey(change.id, version)), + change + .output_version + .map(|version| ObjectKey(change.id, version)), + ] + }) + .flatten(); + + // The set of unchanged shared objects + let unchanged_shared = effects + .unchanged_shared_objects() + .into_iter() + .flat_map(|unchanged| { + if let UnchangedSharedKind::ReadOnlyRoot { version, .. } = unchanged.1 { + Some(ObjectKey(unchanged.0, version)) + } else { + None + } + }); + + input_objects + .chain(modified_set) + .chain(unchanged_shared) + .chain(unchanged_loaded_runtime_objects.iter().copied()) + .collect() +} + +// Returns the ObjectKey's of the objects that were loaded during execution but +// left unchanged: everything in `loaded_runtime_objects` except packages and +// the objects the effects record as changed. +pub fn unchanged_loaded_runtime_objects( + _transaction: &Transaction, + effects: &TransactionEffects, + loaded_runtime_objects: &ObjectSet, +) -> Vec { + let mut unchanged_loaded_runtime_objects: BTreeMap<_, _> = loaded_runtime_objects + .iter() + // Don't include loaded packages (which are used for doing UID tracking inside the VM) + .filter(|o| !o.is_package()) + .map(|o| (o.id(), o.version())) + .collect(); + + // Remove any object that is referenced in the changed objects effects set since + // it would be redundant to include it again. + for change in effects.object_changes() { + unchanged_loaded_runtime_objects.remove(&change.id); + } + + unchanged_loaded_runtime_objects + .into_iter() + .map(|(id, v)| ObjectKey(id, v)) + .collect() +} + +/// Extend a simulation's input objects with the runtime-loaded objects the +/// effects record as modified, read from `object_store` at their pre-state +/// versions. +pub fn extend_input_objects_with_loaded_runtime_objects( + input_objects: &mut BTreeMap, + effects: &TransactionEffects, + loaded_runtime_objects: &BTreeMap, + object_store: &dyn ObjectStore, +) { + let modified_at: BTreeMap<_, _> = effects.modified_at_versions().into_iter().collect(); + for (id, metadata) in loaded_runtime_objects { + if input_objects.contains_key(id) || modified_at.get(id) != Some(&metadata.version) { + continue; + } + if let Some(object) = object_store.get_object_by_key(id, metadata.version) { + input_objects.insert(*id, object); + } + } +} + +// A BackingStore to pass to execution in order to track all objects loaded +// during execution. +// +// Today this is used to very accurately track the objects that were loaded but +// unchanged during execution. +pub struct TrackingBackingStore<'a> { + inner: &'a dyn BackingStore, + read_objects: RefCell, +} + +impl<'a> TrackingBackingStore<'a> { + pub fn new(inner: &'a dyn BackingStore) -> Self { + Self { + inner, + read_objects: Default::default(), + } + } + + pub fn into_read_objects(self) -> ObjectSet { + self.read_objects.into_inner() + } + + fn track_object(&self, object: &Object) { + self.read_objects.borrow_mut().insert(object.clone()); + } +} + +impl BackingPackageStore for TrackingBackingStore<'_> { + fn get_package_object(&self, package_id: &ObjectId) -> IotaResult> { + self.inner.get_package_object(package_id).inspect(|o| { + o.as_ref() + .inspect(|package| self.track_object(package.object())); + }) + } +} + +impl ChildObjectResolver for TrackingBackingStore<'_> { + fn read_child_object( + &self, + parent: &ObjectId, + child: &ObjectId, + child_version_upper_bound: Version, + ) -> IotaResult> { + self.inner + .read_child_object(parent, child, child_version_upper_bound) + .inspect(|o| { + o.as_ref().inspect(|object| self.track_object(object)); + }) + } + + fn get_object_received_at_version( + &self, + owner: &ObjectId, + receiving_object_id: &ObjectId, + receive_object_at_version: Version, + epoch_id: EpochId, + ) -> IotaResult> { + self.inner + .get_object_received_at_version( + owner, + receiving_object_id, + receive_object_at_version, + epoch_id, + ) + .inspect(|o| { + o.as_ref().inspect(|object| self.track_object(object)); + }) + } +} + +impl ObjectStore for TrackingBackingStore<'_> { + fn try_get_object(&self, object_id: &ObjectId) -> error::Result> { + self.inner.try_get_object(object_id).inspect(|o| { + o.as_ref().inspect(|object| self.track_object(object)); + }) + } + + fn try_get_object_by_key( + &self, + object_id: &ObjectId, + version: VersionNumber, + ) -> error::Result> { + self.inner + .try_get_object_by_key(object_id, version) + .inspect(|o| { + o.as_ref().inspect(|object| self.track_object(object)); + }) + } +} diff --git a/crates/iota-types/src/storage/read_store.rs b/crates/iota-types/src/storage/read_store.rs index c2f8f947b410..f957ba7125e0 100644 --- a/crates/iota-types/src/storage/read_store.rs +++ b/crates/iota-types/src/storage/read_store.rs @@ -2,7 +2,10 @@ // Modifications Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{BTreeSet, HashMap}, + sync::Arc, +}; use iota_sdk_types::{ Address, CheckpointContentsDigest, CheckpointDigest, MoveObjectType, ObjectId, @@ -16,14 +19,14 @@ use super::{ObjectStore, error::Result}; use crate::{ base_types::{EpochId, ObjectType}, committee::Committee, - full_checkpoint_content::{CheckpointData, CheckpointTransaction}, + full_checkpoint_content::{Checkpoint, CheckpointTransaction, ExecutedTransaction}, iota_system_state::{IotaSystemState, IotaSystemStateTrait}, messages_checkpoint::{ CertifiedCheckpointSummary, CheckpointContentsExt, CheckpointSequenceNumber, FullCheckpointContents, VerifiedCheckpoint, }, - object::Object, - storage::{get_transaction_input_objects, get_transaction_output_objects}, + object::{Object, ObjectSet}, + storage::{ObjectKey, get_transaction_input_objects, get_transaction_output_objects}, transaction::VerifiedTransaction, }; @@ -271,6 +274,11 @@ pub trait ReadStore: ObjectStore { .expect("storage access failed") } + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option>; + // Extra Checkpoint fetching apis // @@ -421,7 +429,7 @@ pub trait ReadStore: ObjectStore { &self, checkpoint: VerifiedCheckpoint, checkpoint_contents: CheckpointContents, - ) -> anyhow::Result { + ) -> anyhow::Result { let transaction_digests = checkpoint_contents .iter() .map(|execution_digests| execution_digests.transaction) @@ -432,13 +440,54 @@ pub trait ReadStore: ObjectStore { let mut transactions = Vec::with_capacity(txs_with_events_and_effects.len()); for tx_with_events_and_effects in txs_with_events_and_effects { - transactions.push(self.get_checkpoint_transaction(tx_with_events_and_effects)?); + let tx = tx_with_events_and_effects.transaction; + let transaction = ExecutedTransaction { + transaction: tx.data().transaction().clone(), + signatures: tx.data().signatures().to_vec(), + effects: tx_with_events_and_effects.effects, + events: tx_with_events_and_effects.events, + unchanged_loaded_runtime_objects: self + .get_unchanged_loaded_runtime_objects(tx.digest()) + // We don't write empty sets to the DB to save space, so if this load went + // through the writeback cache to the DB itself it wouldn't find an entry. + .unwrap_or_default(), + }; + transactions.push(transaction); } - let checkpoint_data = CheckpointData { - checkpoint_summary: checkpoint.into(), - checkpoint_contents, + let object_set = { + let refs = transactions + .iter() + .flat_map(|tx| { + crate::storage::get_transaction_object_set( + &tx.transaction, + &tx.effects, + &tx.unchanged_loaded_runtime_objects, + ) + }) + .collect::>() + .into_iter() + .collect::>(); + + let objects = self.multi_get_objects_by_key(&refs); + + let mut object_set = ObjectSet::default(); + for (idx, object) in objects.into_iter().enumerate() { + object_set.insert(object.ok_or_else(|| { + crate::storage::error::Error::custom(format!( + "unable to load object {:?}", + refs[idx] + )) + })?); + } + object_set + }; + + let checkpoint_data = Checkpoint { + summary: checkpoint.into(), + contents: checkpoint_contents, transactions, + object_set, }; Ok(checkpoint_data) @@ -449,7 +498,7 @@ pub trait ReadStore: ObjectStore { &self, checkpoint: VerifiedCheckpoint, checkpoint_contents: CheckpointContents, - ) -> CheckpointData { + ) -> Checkpoint { self.try_get_checkpoint_data(checkpoint, checkpoint_contents) .expect("storage access failed") } @@ -551,6 +600,13 @@ impl ReadStore for &T { (*self).try_multi_get_events(digests) } + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option> { + (*self).get_unchanged_loaded_runtime_objects(digest) + } + fn try_get_full_checkpoint_contents_by_sequence_number( &self, sequence_number: CheckpointSequenceNumber, @@ -569,7 +625,7 @@ impl ReadStore for &T { &self, checkpoint: VerifiedCheckpoint, checkpoint_contents: CheckpointContents, - ) -> anyhow::Result { + ) -> anyhow::Result { (*self).try_get_checkpoint_data(checkpoint, checkpoint_contents) } } @@ -670,6 +726,13 @@ impl ReadStore for Box { (**self).try_multi_get_events(digests) } + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option> { + (**self).get_unchanged_loaded_runtime_objects(digest) + } + fn try_get_full_checkpoint_contents_by_sequence_number( &self, sequence_number: CheckpointSequenceNumber, @@ -688,7 +751,7 @@ impl ReadStore for Box { &self, checkpoint: VerifiedCheckpoint, checkpoint_contents: CheckpointContents, - ) -> anyhow::Result { + ) -> anyhow::Result { (**self).try_get_checkpoint_data(checkpoint, checkpoint_contents) } } @@ -789,6 +852,13 @@ impl ReadStore for Arc { (**self).try_multi_get_events(digests) } + fn get_unchanged_loaded_runtime_objects( + &self, + digest: &TransactionDigest, + ) -> Option> { + (**self).get_unchanged_loaded_runtime_objects(digest) + } + fn try_get_full_checkpoint_contents_by_sequence_number( &self, sequence_number: CheckpointSequenceNumber, @@ -807,7 +877,7 @@ impl ReadStore for Arc { &self, checkpoint: VerifiedCheckpoint, checkpoint_contents: CheckpointContents, - ) -> anyhow::Result { + ) -> anyhow::Result { (**self).try_get_checkpoint_data(checkpoint, checkpoint_contents) } } diff --git a/crates/iota-types/src/storage/shared_in_memory_store.rs b/crates/iota-types/src/storage/shared_in_memory_store.rs index c456faeef973..a8aad2f3363f 100644 --- a/crates/iota-types/src/storage/shared_in_memory_store.rs +++ b/crates/iota-types/src/storage/shared_in_memory_store.rs @@ -130,6 +130,13 @@ impl ReadStore for SharedInMemoryStore { .pipe(Ok) } + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + todo!() + } + fn try_get_latest_checkpoint(&self) -> Result { todo!() } @@ -570,6 +577,13 @@ impl ReadStore for SingleCheckpointSharedInMemoryStore { self.0.try_get_events(digest) } + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + todo!() + } + fn try_get_latest_checkpoint(&self) -> Result { todo!() } diff --git a/crates/iota-types/src/transaction_executor.rs b/crates/iota-types/src/transaction_executor.rs index b9e9f832f6e5..f92f8481f001 100644 --- a/crates/iota-types/src/transaction_executor.rs +++ b/crates/iota-types/src/transaction_executor.rs @@ -18,7 +18,7 @@ use crate::{ }, }; -/// Trait to define the interface for how the REST service interacts with a +/// Trait to define the interface for how the gRPC service interacts with a /// QuorumDriver or a simulated transaction executor. #[async_trait::async_trait] pub trait TransactionExecutor: Send + Sync { @@ -83,7 +83,12 @@ pub struct CachedTransactionData { pub struct SimulateTransactionResult { pub effects: TransactionEffects, pub events: Option, + /// Every object the transaction ran with as input — including immutable + /// and read-only shared inputs, the packages it calls, and the gas coins + /// (the mock one included) — plus the runtime-loaded objects (e.g. dynamic + /// fields) it modified, at their pre-state versions, keyed by id. pub input_objects: BTreeMap, + /// The objects written by the transaction, keyed by id. pub output_objects: BTreeMap, /// The return values and mutable-reference outputs of every command, under /// either [`VmChecks`] — both run through the executor's dev-inspect entry diff --git a/crates/iota-vm-sdk/src/executor/local_vm.rs b/crates/iota-vm-sdk/src/executor/local_vm.rs index f71504343629..fdd88b94b32a 100644 --- a/crates/iota-vm-sdk/src/executor/local_vm.rs +++ b/crates/iota-vm-sdk/src/executor/local_vm.rs @@ -438,12 +438,14 @@ impl LocalVm { self.apply_effects(&sim); } + let input_objects = sim.input_objects.into_values().collect(); + let output_objects = sim.output_objects.into_values().collect(); Ok(ExecutionResult { effects: sim.effects, events: sim.events, command_results: sim.execution_result.unwrap_or_default(), - input_objects: sim.input_objects.into_values().collect(), - output_objects: sim.output_objects.into_values().collect(), + input_objects, + output_objects, gas_summary, mock_gas_id: sim.mock_gas_id, status, @@ -459,8 +461,8 @@ impl LocalVm { // `output_objects` is authoritative for what survives; then drop what // was deleted or wrapped. `unwrapped_then_deleted` objects were nested, // never standalone store entries, so they need no removal. - for obj in sim.output_objects.values() { - self.store.insert(obj.clone()); + for obj in sim.output_objects.values().cloned() { + self.store.insert(obj); } for objref in sim.effects.deleted() { self.store.remove(&objref.object_id); diff --git a/crates/iota-vm-sdk/src/executor/prepare.rs b/crates/iota-vm-sdk/src/executor/prepare.rs index e635021bcf56..303a87deefb7 100644 --- a/crates/iota-vm-sdk/src/executor/prepare.rs +++ b/crates/iota-vm-sdk/src/executor/prepare.rs @@ -262,6 +262,7 @@ pub(super) fn execute_prepared( let dev_inspect = matches!(mode, ExecutionMode::DevInspect); let (kind, signer, gas_data) = transaction.execution_parts(); + // `dev_inspect_transaction` accepts no `MoveTraceBuilder`; tracing is only // available on the `authenticate_then_execute_transaction_to_effects` path. let (inner_temp_store, _, effects, execution_result) = env.executor.dev_inspect_transaction( @@ -282,6 +283,7 @@ pub(super) fn execute_prepared( ); Ok(simulation_result( + store, inner_temp_store, effects, execution_result, @@ -292,14 +294,23 @@ pub(super) fn execute_prepared( /// Assemble the engine's raw outputs into a [`SimulateTransactionResult`]. fn simulation_result( + store: &dyn BackingStore, inner_temp_store: InnerTemporaryStore, effects: TransactionEffects, execution_result: Result, iota_types::error::ExecutionError>, mock_gas_id: Option, gas_data: GasPayment, ) -> SimulateTransactionResult { + let mut input_objects = inner_temp_store.input_objects; + iota_types::storage::extend_input_objects_with_loaded_runtime_objects( + &mut input_objects, + &effects, + &inner_temp_store.loaded_runtime_objects, + store.as_object_store(), + ); + SimulateTransactionResult { - input_objects: inner_temp_store.input_objects, + input_objects, output_objects: inner_temp_store.written, events: effects.events_digest().map(|_| inner_temp_store.events), effects, @@ -456,6 +467,7 @@ pub(super) fn execute_with_move_authenticators( // results, so a signed `MoveAuthenticator` run carries none. Ok(( simulation_result( + store, inner_temp_store, effects, execution_result.map(|_| Vec::new()), diff --git a/crates/simulacrum/src/epoch_state.rs b/crates/simulacrum/src/epoch_state.rs index 481377defdd8..66cd9a10d2ac 100644 --- a/crates/simulacrum/src/epoch_state.rs +++ b/crates/simulacrum/src/epoch_state.rs @@ -289,8 +289,16 @@ impl EpochState { checks.disabled(), ); + let mut input_objects = inner_temp_store.input_objects; + iota_types::storage::extend_input_objects_with_loaded_runtime_objects( + &mut input_objects, + &effects, + &inner_temp_store.loaded_runtime_objects, + store.backing_store().as_object_store(), + ); + Ok(SimulateTransactionResult { - input_objects: inner_temp_store.input_objects, + input_objects, output_objects: inner_temp_store.written, events: effects.events_digest().map(|_| inner_temp_store.events), effects, diff --git a/crates/simulacrum/src/lib.rs b/crates/simulacrum/src/lib.rs index 52530869b66a..0279346466ed 100644 --- a/crates/simulacrum/src/lib.rs +++ b/crates/simulacrum/src/lib.rs @@ -57,7 +57,7 @@ use iota_types::{ object::Object, programmable_transaction_builder::ProgrammableTransactionBuilder, signature::VerifyParams, - storage::{EpochInfoV2, ObjectStore, ReadStore, TransactionInfo}, + storage::{EpochInfoV2, ObjectKey, ObjectStore, ReadStore, TransactionInfo}, transaction::{TransactionAPI, TransactionEnvelope, VerifiedTransaction}, }; use rand::rngs::OsRng; @@ -551,7 +551,9 @@ impl Simulacrum { let path = self.inner.read().unwrap().data_ingestion_path.clone(); if let Some(data_path) = path { let file_name = format!("{}.chk", checkpoint.sequence_number); - let checkpoint_data = self.try_get_checkpoint_data(checkpoint, checkpoint_contents)?; + let checkpoint_data: iota_types::full_checkpoint_content::CheckpointData = self + .try_get_checkpoint_data(checkpoint, checkpoint_contents)? + .into(); std::fs::create_dir_all(&data_path)?; let blob = Blob::encode(&checkpoint_data, BlobEncoding::Bcs)?; std::fs::write(data_path.join(file_name), blob.to_bytes())?; @@ -732,6 +734,13 @@ impl ReadStore for Simulacrum { }) }) } + + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + None + } } impl GrpcStateReader for Simulacrum { diff --git a/crates/simulacrum/src/store/in_mem_store.rs b/crates/simulacrum/src/store/in_mem_store.rs index 5c1381f47a59..8a3d0fe9d2cd 100644 --- a/crates/simulacrum/src/store/in_mem_store.rs +++ b/crates/simulacrum/src/store/in_mem_store.rs @@ -500,6 +500,13 @@ impl ReadStore for InMemoryStore { ) }) } + + fn get_unchanged_loaded_runtime_objects( + &self, + _digest: &TransactionDigest, + ) -> Option> { + None + } } #[derive(Debug)]