Feature add block import trace to eth simulate (#10211)

* initial addition of optional block import tracing to eth_simulateV1
* move block traceEnd prior to trielog write


---------

Signed-off-by: garyschulte <garyschulte@gmail.com>
Co-authored-by: Justin Florentine <justin+github@florentine.us>

Author: garyschulte

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthSimulateV1.java

@@ -40,6 +40,10 @@
 import org.hyperledger.besu.ethereum.transaction.TransactionSimulator;
 import org.hyperledger.besu.ethereum.transaction.exceptions.BlockStateCallError;
 import org.hyperledger.besu.ethereum.transaction.exceptions.BlockStateCallException;
+import org.hyperledger.besu.evm.tracing.OperationTracer;
+import org.hyperledger.besu.plugin.ServiceManager;
+import org.hyperledger.besu.plugin.services.BlockImportTracerProvider;
+import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
 
 import java.util.List;
 import java.util.Optional;
@@ -52,8 +56,10 @@ public class EthSimulateV1 extends AbstractBlockParameterOrBlockHashMethod {
 
   private final BlockSimulator blockSimulator;
   private final ProtocolSchedule protocolSchedule;
+  private final BlockImportTracerProvider blockImportTracerProvider;
 
   public EthSimulateV1(
+      final ServiceManager serviceManager,
       final BlockchainQueries blockchainQueries,
       final ProtocolSchedule protocolSchedule,
       final TransactionSimulator transactionSimulator,
@@ -69,6 +75,12 @@ public EthSimulateV1(
             miningConfiguration,
             blockchainQueries.getBlockchain(),
             apiConfiguration.getGasCap());
+
+    this.blockImportTracerProvider =
+        Optional.ofNullable(serviceManager)
+            .flatMap(mgr -> mgr.getService(BlockImportTracerProvider.class))
+            // if block import tracer provider is not specified by plugin, default to no tracing
+            .orElse(__ -> BlockAwareOperationTracer.NO_TRACING);
   }
 
   @VisibleForTesting
@@ -79,6 +91,7 @@ public EthSimulateV1(
     super(blockchainQueries);
     this.protocolSchedule = protocolSchedule;
     this.blockSimulator = blockSimulator;
+    this.blockImportTracerProvider = __ -> BlockAwareOperationTracer.NO_TRACING;
   }
 
   @Override
@@ -143,8 +156,14 @@ protected Object resultByBlockHeader(
   }
 
   private Object process(final BlockHeader header, final SimulateV1Parameter simulateV1Parameter) {
+    var blockImportTracer =
+        simulateV1Parameter.isTraceBlockImport()
+            ? blockImportTracerProvider.getBlockImportTracer(header)
+            : OperationTracer.NO_TRACING;
+
     final List<BlockSimulationResult> simulationResults =
-        blockSimulator.process(header, simulateV1Parameter);
+        blockSimulator.process(header, simulateV1Parameter, blockImportTracer);
+
     return simulationResults.stream()
         .map(
             result ->

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/SimulateV1Parameter.java

@@ -23,13 +23,21 @@
 
 public class SimulateV1Parameter extends BlockSimulationParameter {
 
+  private final boolean traceBlockImport;
+
   @JsonCreator
   public SimulateV1Parameter(
       @JsonProperty("blockStateCalls") final List<JsonBlockStateCallParameter> blockStateCalls,
       @JsonProperty("validation") final boolean validation,
       @JsonProperty("traceTransfers") final boolean traceTransfers,
       @JsonProperty("returnFullTransactions") final boolean returnFullTransactions,
-      @JsonProperty("returnTrieLog") final boolean returnTrieLog) {
+      @JsonProperty("returnTrieLog") final boolean returnTrieLog,
+      @JsonProperty("traceBlockImport") final boolean traceBlockImport) {
     super(blockStateCalls, validation, traceTransfers, returnFullTransactions, returnTrieLog);
+    this.traceBlockImport = traceBlockImport;
+  }
+
+  public boolean isTraceBlockImport() {
+    return traceBlockImport;
   }
 }

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthJsonRpcMethods.java

@@ -71,6 +71,7 @@
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.p2p.rlpx.wire.Capability;
 import org.hyperledger.besu.ethereum.transaction.TransactionSimulator;
+import org.hyperledger.besu.plugin.ServiceManager;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 import java.util.Map;
@@ -92,6 +93,7 @@ public class EthJsonRpcMethods extends ApiGroupJsonRpcMethods {
   private final ApiConfiguration apiConfiguration;
   private final GenesisConfigOptions genesisConfigOptions;
   private final TransactionSimulator transactionSimulator;
+  private final ServiceManager serviceManager;
   private final MetricsSystem metricsSystem;
 
   public EthJsonRpcMethods(
@@ -106,6 +108,7 @@ public EthJsonRpcMethods(
       final ApiConfiguration apiConfiguration,
       final GenesisConfigOptions genesisConfigOptions,
       final TransactionSimulator transactionSimulator,
+      final ServiceManager serviceManager,
       final MetricsSystem metricsSystem) {
     this.blockchainQueries = blockchainQueries;
     this.synchronizer = synchronizer;
@@ -118,6 +121,7 @@ public EthJsonRpcMethods(
     this.apiConfiguration = apiConfiguration;
     this.genesisConfigOptions = genesisConfigOptions;
     this.transactionSimulator = transactionSimulator;
+    this.serviceManager = serviceManager;
     this.metricsSystem = metricsSystem;
   }
 
@@ -173,6 +177,7 @@ protected Map<String, JsonRpcMethod> create() {
             new EthBlobBaseFee(blockchainQueries.getBlockchain(), protocolSchedule),
             new EthMaxPriorityFeePerGas(blockchainQueries),
             new EthSimulateV1(
+                serviceManager,
                 blockchainQueries,
                 protocolSchedule,
                 transactionSimulator,

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java

@@ -136,6 +136,7 @@ public Map<String, JsonRpcMethod> methods(
                   apiConfiguration,
                   genesisConfigOptions,
                   transactionSimulator,
+                  protocolContext.getPluginServiceManager(),
                   metricsSystem),
               new NetJsonRpcMethods(
                   p2pNetwork,

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthSimulateV1Test.java

@@ -39,6 +39,7 @@
 import org.hyperledger.besu.ethereum.transaction.exceptions.BlockStateCallError;
 import org.hyperledger.besu.ethereum.transaction.exceptions.BlockStateCallException;
 import org.hyperledger.besu.evm.precompile.PrecompileContractRegistry;
+import org.hyperledger.besu.evm.tracing.OperationTracer;
 
 import java.util.List;
 import java.util.Map;
@@ -76,6 +77,7 @@ public class EthSimulateV1Test {
   public void setUp() {
     method =
         new EthSimulateV1(
+            null,
             blockchainQueries,
             protocolSchedule,
             transactionSimulator,
@@ -107,7 +109,7 @@ public void shouldReturnBlockNotFoundErrorWhenFutureBlockNumberSpecified() {
   public void shouldReturnInvalidParamsWhenUpfrontCostExceedsBalanceWithValidation() {
     setupMethodWithMockSimulator();
     setupBlockchainForLatest();
-    when(blockSimulator.process(any(BlockHeader.class), any()))
+    when(blockSimulator.process(any(BlockHeader.class), any(), any(OperationTracer.class)))
         .thenThrow(
             new BlockStateCallException(
                 "Upfront cost exceeds balance", BlockStateCallError.UPFRONT_COST_EXCEEDS_BALANCE));
@@ -125,7 +127,7 @@ public void shouldReturnInvalidParamsWhenUpfrontCostExceedsBalanceWithValidation
   public void shouldReturnOriginalErrorCodeWhenUpfrontCostExceedsBalanceWithoutValidation() {
     setupMethodWithMockSimulator();
     setupBlockchainForLatest();
-    when(blockSimulator.process(any(BlockHeader.class), any()))
+    when(blockSimulator.process(any(BlockHeader.class), any(), any(OperationTracer.class)))
         .thenThrow(
             new BlockStateCallException(
                 "Upfront cost exceeds balance", BlockStateCallError.UPFRONT_COST_EXCEEDS_BALANCE));
@@ -143,7 +145,8 @@ public void shouldReturnOriginalErrorCodeWhenUpfrontCostExceedsBalanceWithoutVal
   public void shouldNotReturnInvalidParamsWhenInputAndDataHaveDifferentValues() {
     setupMethodWithMockSimulator();
     setupBlockchainForLatest();
-    when(blockSimulator.process(any(BlockHeader.class), any())).thenReturn(List.of());
+    when(blockSimulator.process(any(BlockHeader.class), any(), any(OperationTracer.class)))
+        .thenReturn(List.of());
 
     // Reproduces issue #9960: both input and data provided with different values.
     // Other EL clients (Geth, Nethermind, Reth, Erigon) accept this and use input.
@@ -166,7 +169,8 @@ public void shouldNotReturnInvalidParamsWhenInputAndDataHaveDifferentValues() {
 
     final ArgumentCaptor<BlockSimulationParameter> captor =
         ArgumentCaptor.forClass(BlockSimulationParameter.class);
-    verify(blockSimulator).process(any(BlockHeader.class), captor.capture());
+    verify(blockSimulator)
+        .process(any(BlockHeader.class), captor.capture(), any(OperationTracer.class));
     final Bytes payload =
         captor.getValue().getBlockStateCalls().get(0).getCalls().get(0).getPayload().orElseThrow();
     assertThat(payload).isEqualTo(Bytes.fromHexString("0xDEADBEEF"));
@@ -202,7 +206,7 @@ private void setupBlockchainForLatest() {
   }
 
   private SimulateV1Parameter simulateParameter(final boolean validation) {
-    return new SimulateV1Parameter(List.of(), validation, false, false, false);
+    return new SimulateV1Parameter(List.of(), validation, false, false, false, false);
   }
 
   private JsonRpcRequestContext ethSimulateV1Request(

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthSimulateV1TrielogTest.java

@@ -90,6 +90,7 @@ public void createStorage() {
 
     method =
         new EthSimulateV1(
+            null,
             blockchainQueries,
             protocolSchedule,
             new TransactionSimulator(
@@ -115,7 +116,7 @@ public void shouldReturnTrielogWhenReturnTrieLogTrue() {
 
     // Create simulation parameter with returnTrieLog = true
     SimulateV1Parameter simulateV1Parameter =
-        new SimulateV1Parameter(List.of(blockStateCall), false, false, false, true);
+        new SimulateV1Parameter(List.of(blockStateCall), false, false, false, true, false);
 
     JsonRpcRequestContext request =
         new JsonRpcRequestContext(
@@ -166,7 +167,7 @@ public void shouldNotReturnTrielogWhenReturnTrieLogFalse() {
 
     // Create simulation parameter with returnTrieLog = false
     SimulateV1Parameter simulateV1Parameter =
-        new SimulateV1Parameter(List.of(blockStateCall), false, false, false, false);
+        new SimulateV1Parameter(List.of(blockStateCall), false, false, false, false, false);
 
     JsonRpcRequestContext request =
         new JsonRpcRequestContext(
@@ -199,7 +200,7 @@ public void shouldReturnTrielogWithMultipleTransactions() {
         new JsonBlockStateCallParameter(List.of(callParameter1, callParameter2), null, null);
 
     SimulateV1Parameter simulateV1Parameter =
-        new SimulateV1Parameter(List.of(blockStateCall), false, false, false, true);
+        new SimulateV1Parameter(List.of(blockStateCall), false, false, false, true, false);
 
     JsonRpcRequestContext request =
         new JsonRpcRequestContext(

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/SimulateV1ParameterTest.java

@@ -38,7 +38,7 @@ private void validateSimulateV1Parameter(
       final List<JsonBlockStateCallParameter> blockStateCalls,
       final BlockStateCallError expectedError) {
     SimulateV1Parameter simulateV1Parameter =
-        new SimulateV1Parameter(blockStateCalls, false, false, false, false);
+        new SimulateV1Parameter(blockStateCalls, false, false, false, false, false);
     Optional<BlockStateCallError> maybeValidationError =
         simulateV1Parameter.validate(VALID_PRECOMPILE_ADDRESSES);
     assertThat(maybeValidationError).isPresent();

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/BlockSimulator.java

@@ -51,7 +51,6 @@
 import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList.BlockAccessListBuilder;
 import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessListFactory;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket;
-import org.hyperledger.besu.ethereum.mainnet.feemarket.ExcessBlobGasCalculator;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket;
 import org.hyperledger.besu.ethereum.mainnet.requests.RequestProcessingContext;
 import org.hyperledger.besu.ethereum.mainnet.requests.RequestProcessorCoordinator;
@@ -67,6 +66,7 @@
 import org.hyperledger.besu.evm.tracing.OperationTracer;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 import org.hyperledger.besu.plugin.data.BlockOverrides;
+import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -79,6 +79,8 @@
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Simulates the execution of a block, processing transactions and applying state overrides. This
@@ -91,6 +93,8 @@
  */
 public class BlockSimulator {
 
+  private static final Logger LOG = LoggerFactory.getLogger(BlockSimulator.class);
+
   private static final TransactionValidationParams STRICT_VALIDATION_PARAMS =
       TransactionValidationParams.blockSimulatorStrict();
 
@@ -280,9 +284,22 @@ private BlockSimulationResult processBlockStateCall(
             ws,
             protocolSpec,
             blockHashLookup,
-            OperationTracer.NO_TRACING,
+            operationTracer,
             Optional.empty());
 
+    // if operationTracer is block-aware, traceStart and hold onto the option ref for traceEnd
+    var maybeBlockAwareOperationTracer =
+        Optional.of(operationTracer)
+            .filter(z -> z instanceof BlockAwareOperationTracer)
+            .map(BlockAwareOperationTracer.class::cast);
+
+    maybeBlockAwareOperationTracer.ifPresent(
+        tracer -> {
+          LOG.trace("traceStartBlock sim for {}", overridenBaseBlockHeader.toLogString());
+          tracer.traceStartBlock(
+              ws, overridenBaseBlockHeader, overridenBaseBlockHeader.getCoinbase());
+        });
+
     protocolSpec
         .getPreExecutionProcessor()
         .process(blockProcessingContext, preExecutionAccessLocationTracker);
@@ -340,14 +357,18 @@ private BlockSimulationResult processBlockStateCall(
       rewardUpdater.commit();
     }
 
-    return createFinalBlock(
-        overridenBaseBlockHeader,
-        blockStateCallSimulationResult,
-        blockOverrides,
-        protocolSpec,
-        ws,
-        maybeRequests,
-        returnTrieLog);
+    var finalBlock =
+        createFinalBlock(
+            overridenBaseBlockHeader,
+            blockStateCallSimulationResult,
+            blockOverrides,
+            protocolSpec,
+            ws,
+            maybeRequests,
+            maybeBlockAwareOperationTracer,
+            returnTrieLog);
+
+    return finalBlock;
   }
 
   protected BlockStateCallSimulationResult processTransactions(
@@ -516,6 +537,7 @@ private BlockSimulationResult createFinalBlock(
       final ProtocolSpec protocolSpec,
       final MutableWorldState ws,
       final Optional<List<Request>> maybeRequests,
+      final Optional<BlockAwareOperationTracer> maybeBlockAwareOperationTracer,
       final boolean returnTrieLog) {
 
     List<Transaction> transactions = simResult.getTransactions();
@@ -557,6 +579,13 @@ private BlockSimulationResult createFinalBlock(
 
     Block block = new Block(finalBlockHeader, new BlockBody(transactions, List.of(), withdrawals));
 
+    // if we have a block-aware operation tracer, trace end block here
+    maybeBlockAwareOperationTracer.ifPresent(
+        tracer -> {
+          LOG.trace("traceEndBlock sim for {}", blockHeader.toLogString());
+          tracer.traceEndBlock(blockHeader, block.getBody());
+        });
+
     if (returnTrieLog && ws instanceof PathBasedWorldState) {
       // if requested and path-based worldstate, return result with trielog and serializer:
       var pathBasedArchive = (PathBasedWorldStateProvider) worldStateArchive;
@@ -657,8 +686,7 @@ protected BlockHeader overrideBlockHeader(
 
     // Cancun+: excessBlobGas
     if (newProtocolSpec.getFeeMarket().implementsBlobFee()) {
-      builder.excessBlobGas(
-          ExcessBlobGasCalculator.calculateExcessBlobGasForParent(newProtocolSpec, header));
+      builder.excessBlobGas(calculateExcessBlobGasForParent(newProtocolSpec, header));
     } else {
       builder.excessBlobGas(null);
       builder.blobGasUsed(null);