From 3f840de84f52bc7d73e1f5278227791506cf5034 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Fri, 14 Feb 2025 14:55:01 -0600 Subject: [PATCH 1/4] initial logic, not completely finished. Signed-off-by: Alfredo Gutierrez --- .../common/hasher/MerkleProofCalculator.java | 35 ++++++++ .../hasher/NaiveStreamingTreeHasher.java | 2 + .../hasher/MerkleProofCalculatorTest.java | 90 +++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java create mode 100644 common/src/test/java/com/hedera/block/common/hasher/MerkleProofCalculatorTest.java diff --git a/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java b/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java new file mode 100644 index 000000000..513ed4ee9 --- /dev/null +++ b/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java @@ -0,0 +1,35 @@ +package com.hedera.block.common.hasher; + +import java.util.ArrayList; +import java.util.List; + +public class MerkleProofCalculator { + + // Calculate the Merkle root hash of a list of leaf hashes. + // Requires the completeMerkleTree to be a list of lists of byte arrays, where each list represents a level of the tree, fully padded. + public List calculateMerkleProof(List> completeMerkleTree, int leafIndex) { + List proof = new ArrayList<>(); + int index = leafIndex; + + // Iterate over each level except the root. + for(int level = 0; level < completeMerkleTree.size() - 1; level++) { + List levelHashes = completeMerkleTree.get(level); + if(index % 2 == 0) { + // If the index is even, the sibling is the next hash in the list. + proof.add(levelHashes.get(index + 1)); + } else { + // If the index is odd, the sibling is the previous hash in the list. + proof.add(levelHashes.get(index - 1)); + } + // Move up to the parent level. + index /= 2; + } + + return proof; + } + + public int indexIfExist(List leafHashes, byte[] leafHash) { + return leafHashes.indexOf(leafHash); + } + +} diff --git a/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java b/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java index 33844edf1..296531ee5 100644 --- a/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java +++ b/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java @@ -22,6 +22,7 @@ public class NaiveStreamingTreeHasher implements StreamingTreeHasher { private final List leafHashes = new ArrayList<>(); private boolean rootHashRequested = false; + private final List> completeMerkleTree = new LinkedList<>(); /** * Constructor for the {@link NaiveStreamingTreeHasher}. @@ -56,6 +57,7 @@ public CompletableFuture rootHash() { } } while (hashes.size() > 1) { + completeMerkleTree.add(new ArrayList<>(hashes)); final Queue newLeafHashes = new LinkedList<>(); while (!hashes.isEmpty()) { final byte[] left = hashes.poll(); diff --git a/common/src/test/java/com/hedera/block/common/hasher/MerkleProofCalculatorTest.java b/common/src/test/java/com/hedera/block/common/hasher/MerkleProofCalculatorTest.java new file mode 100644 index 000000000..661f611fe --- /dev/null +++ b/common/src/test/java/com/hedera/block/common/hasher/MerkleProofCalculatorTest.java @@ -0,0 +1,90 @@ +package com.hedera.block.common.hasher; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class MerkleProofCalculatorTest { + + @Test + public void testCalculateMerkleProof() { + // Build a simple Merkle tree: + // Level 0 (leaves): [L0, L1, L2, L3] + // Level 1 (internal nodes): [H0_1, H2_3] + // Level 2 (root): [H0_3] + // + // In this precomputed tree, assume: + // - H0_1 is the hash of L0 and L1. + // - H2_3 is the hash of L2 and L3. + // - H0_3 is the hash of H0_1 and H2_3. + // For our test, we use literal byte arrays with the given names. + + byte[] L0 = "L0".getBytes(StandardCharsets.UTF_8); + byte[] L1 = "L1".getBytes(StandardCharsets.UTF_8); + byte[] L2 = "L2".getBytes(StandardCharsets.UTF_8); + byte[] L3 = "L3".getBytes(StandardCharsets.UTF_8); + byte[] H0_1 = "H0_1".getBytes(StandardCharsets.UTF_8); + byte[] H2_3 = "H2_3".getBytes(StandardCharsets.UTF_8); + byte[] H0_3 = "H0_3".getBytes(StandardCharsets.UTF_8); + + List> completeMerkleTree = new ArrayList<>(); + + // Level 0: Leaves + completeMerkleTree.add(Arrays.asList(L0, L1, L2, L3)); + // Level 1: Intermediate nodes + completeMerkleTree.add(Arrays.asList(H0_1, H2_3)); + // Level 2: Root + completeMerkleTree.add(Arrays.asList(H0_3)); + + MerkleProofCalculator calculator = new MerkleProofCalculator(); + + // For this test, we choose leaf index 2 (which corresponds to L2). + // Expected proof: + // - At Level 0: sibling of index 2 is at index 3 (L3). + // - At Level 1: then parent's index becomes 2/2 = 1; its sibling is at index 0 (H0_1). + // So the expected proof is [L3, H0_1]. + int leafIndex = 2; + List proof = calculator.calculateMerkleProof(completeMerkleTree, leafIndex); + + List expectedProof = new ArrayList<>(); + expectedProof.add(L3); + expectedProof.add(H0_1); + + // Check that the proof has the expected size. + assertEquals(expectedProof.size(), proof.size(), "Proof size does not match expected size."); + + // Verify that each element in the proof matches the expected value. + for (int i = 0; i < expectedProof.size(); i++) { + assertArrayEquals(expectedProof.get(i), proof.get(i), "Proof element at index " + i + " does not match."); + } + } + + @Test + public void testIndexIfExist() { + // Prepare some sample leaf hashes. + byte[] leaf1 = "leaf1".getBytes(StandardCharsets.UTF_8); + byte[] leaf2 = "leaf2".getBytes(StandardCharsets.UTF_8); + byte[] leaf3 = "leaf3".getBytes(StandardCharsets.UTF_8); + + List leafHashes = new ArrayList<>(); + leafHashes.add(leaf1); + leafHashes.add(leaf2); + leafHashes.add(leaf3); + + MerkleProofCalculator calculator = new MerkleProofCalculator(); + + // Test finding an existing element. + int foundIndex = calculator.indexIfExist(leafHashes, leaf2); + assertEquals(1, foundIndex, "The index of leaf2 should be 1."); + + // Test with an element that does not exist. + byte[] nonExistingLeaf = "nonExisting".getBytes(StandardCharsets.UTF_8); + int notFoundIndex = calculator.indexIfExist(leafHashes, nonExistingLeaf); + assertEquals(-1, notFoundIndex, "A non-existing leaf should return an index of -1."); + } +} From cd5705d33aa1f4f9c2a5f9e49d5738852a9d9d0e Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Sat, 15 Feb 2025 09:37:50 -0600 Subject: [PATCH 2/4] first working version Signed-off-by: Alfredo Gutierrez --- .../common/hasher/BlockMerkleTreeInfo.java | 12 +++ .../hasher/ConcurrentStreamingTreeHasher.java | 6 ++ .../common/hasher/MerkleProofCalculator.java | 89 +++++++++++++++++-- .../common/hasher/MerkleProofElement.java | 5 ++ .../hasher/NaiveStreamingTreeHasher.java | 10 ++- .../common/hasher/StreamingTreeHasher.java | 2 + .../hasher/MerkleProofCalculatorTest.java | 35 +++++--- .../verification/VerificationConfig.java | 2 +- .../verification/VerificationResult.java | 7 +- .../session/BlockVerificationSessionBase.java | 13 ++- .../BlockVerificationServiceImplTest.java | 5 -- .../BlockVerificationSessionBaseTest.java | 18 ++++ 12 files changed, 173 insertions(+), 31 deletions(-) create mode 100644 common/src/main/java/com/hedera/block/common/hasher/BlockMerkleTreeInfo.java create mode 100644 common/src/main/java/com/hedera/block/common/hasher/MerkleProofElement.java diff --git a/common/src/main/java/com/hedera/block/common/hasher/BlockMerkleTreeInfo.java b/common/src/main/java/com/hedera/block/common/hasher/BlockMerkleTreeInfo.java new file mode 100644 index 000000000..af05fbfce --- /dev/null +++ b/common/src/main/java/com/hedera/block/common/hasher/BlockMerkleTreeInfo.java @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.block.common.hasher; + +import com.hedera.pbj.runtime.io.buffer.Bytes; +import java.util.List; + +public record BlockMerkleTreeInfo( + List> inputsMerkleTree, + List> outputsMerkleTree, + Bytes previousBlockHash, + Bytes stateRootHash, + Bytes blockHash) {} diff --git a/common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java b/common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java index 09b4c69eb..73942bbca 100644 --- a/common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java +++ b/common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java @@ -103,6 +103,12 @@ public CompletableFuture rootHash() { return combiner.finalCombination(); } + @Override + public CompletableFuture>> merkleTree() { + List> merkleTree = new ArrayList<>(); + return CompletableFuture.completedFuture(merkleTree); + } + @Override public Status status() { if (numLeaves == 0) { diff --git a/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java b/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java index 513ed4ee9..7a05758df 100644 --- a/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java +++ b/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java @@ -1,25 +1,82 @@ package com.hedera.block.common.hasher; +import com.hedera.pbj.runtime.io.buffer.Bytes; + import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class MerkleProofCalculator { + public static int indexOfByteArray(List list, Bytes target) { + for (int i = 0; i < list.size(); i++) { + if (list.get(i).equals(target)) { + return i; + } + } + return -1; + } + + public List calculateBlockMerkleProof(BlockMerkleTreeInfo blockMerkleTreeInfo, Bytes leafHash) { + + // is the leaf in the input tree + // all leaf hashes are at the lowest level = 0 + int leafIndex = indexOfByteArray(blockMerkleTreeInfo.inputsMerkleTree().getFirst(), leafHash); + boolean isInputLeaf = leafIndex != -1; + // is the leaf in the output tree + if(leafIndex == -1) { + leafIndex = indexOfByteArray(blockMerkleTreeInfo.outputsMerkleTree().getFirst(), leafHash); + isInputLeaf = false; + } + + // if there is no match + if(leafIndex == -1) { + return null; + } + + // if there is a match + List proof = new ArrayList<>(); + // get proof elements up to the root of the subtree + if(isInputLeaf) { + proof.addAll(calculateMerkleProof(blockMerkleTreeInfo.inputsMerkleTree(), leafIndex)); + //proof.add(new MerkleProofElement(blockMerkleTreeInfo.outputsMerkleTree().getLast().getFirst(), false)); + } else { + proof.addAll(calculateMerkleProof(blockMerkleTreeInfo.outputsMerkleTree(), leafIndex)); + //proof.add(new MerkleProofElement(blockMerkleTreeInfo.inputsMerkleTree().getLast().getFirst(), false)); + } + + // get proof elements from the root of the subtree to the root of the blockRootHash + // the last levels of the tree are like this: + // leftSide: combine(previousBlockHash, inputsRootHash) + // rightSide: combine(outputsRootHash, stateRootHash) + if(isInputLeaf) { + proof.add(new MerkleProofElement(blockMerkleTreeInfo.previousBlockHash(), true)); + Bytes sibling = HashingUtilities.combine(blockMerkleTreeInfo.outputsMerkleTree().getLast().getFirst(), blockMerkleTreeInfo.stateRootHash()); + proof.add(new MerkleProofElement(sibling, false)); + } else { + proof.add(new MerkleProofElement(blockMerkleTreeInfo.stateRootHash(), false)); + Bytes sibling = HashingUtilities.combine(blockMerkleTreeInfo.previousBlockHash(), blockMerkleTreeInfo.inputsMerkleTree().getLast().getFirst()); + proof.add(new MerkleProofElement(sibling, true)); + } + + return new ArrayList<>(proof); + } + // Calculate the Merkle root hash of a list of leaf hashes. // Requires the completeMerkleTree to be a list of lists of byte arrays, where each list represents a level of the tree, fully padded. - public List calculateMerkleProof(List> completeMerkleTree, int leafIndex) { - List proof = new ArrayList<>(); + public List calculateMerkleProof(List> completeMerkleTree, int leafIndex) { + List proof = new ArrayList<>(); int index = leafIndex; // Iterate over each level except the root. for(int level = 0; level < completeMerkleTree.size() - 1; level++) { - List levelHashes = completeMerkleTree.get(level); + List levelHashes = completeMerkleTree.get(level); if(index % 2 == 0) { // If the index is even, the sibling is the next hash in the list. - proof.add(levelHashes.get(index + 1)); + proof.add(new MerkleProofElement(levelHashes.get(index + 1), false)); } else { // If the index is odd, the sibling is the previous hash in the list. - proof.add(levelHashes.get(index - 1)); + proof.add(new MerkleProofElement(levelHashes.get(index - 1), true)); } // Move up to the parent level. index /= 2; @@ -28,8 +85,26 @@ public List calculateMerkleProof(List> completeMerkleTree, return proof; } - public int indexIfExist(List leafHashes, byte[] leafHash) { - return leafHashes.indexOf(leafHash); + /** + * Verifies a Merkle proof for a given leaf hash. + * + * @param proof A list of MerkleProofElement that include sibling hashes and whether they're on the left. + * @param leafHash The hash (as a byte array) of the leaf node. + * @param rootHash The expected Merkle root hash. + * @return true if the proof is valid and the computed root matches rootHash; false otherwise. + */ + public boolean verifyMerkleProof(List proof, Bytes leafHash, Bytes rootHash) { + Bytes computedHash = leafHash; + for (MerkleProofElement element : proof) { + if (element.isLeft()) { + // Sibling is on the left: concatenate sibling hash + computed hash. + computedHash = HashingUtilities.combine(element.hash(), computedHash); + } else { + // Sibling is on the right: concatenate computed hash + sibling hash. + computedHash = HashingUtilities.combine(computedHash, element.hash()); + } + } + return computedHash.equals(rootHash); } } diff --git a/common/src/main/java/com/hedera/block/common/hasher/MerkleProofElement.java b/common/src/main/java/com/hedera/block/common/hasher/MerkleProofElement.java new file mode 100644 index 000000000..f655a420d --- /dev/null +++ b/common/src/main/java/com/hedera/block/common/hasher/MerkleProofElement.java @@ -0,0 +1,5 @@ +package com.hedera.block.common.hasher; + +import com.hedera.pbj.runtime.io.buffer.Bytes; + +public record MerkleProofElement(Bytes hash, boolean isLeft) { } diff --git a/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java b/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java index 296531ee5..cc13e13bb 100644 --- a/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java +++ b/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; /** * A naive implementation of {@link StreamingTreeHasher} that computes the root hash of a perfect binary Merkle tree of @@ -22,7 +23,7 @@ public class NaiveStreamingTreeHasher implements StreamingTreeHasher { private final List leafHashes = new ArrayList<>(); private boolean rootHashRequested = false; - private final List> completeMerkleTree = new LinkedList<>(); + private final List> completeMerkleTree = new LinkedList<>(); /** * Constructor for the {@link NaiveStreamingTreeHasher}. @@ -57,7 +58,7 @@ public CompletableFuture rootHash() { } } while (hashes.size() > 1) { - completeMerkleTree.add(new ArrayList<>(hashes)); + completeMerkleTree.add(new ArrayList<>(hashes.stream().map(Bytes::wrap).toList())); final Queue newLeafHashes = new LinkedList<>(); while (!hashes.isEmpty()) { final byte[] left = hashes.poll(); @@ -69,6 +70,11 @@ public CompletableFuture rootHash() { } hashes = newLeafHashes; } + completeMerkleTree.add(new ArrayList<>(hashes.stream().map(Bytes::wrap).toList())); return CompletableFuture.completedFuture(Bytes.wrap(requireNonNull(hashes.poll()))); } + + public CompletableFuture>> merkleTree() { + return CompletableFuture.completedFuture(completeMerkleTree); + } } diff --git a/common/src/main/java/com/hedera/block/common/hasher/StreamingTreeHasher.java b/common/src/main/java/com/hedera/block/common/hasher/StreamingTreeHasher.java index db573bbcc..4b0ac0bf1 100644 --- a/common/src/main/java/com/hedera/block/common/hasher/StreamingTreeHasher.java +++ b/common/src/main/java/com/hedera/block/common/hasher/StreamingTreeHasher.java @@ -52,6 +52,8 @@ public boolean isEmpty() { */ CompletableFuture rootHash(); + CompletableFuture>> merkleTree(); + /** * If supported, blocks until this hasher can give a deterministic summary of the status of the * tree hash computation. diff --git a/common/src/test/java/com/hedera/block/common/hasher/MerkleProofCalculatorTest.java b/common/src/test/java/com/hedera/block/common/hasher/MerkleProofCalculatorTest.java index 661f611fe..48713fded 100644 --- a/common/src/test/java/com/hedera/block/common/hasher/MerkleProofCalculatorTest.java +++ b/common/src/test/java/com/hedera/block/common/hasher/MerkleProofCalculatorTest.java @@ -7,6 +7,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; + +import com.hedera.pbj.runtime.io.buffer.Bytes; import org.junit.jupiter.api.Test; public class MerkleProofCalculatorTest { @@ -24,15 +26,15 @@ public void testCalculateMerkleProof() { // - H0_3 is the hash of H0_1 and H2_3. // For our test, we use literal byte arrays with the given names. - byte[] L0 = "L0".getBytes(StandardCharsets.UTF_8); - byte[] L1 = "L1".getBytes(StandardCharsets.UTF_8); - byte[] L2 = "L2".getBytes(StandardCharsets.UTF_8); - byte[] L3 = "L3".getBytes(StandardCharsets.UTF_8); - byte[] H0_1 = "H0_1".getBytes(StandardCharsets.UTF_8); - byte[] H2_3 = "H2_3".getBytes(StandardCharsets.UTF_8); - byte[] H0_3 = "H0_3".getBytes(StandardCharsets.UTF_8); + Bytes L0 = Bytes.wrap("L0".getBytes(StandardCharsets.UTF_8)); + Bytes L1 = Bytes.wrap("L1".getBytes(StandardCharsets.UTF_8)); + Bytes L2 = Bytes.wrap("L2".getBytes(StandardCharsets.UTF_8)); + Bytes L3 = Bytes.wrap("L3".getBytes(StandardCharsets.UTF_8)); + Bytes H0_1 = Bytes.wrap("H0_1".getBytes(StandardCharsets.UTF_8)); + Bytes H2_3 = Bytes.wrap("H2_3".getBytes(StandardCharsets.UTF_8)); + Bytes H0_3 = Bytes.wrap("H0_3".getBytes(StandardCharsets.UTF_8)); - List> completeMerkleTree = new ArrayList<>(); + List> completeMerkleTree = new ArrayList<>(); // Level 0: Leaves completeMerkleTree.add(Arrays.asList(L0, L1, L2, L3)); @@ -49,18 +51,25 @@ public void testCalculateMerkleProof() { // - At Level 1: then parent's index becomes 2/2 = 1; its sibling is at index 0 (H0_1). // So the expected proof is [L3, H0_1]. int leafIndex = 2; - List proof = calculator.calculateMerkleProof(completeMerkleTree, leafIndex); + List proof = calculator.calculateMerkleProof(completeMerkleTree, leafIndex); + + List expectedProof = new ArrayList<>(); + //expectedProof.add(L3); + expectedProof.add(new MerkleProofElement(L3, false)); - List expectedProof = new ArrayList<>(); - expectedProof.add(L3); - expectedProof.add(H0_1); + //expectedProof.add(H0_1); + expectedProof.add(new MerkleProofElement(H0_1, true)); // Check that the proof has the expected size. assertEquals(expectedProof.size(), proof.size(), "Proof size does not match expected size."); // Verify that each element in the proof matches the expected value. for (int i = 0; i < expectedProof.size(); i++) { - assertArrayEquals(expectedProof.get(i), proof.get(i), "Proof element at index " + i + " does not match."); + MerkleProofElement expectedElement = expectedProof.get(i); + MerkleProofElement actualElement = proof.get(i); + + assertArrayEquals(expectedElement.hash().toByteArray(), actualElement.hash().toByteArray(), "Hashes do not match."); + assertEquals(expectedElement.isLeft(), actualElement.isLeft(), "isLeft does not match."); } } diff --git a/server/src/main/java/com/hedera/block/server/verification/VerificationConfig.java b/server/src/main/java/com/hedera/block/server/verification/VerificationConfig.java index 188c3ded7..0f5ce51d7 100644 --- a/server/src/main/java/com/hedera/block/server/verification/VerificationConfig.java +++ b/server/src/main/java/com/hedera/block/server/verification/VerificationConfig.java @@ -17,7 +17,7 @@ @ConfigData("verification") public record VerificationConfig( @Loggable @ConfigProperty(defaultValue = "PRODUCTION") VerificationServiceType type, - @Loggable @ConfigProperty(defaultValue = "ASYNC") BlockVerificationSessionType sessionType, + @Loggable @ConfigProperty(defaultValue = "SYNC") BlockVerificationSessionType sessionType, @Loggable @ConfigProperty(defaultValue = "32") int hashCombineBatchSize) { /** diff --git a/server/src/main/java/com/hedera/block/server/verification/VerificationResult.java b/server/src/main/java/com/hedera/block/server/verification/VerificationResult.java index 8327c69aa..8129efa30 100644 --- a/server/src/main/java/com/hedera/block/server/verification/VerificationResult.java +++ b/server/src/main/java/com/hedera/block/server/verification/VerificationResult.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.block.server.verification; +import com.hedera.block.common.hasher.BlockMerkleTreeInfo; import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; @@ -11,4 +12,8 @@ * @param blockHash the block hash * @param status the verification status */ -public record VerificationResult(long blockNumber, @NonNull Bytes blockHash, @NonNull BlockVerificationStatus status) {} +public record VerificationResult( + long blockNumber, + @NonNull Bytes blockHash, + @NonNull BlockVerificationStatus status, + BlockMerkleTreeInfo blockMerkleTreeInfo) {} diff --git a/server/src/main/java/com/hedera/block/server/verification/session/BlockVerificationSessionBase.java b/server/src/main/java/com/hedera/block/server/verification/session/BlockVerificationSessionBase.java index 4d15117cf..019677b2f 100644 --- a/server/src/main/java/com/hedera/block/server/verification/session/BlockVerificationSessionBase.java +++ b/server/src/main/java/com/hedera/block/server/verification/session/BlockVerificationSessionBase.java @@ -3,6 +3,7 @@ import static java.lang.System.Logger.Level.INFO; +import com.hedera.block.common.hasher.BlockMerkleTreeInfo; import com.hedera.block.common.hasher.Hashes; import com.hedera.block.common.hasher.HashingUtilities; import com.hedera.block.common.hasher.StreamingTreeHasher; @@ -134,6 +135,12 @@ protected void finalizeVerification(BlockProof blockProof) { Bytes blockHash = HashingUtilities.computeFinalBlockHash(blockProof, inputTreeHasher, outputTreeHasher); VerificationResult result; boolean verified = signatureVerifier.verifySignature(blockHash, blockProof.blockSignature()); + BlockMerkleTreeInfo blockMerkleTreeInfo = new BlockMerkleTreeInfo( + inputTreeHasher.merkleTree().join(), + outputTreeHasher.merkleTree().join(), + blockProof.previousBlockRootHash(), + blockProof.startOfBlockStateRootHash(), + blockHash); if (verified) { long verificationLatency = System.nanoTime() - blockWorkStartTime; metricsService @@ -143,14 +150,16 @@ protected void finalizeVerification(BlockProof blockProof) { .get(BlockNodeMetricTypes.Counter.VerificationBlocksVerified) .increment(); - result = new VerificationResult(blockNumber, blockHash, BlockVerificationStatus.VERIFIED); + result = new VerificationResult( + blockNumber, blockHash, BlockVerificationStatus.VERIFIED, blockMerkleTreeInfo); } else { LOGGER.log(INFO, "Block verification failed for block number: {0}", blockNumber); metricsService .get(BlockNodeMetricTypes.Counter.VerificationBlocksFailed) .increment(); - result = new VerificationResult(blockNumber, blockHash, BlockVerificationStatus.INVALID_HASH_OR_SIGNATURE); + result = new VerificationResult( + blockNumber, blockHash, BlockVerificationStatus.INVALID_HASH_OR_SIGNATURE, blockMerkleTreeInfo); } shutdownSession(); verificationResultFuture.complete(result); diff --git a/server/src/test/java/com/hedera/block/server/verification/BlockVerificationServiceImplTest.java b/server/src/test/java/com/hedera/block/server/verification/BlockVerificationServiceImplTest.java index dc691deb9..3ff046dbc 100644 --- a/server/src/test/java/com/hedera/block/server/verification/BlockVerificationServiceImplTest.java +++ b/server/src/test/java/com/hedera/block/server/verification/BlockVerificationServiceImplTest.java @@ -93,11 +93,6 @@ void testOnBlockItemsReceivedNoBlockHeaderWithCurrentSession() throws ParseExcep verifyNoInteractions(verificationBlocksReceived, verificationBlocksFailed); } - private VerificationResult getVerificationResult(long blockNumber) { - return new VerificationResult( - blockNumber, Bytes.wrap(("hash" + blockNumber).getBytes()), BlockVerificationStatus.VERIFIED); - } - private BlockHeader getBlockHeader(long blockNumber) { long previousBlockNumber = blockNumber - 1; diff --git a/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java b/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java index 8e8521c48..bc30dbe57 100644 --- a/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java +++ b/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.block.server.verification.session; +import static com.hedera.block.common.hasher.HashingUtilities.getBlockItemHash; import static com.hedera.block.common.utils.FileUtilities.readGzipFileUnsafe; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.VerificationBlockTime; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.VerificationBlocksError; @@ -16,6 +17,9 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.hedera.block.common.hasher.HashingUtilities; +import com.hedera.block.common.hasher.MerkleProofCalculator; +import com.hedera.block.common.hasher.MerkleProofElement; import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.verification.BlockVerificationStatus; import com.hedera.block.server.verification.VerificationResult; @@ -32,6 +36,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -103,6 +109,18 @@ void testSuccessfulVerification() throws Exception { verify(verificationBlocksVerified, times(1)).increment(); verify(verificationBlockTime, times(1)).add(any(Long.class)); verifyNoMoreInteractions(verificationBlocksFailed); + + // lets get the first transaction result + BlockItemUnparsed blockItem = blockItems.stream().filter(item -> item.hasStateChanges()).collect(Collectors.toList()).get(0); + Bytes blockItemHash = Bytes.wrap(getBlockItemHash(blockItem).array()); + + // get proof for the item + MerkleProofCalculator calculator = new MerkleProofCalculator(); + List proof = calculator.calculateBlockMerkleProof(result.blockMerkleTreeInfo(), blockItemHash); + + // verify that the proof is valid + boolean isBlockItemVerified = calculator.verifyMerkleProof(proof, blockItemHash, result.blockHash()); + assertTrue(isBlockItemVerified, "Block item proof is not valid"); } @Test From 50fad114b035ee2cdc41cd7650a9d273715c463d Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Sat, 15 Feb 2025 09:38:14 -0600 Subject: [PATCH 3/4] stylefix Signed-off-by: Alfredo Gutierrez --- .../session/BlockVerificationSessionBaseTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java b/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java index bc30dbe57..82525ec66 100644 --- a/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java +++ b/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java @@ -17,7 +17,6 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import com.hedera.block.common.hasher.HashingUtilities; import com.hedera.block.common.hasher.MerkleProofCalculator; import com.hedera.block.common.hasher.MerkleProofElement; import com.hedera.block.server.metrics.MetricsService; @@ -37,7 +36,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -111,12 +109,16 @@ void testSuccessfulVerification() throws Exception { verifyNoMoreInteractions(verificationBlocksFailed); // lets get the first transaction result - BlockItemUnparsed blockItem = blockItems.stream().filter(item -> item.hasStateChanges()).collect(Collectors.toList()).get(0); + BlockItemUnparsed blockItem = blockItems.stream() + .filter(item -> item.hasStateChanges()) + .collect(Collectors.toList()) + .get(0); Bytes blockItemHash = Bytes.wrap(getBlockItemHash(blockItem).array()); // get proof for the item MerkleProofCalculator calculator = new MerkleProofCalculator(); - List proof = calculator.calculateBlockMerkleProof(result.blockMerkleTreeInfo(), blockItemHash); + List proof = + calculator.calculateBlockMerkleProof(result.blockMerkleTreeInfo(), blockItemHash); // verify that the proof is valid boolean isBlockItemVerified = calculator.verifyMerkleProof(proof, blockItemHash, result.blockHash()); From 903b94223e8913918c2a65c996d4adc8cce392ea Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Sun, 16 Feb 2025 22:09:33 -0600 Subject: [PATCH 4/4] general improvements as making MerkleProofCalculator an Utility class, improving handling of completable futures to avoid duplication of work or stales. Most importantly implementation of the MerkleTree completable future on the ConcurrentStreamingTreeHasher Signed-off-by: Alfredo Gutierrez --- .../hasher/ConcurrentStreamingTreeHasher.java | 39 ++++++++++++++++++- .../common/hasher/MerkleProofCalculator.java | 32 ++++++++++++--- .../hasher/NaiveStreamingTreeHasher.java | 6 ++- .../BlockVerificationSessionBaseTest.java | 5 +-- 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java b/common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java index 73942bbca..5f37a4fcb 100644 --- a/common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java +++ b/common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java @@ -10,6 +10,7 @@ import java.nio.ByteBuffer; import java.security.MessageDigest; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -56,6 +57,10 @@ public class ConcurrentStreamingTreeHasher implements StreamingTreeHasher { */ private boolean rootHashRequested = false; + private final CompletableFuture>> futureMerkleTree = new CompletableFuture<>(); + + private final List> merkleTree = new ArrayList<>(); + /** * Constructs a new {@link ConcurrentStreamingTreeHasher} with the given {@link ExecutorService}. * @@ -78,6 +83,7 @@ public ConcurrentStreamingTreeHasher( this.executorService = requireNonNull(executorService); this.hashCombineBatchSize = Preconditions.requireEven(hashCombineBatchSize, "Hash combine batch size must be an even number"); + merkleTree.add(new LinkedList<>()); ; } @@ -93,6 +99,7 @@ public void addLeaf(@NonNull final ByteBuffer hash) { numLeaves++; final byte[] bytes = new byte[HASH_LENGTH]; hash.get(bytes); + merkleTree.getFirst().add(Bytes.wrap(bytes)); // adding leafs as they come. combiner.combine(bytes); } @@ -103,10 +110,16 @@ public CompletableFuture rootHash() { return combiner.finalCombination(); } + /** + * Returns the complete Merkle tree as a list of levels (each level is a {@code List}). + * Level 0 contains the original leaves. + * The last level contains the root hash. + * + * @return a future that completes with the full Merkle tree + */ @Override public CompletableFuture>> merkleTree() { - List> merkleTree = new ArrayList<>(); - return CompletableFuture.completedFuture(merkleTree); + return rootHash().thenCompose(ignore -> futureMerkleTree); } @Override @@ -182,6 +195,7 @@ public void combine(@NonNull final byte[] hash) { public CompletableFuture finalCombination() { if (height == rootHeight) { final byte[] rootHash = pendingHashes.isEmpty() ? EMPTY_HASHES[0] : pendingHashes.getFirst(); + completeMerkleTree(); return CompletableFuture.completedFuture(Bytes.wrap(rootHash)); } else { if (!pendingHashes.isEmpty()) { @@ -191,6 +205,19 @@ public CompletableFuture finalCombination() { } } + private void completeMerkleTree() { + + // pad right-most hashes with empty hashes if needed to keep the tree balanced + for(int i = 0; i < merkleTree.size(); i++){ + int leaves = merkleTree.get(i).size(); + if(leaves % 2 != 0){ + merkleTree.get(i).add(Bytes.wrap(EMPTY_HASHES[i])); + } + } + + futureMerkleTree.complete(merkleTree); + } + public void flushAvailable(@NonNull final List rightmostHashes, final int stopHeight) { if (height < stopHeight) { final byte[] newPendingHash = pendingHashes.size() % 2 == 0 ? null : pendingHashes.removeLast(); @@ -218,12 +245,20 @@ private void schedulePendingWork() { pendingCombination = CompletableFuture.supplyAsync(() -> combine(hashes), executorService); } combination = combination.thenCombine(pendingCombination, (ignore, combined) -> { + appendLevelToMerkleTree(combined); combined.forEach(delegate::combine); return null; }); pendingHashes = new ArrayList<>(); } + private void appendLevelToMerkleTree(List combined){ + if(merkleTree.size() <= height + 1) { + merkleTree.add(new LinkedList<>()); + } + merkleTree.get(height + 1).addAll(combined.stream().map(Bytes::wrap).toList()); + } + private List combine(@NonNull final List hashes) { final List result = new ArrayList<>(); final MessageDigest digest = DIGESTS.get(); diff --git a/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java b/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java index 7a05758df..2ef7ee192 100644 --- a/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java +++ b/common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java @@ -3,11 +3,20 @@ import com.hedera.pbj.runtime.io.buffer.Bytes; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; public class MerkleProofCalculator { + private MerkleProofCalculator() { + throw new UnsupportedOperationException("Utility Class"); + } + + /** + * Find the index of a byte array in a list of byte arrays. + * @param list list of hashes as Bytes + * @param target leaf to find index within the list + * @return index of the target leaf in the list, or -1 if not found + */ public static int indexOfByteArray(List list, Bytes target) { for (int i = 0; i < list.size(); i++) { if (list.get(i).equals(target)) { @@ -17,7 +26,13 @@ public static int indexOfByteArray(List list, Bytes target) { return -1; } - public List calculateBlockMerkleProof(BlockMerkleTreeInfo blockMerkleTreeInfo, Bytes leafHash) { + /** + * Calculate the Merkle proof for a leaf hash in a block Merkle tree + * @param blockMerkleTreeInfo block Merkle tree information + * @param leafHash leaf hash to calculate proof for + * @return list of MerkleProofElement representing the proof, if item is not found within block, return null + */ + public static List calculateBlockMerkleProof(BlockMerkleTreeInfo blockMerkleTreeInfo, Bytes leafHash) { // is the leaf in the input tree // all leaf hashes are at the lowest level = 0 @@ -62,9 +77,14 @@ public List calculateBlockMerkleProof(BlockMerkleTreeInfo bl return new ArrayList<>(proof); } - // Calculate the Merkle root hash of a list of leaf hashes. - // Requires the completeMerkleTree to be a list of lists of byte arrays, where each list represents a level of the tree, fully padded. - public List calculateMerkleProof(List> completeMerkleTree, int leafIndex) { + /** + * Calculate the Merkle root hash of a list of leaf hashes. + * Requires the completeMerkleTree to be a list of lists of byte arrays, where each list represents a level of the tree, fully padded. + * + * @param completeMerkleTree A list of lists of byte arrays, where each list represents a level of the tree, fully padded. + * @return The Merkle root hash. + */ + public static List calculateMerkleProof(List> completeMerkleTree, int leafIndex) { List proof = new ArrayList<>(); int index = leafIndex; @@ -93,7 +113,7 @@ public List calculateMerkleProof(List> completeM * @param rootHash The expected Merkle root hash. * @return true if the proof is valid and the computed root matches rootHash; false otherwise. */ - public boolean verifyMerkleProof(List proof, Bytes leafHash, Bytes rootHash) { + public static boolean verifyMerkleProof(List proof, Bytes leafHash, Bytes rootHash) { Bytes computedHash = leafHash; for (MerkleProofElement element : proof) { if (element.isLeft()) { diff --git a/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java b/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java index cc13e13bb..ee8bb99dd 100644 --- a/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java +++ b/common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java @@ -23,7 +23,7 @@ public class NaiveStreamingTreeHasher implements StreamingTreeHasher { private final List leafHashes = new ArrayList<>(); private boolean rootHashRequested = false; - private final List> completeMerkleTree = new LinkedList<>(); + private final CompletableFuture>> futureMerkleTree = new CompletableFuture<>(); /** * Constructor for the {@link NaiveStreamingTreeHasher}. @@ -57,6 +57,7 @@ public CompletableFuture rootHash() { hashes.add(EMPTY_HASH); } } + final List> completeMerkleTree = new LinkedList<>(); while (hashes.size() > 1) { completeMerkleTree.add(new ArrayList<>(hashes.stream().map(Bytes::wrap).toList())); final Queue newLeafHashes = new LinkedList<>(); @@ -71,10 +72,11 @@ public CompletableFuture rootHash() { hashes = newLeafHashes; } completeMerkleTree.add(new ArrayList<>(hashes.stream().map(Bytes::wrap).toList())); + futureMerkleTree.complete(completeMerkleTree); return CompletableFuture.completedFuture(Bytes.wrap(requireNonNull(hashes.poll()))); } public CompletableFuture>> merkleTree() { - return CompletableFuture.completedFuture(completeMerkleTree); + return rootHash().thenCompose(ignore -> futureMerkleTree); } } diff --git a/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java b/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java index 82525ec66..617885ec6 100644 --- a/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java +++ b/server/src/test/java/com/hedera/block/server/verification/session/BlockVerificationSessionBaseTest.java @@ -116,12 +116,11 @@ void testSuccessfulVerification() throws Exception { Bytes blockItemHash = Bytes.wrap(getBlockItemHash(blockItem).array()); // get proof for the item - MerkleProofCalculator calculator = new MerkleProofCalculator(); List proof = - calculator.calculateBlockMerkleProof(result.blockMerkleTreeInfo(), blockItemHash); + MerkleProofCalculator.calculateBlockMerkleProof(result.blockMerkleTreeInfo(), blockItemHash); // verify that the proof is valid - boolean isBlockItemVerified = calculator.verifyMerkleProof(proof, blockItemHash, result.blockHash()); + boolean isBlockItemVerified = MerkleProofCalculator.verifyMerkleProof(proof, blockItemHash, result.blockHash()); assertTrue(isBlockItemVerified, "Block item proof is not valid"); }