POC: Block item proof POC

common/src/main/java/com/hedera/block/common/hasher/BlockMerkleTreeInfo.java

@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: Apache-2.0
+package com.hedera.block.common.hasher;
+
+import com.hedera.pbj.runtime.io.buffer.Bytes;
+import java.util.List;
+
+public record BlockMerkleTreeInfo(
+        List<List<Bytes>> inputsMerkleTree,
+        List<List<Bytes>> outputsMerkleTree,
+        Bytes previousBlockHash,
+        Bytes stateRootHash,
+        Bytes blockHash) {}

common/src/main/java/com/hedera/block/common/hasher/ConcurrentStreamingTreeHasher.java

@@ -10,6 +10,7 @@
 import java.nio.ByteBuffer;
 import java.security.MessageDigest;
 import java.util.ArrayList;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
@@ -56,6 +57,10 @@ public class ConcurrentStreamingTreeHasher implements StreamingTreeHasher {
      */
     private boolean rootHashRequested = false;
 
+    private final CompletableFuture<List<List<Bytes>>> futureMerkleTree = new CompletableFuture<>();
+
+    private final List<List<Bytes>> merkleTree = new ArrayList<>();
+
     /**
      * Constructs a new {@link ConcurrentStreamingTreeHasher} with the given {@link ExecutorService}.
      *
@@ -78,6 +83,7 @@ public ConcurrentStreamingTreeHasher(
         this.executorService = requireNonNull(executorService);
         this.hashCombineBatchSize =
                 Preconditions.requireEven(hashCombineBatchSize, "Hash combine batch size must be an even number");
+        merkleTree.add(new LinkedList<>());
         ;
     }
 
@@ -93,6 +99,7 @@ public void addLeaf(@NonNull final ByteBuffer hash) {
         numLeaves++;
         final byte[] bytes = new byte[HASH_LENGTH];
         hash.get(bytes);
+        merkleTree.getFirst().add(Bytes.wrap(bytes)); // adding leafs as they come.
         combiner.combine(bytes);
     }
 
@@ -103,6 +110,18 @@ public CompletableFuture<Bytes> rootHash() {
         return combiner.finalCombination();
     }
 
+    /**
+     * Returns the complete Merkle tree as a list of levels (each level is a {@code List<Bytes>}).
+     * Level 0 contains the original leaves.
+     * The last level contains the root hash.
+     *
+     * @return a future that completes with the full Merkle tree
+     */
+    @Override
+    public CompletableFuture<List<List<Bytes>>> merkleTree() {
+        return rootHash().thenCompose(ignore -> futureMerkleTree);
+    }
+
     @Override
     public Status status() {
         if (numLeaves == 0) {
@@ -176,6 +195,7 @@ public void combine(@NonNull final byte[] hash) {
         public CompletableFuture<Bytes> finalCombination() {
             if (height == rootHeight) {
                 final byte[] rootHash = pendingHashes.isEmpty() ? EMPTY_HASHES[0] : pendingHashes.getFirst();
+                completeMerkleTree();
                 return CompletableFuture.completedFuture(Bytes.wrap(rootHash));
             } else {
                 if (!pendingHashes.isEmpty()) {
@@ -185,6 +205,19 @@ public CompletableFuture<Bytes> finalCombination() {
             }
         }
 
+        private void completeMerkleTree() {
+
+            // pad right-most hashes with empty hashes if needed to keep the tree balanced
+            for(int i = 0; i < merkleTree.size(); i++){
+                int leaves = merkleTree.get(i).size();
+                if(leaves % 2 != 0){
+                    merkleTree.get(i).add(Bytes.wrap(EMPTY_HASHES[i]));
+                }
+            }
+
+            futureMerkleTree.complete(merkleTree);
+        }
+
         public void flushAvailable(@NonNull final List<Bytes> rightmostHashes, final int stopHeight) {
             if (height < stopHeight) {
                 final byte[] newPendingHash = pendingHashes.size() % 2 == 0 ? null : pendingHashes.removeLast();
@@ -212,12 +245,20 @@ private void schedulePendingWork() {
                 pendingCombination = CompletableFuture.supplyAsync(() -> combine(hashes), executorService);
             }
             combination = combination.thenCombine(pendingCombination, (ignore, combined) -> {
+                appendLevelToMerkleTree(combined);
                 combined.forEach(delegate::combine);
                 return null;
             });
             pendingHashes = new ArrayList<>();
         }
 
+        private void appendLevelToMerkleTree(List<byte[]> combined){
+            if(merkleTree.size() <= height + 1) {
+                merkleTree.add(new LinkedList<>());
+            }
+            merkleTree.get(height + 1).addAll(combined.stream().map(Bytes::wrap).toList());
+        }
+
         private List<byte[]> combine(@NonNull final List<byte[]> hashes) {
             final List<byte[]> result = new ArrayList<>();
             final MessageDigest digest = DIGESTS.get();

common/src/main/java/com/hedera/block/common/hasher/MerkleProofCalculator.java

@@ -0,0 +1,130 @@
+package com.hedera.block.common.hasher;
+
+import com.hedera.pbj.runtime.io.buffer.Bytes;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MerkleProofCalculator {
+
+    private MerkleProofCalculator() {
+        throw new UnsupportedOperationException("Utility Class");
+    }
+
+    /**
+     * Find the index of a byte array in a list of byte arrays.
+     * @param list list of hashes as Bytes
+     * @param target leaf to find index within the list
+     * @return index of the target leaf in the list, or -1 if not found
+     */
+    public static int indexOfByteArray(List<Bytes> list, Bytes target) {
+        for (int i = 0; i < list.size(); i++) {
+            if (list.get(i).equals(target)) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    /**
+     * Calculate the Merkle proof for a leaf hash in a block Merkle tree
+     * @param blockMerkleTreeInfo block Merkle tree information
+     * @param leafHash leaf hash to calculate proof for
+     * @return list of MerkleProofElement representing the proof, if item is not found within block, return null
+     */
+    public static List<MerkleProofElement> calculateBlockMerkleProof(BlockMerkleTreeInfo blockMerkleTreeInfo, Bytes leafHash) {
+
+        // is the leaf in the input tree
+        // all leaf hashes are at the lowest level = 0
+        int leafIndex = indexOfByteArray(blockMerkleTreeInfo.inputsMerkleTree().getFirst(), leafHash);
+        boolean isInputLeaf = leafIndex != -1;
+        // is the leaf in the output tree
+        if(leafIndex == -1) {
+            leafIndex = indexOfByteArray(blockMerkleTreeInfo.outputsMerkleTree().getFirst(), leafHash);
+            isInputLeaf = false;
+        }
+
+        // if there is no match
+        if(leafIndex == -1) {
+            return null;
+        }
+
+        // if there is a match
+        List<MerkleProofElement> proof = new ArrayList<>();
+        // get proof elements up to the root of the subtree
+        if(isInputLeaf) {
+            proof.addAll(calculateMerkleProof(blockMerkleTreeInfo.inputsMerkleTree(), leafIndex));
+            //proof.add(new MerkleProofElement(blockMerkleTreeInfo.outputsMerkleTree().getLast().getFirst(), false));
+        } else {
+            proof.addAll(calculateMerkleProof(blockMerkleTreeInfo.outputsMerkleTree(), leafIndex));
+            //proof.add(new MerkleProofElement(blockMerkleTreeInfo.inputsMerkleTree().getLast().getFirst(), false));
+        }
+
+        // get proof elements from the root of the subtree to the root of the blockRootHash
+        // the last levels of the tree are like this:
+        // leftSide: combine(previousBlockHash, inputsRootHash)
+        // rightSide: combine(outputsRootHash, stateRootHash)
+        if(isInputLeaf) {
+            proof.add(new MerkleProofElement(blockMerkleTreeInfo.previousBlockHash(), true));
+            Bytes sibling = HashingUtilities.combine(blockMerkleTreeInfo.outputsMerkleTree().getLast().getFirst(), blockMerkleTreeInfo.stateRootHash());
+            proof.add(new MerkleProofElement(sibling, false));
+        } else {
+           proof.add(new MerkleProofElement(blockMerkleTreeInfo.stateRootHash(), false));
+            Bytes sibling = HashingUtilities.combine(blockMerkleTreeInfo.previousBlockHash(), blockMerkleTreeInfo.inputsMerkleTree().getLast().getFirst());
+            proof.add(new MerkleProofElement(sibling, true));
+        }
+
+        return new ArrayList<>(proof);
+    }
+
+    /**
+     * Calculate the Merkle root hash of a list of leaf hashes.
+     * Requires the completeMerkleTree to be a list of lists of byte arrays, where each list represents a level of the tree, fully padded.
+     *
+     * @param completeMerkleTree A list of lists of byte arrays, where each list represents a level of the tree, fully padded.
+     * @return The Merkle root hash.
+     */
+    public static List<MerkleProofElement> calculateMerkleProof(List<List<Bytes>> completeMerkleTree, int leafIndex) {
+        List<MerkleProofElement> proof = new ArrayList<>();
+        int index = leafIndex;
+
+        // Iterate over each level except the root.
+        for(int level = 0; level < completeMerkleTree.size() - 1; level++) {
+            List<Bytes> levelHashes = completeMerkleTree.get(level);
+            if(index % 2 == 0) {
+                // If the index is even, the sibling is the next hash in the list.
+                proof.add(new MerkleProofElement(levelHashes.get(index + 1), false));
+            } else {
+                // If the index is odd, the sibling is the previous hash in the list.
+                proof.add(new MerkleProofElement(levelHashes.get(index - 1), true));
+            }
+            // Move up to the parent level.
+            index /= 2;
+        }
+
+        return proof;
+    }
+
+    /**
+     * Verifies a Merkle proof for a given leaf hash.
+     *
+     * @param proof    A list of MerkleProofElement that include sibling hashes and whether they're on the left.
+     * @param leafHash The hash (as a byte array) of the leaf node.
+     * @param rootHash The expected Merkle root hash.
+     * @return true if the proof is valid and the computed root matches rootHash; false otherwise.
+     */
+    public static boolean verifyMerkleProof(List<MerkleProofElement> proof, Bytes leafHash, Bytes rootHash) {
+        Bytes computedHash = leafHash;
+        for (MerkleProofElement element : proof) {
+            if (element.isLeft()) {
+                // Sibling is on the left: concatenate sibling hash + computed hash.
+                computedHash = HashingUtilities.combine(element.hash(), computedHash);
+            } else {
+                // Sibling is on the right: concatenate computed hash + sibling hash.
+                computedHash = HashingUtilities.combine(computedHash, element.hash());
+            }
+        }
+        return computedHash.equals(rootHash);
+    }
+
+}

common/src/main/java/com/hedera/block/common/hasher/MerkleProofElement.java

@@ -0,0 +1,5 @@
+package com.hedera.block.common.hasher;
+
+import com.hedera.pbj.runtime.io.buffer.Bytes;
+
+public record MerkleProofElement(Bytes hash, boolean isLeft) { }

common/src/main/java/com/hedera/block/common/hasher/NaiveStreamingTreeHasher.java

@@ -12,6 +12,7 @@
 import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 
 /**
  * A naive implementation of {@link StreamingTreeHasher} that computes the root hash of a perfect binary Merkle tree of
@@ -22,6 +23,7 @@ public class NaiveStreamingTreeHasher implements StreamingTreeHasher {
 
     private final List<byte[]> leafHashes = new ArrayList<>();
     private boolean rootHashRequested = false;
+    private final CompletableFuture<List<List<Bytes>>> futureMerkleTree = new CompletableFuture<>();
 
     /**
      * Constructor for the {@link NaiveStreamingTreeHasher}.
@@ -55,7 +57,9 @@ public CompletableFuture<Bytes> rootHash() {
                 hashes.add(EMPTY_HASH);
             }
         }
+        final List<List<Bytes>> completeMerkleTree = new LinkedList<>();
         while (hashes.size() > 1) {
+            completeMerkleTree.add(new ArrayList<>(hashes.stream().map(Bytes::wrap).toList()));
             final Queue<byte[]> newLeafHashes = new LinkedList<>();
             while (!hashes.isEmpty()) {
                 final byte[] left = hashes.poll();
@@ -67,6 +71,12 @@ public CompletableFuture<Bytes> rootHash() {
             }
             hashes = newLeafHashes;
         }
+        completeMerkleTree.add(new ArrayList<>(hashes.stream().map(Bytes::wrap).toList()));
+        futureMerkleTree.complete(completeMerkleTree);
         return CompletableFuture.completedFuture(Bytes.wrap(requireNonNull(hashes.poll())));
     }
+
+    public CompletableFuture<List<List<Bytes>>> merkleTree() {
+        return rootHash().thenCompose(ignore -> futureMerkleTree);
+    }
 }

common/src/main/java/com/hedera/block/common/hasher/StreamingTreeHasher.java

@@ -52,6 +52,8 @@ public boolean isEmpty() {
      */
     CompletableFuture<Bytes> rootHash();
 
+    CompletableFuture<List<List<Bytes>>> merkleTree();
+
     /**
      * If supported, blocks until this hasher can give a deterministic summary of the status of the
      * tree hash computation.