createCommitment & blob to shares

foundry.toml

@@ -2,3 +2,4 @@
 solc_version = "0.8.22"
 via_ir = true
 gas_reports = ["*"]
+fs_permissions = [{ access = "read", path = "./"}]

src/lib/commitment/Commitment.sol

@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.8.22;
+
+import {Namespace, isReservedNamespace} from "../tree/Types.sol";
+import "../tree/namespace/NamespaceMerkleTree.sol";
+import "../tree/binary/BinaryMerkleTree.sol";
+import "../tree/binary/BinaryMerkleMultiproof.sol";
+import "../tree/namespace/NamespaceNode.sol";
+import "../tree/namespace/NamespaceMerkleMultiproof.sol";
+import {leafDigest} from "../tree/namespace/TreeHasher.sol";
+import {leafDigest as bLeafDigest} from "../tree/binary/TreeHasher.sol";
+import "../../../lib/openzeppelin-contracts-upgradeable/contracts/utils/math/MathUpgradeable.sol";
+import "forge-std/console.sol";
+
+uint256 constant SUBTREE_ROOT_THRESHOLD = 64;
+
+function _divCeil(uint256 x, uint256 y) pure returns (uint256 z) {
+    z = x / y + (x % y == 0 ? 0 : 1);
+}
+
+function _numShares(uint256 blobSize) pure returns (uint256) {
+    return _divCeil((MathUpgradeable.max(blobSize, 478) - 478), 482) + 1;
+}
+
+function _copyNamespace(bytes memory share, bytes29 namespaceBytes) pure {
+    for (uint256 i = 0; i < namespaceBytes.length; i++) {
+        share[i] = namespaceBytes[i];
+    }
+}
+
+function _writeInfoByteV0(bytes memory share, bool startingSequence) pure {
+    share[29] = bytes1(startingSequence ? 1 : 0);
+}
+
+function _writeSequenceLength(bytes memory share, uint32 sequenceLength) pure {
+    // Removed the "reverse", because it didn't work- maybe it's already big-endian?
+    //bytes4 sequenceLengthBigEndianBytes = bytes4(abi.encodePacked(reverse(sequenceLength)));
+    bytes4 sequenceLengthBigEndianBytes = bytes4(abi.encodePacked(sequenceLength));
+    share[30] = sequenceLengthBigEndianBytes[0];
+    share[31] = sequenceLengthBigEndianBytes[1];
+    share[32] = sequenceLengthBigEndianBytes[2];
+    share[33] = sequenceLengthBigEndianBytes[3];
+}
+
+function _copyBytes(bytes memory buffer, uint32 cursor, bytes memory data, uint32 length) pure returns (uint32) {
+
+    uint256 start = buffer.length - length;
+    for (uint256 i = start; i < buffer.length; i++) {
+        if (cursor < data.length) {
+            buffer[i] = data[cursor];
+            cursor++;
+        }
+        else {
+            buffer[i] = 0;
+        }
+    }
+    return cursor;
+}
+
+function _bytesToHexString(bytes memory buffer) pure returns (string memory) {
+
+    // Fixed buffer size for hexadecimal convertion
+    bytes memory converted = new bytes(buffer.length * 2);
+
+    bytes memory _base = "0123456789abcdef";
+
+    for (uint256 i = 0; i < buffer.length; i++) {
+        converted[i * 2] = _base[uint8(buffer[i]) / _base.length];
+        converted[i * 2 + 1] = _base[uint8(buffer[i]) % _base.length];
+    }
+
+    return string(abi.encodePacked(converted));
+}
+
+// Share Version 0
+function _bytesToSharesV0(bytes memory blobData, Namespace memory namespace) pure returns (bytes[] memory shares, bool err) {
+    if (namespace.version != 0) {
+        return (new bytes[](0), true);
+    }
+    if (isReservedNamespace(namespace)) {
+        return (new bytes[](0), true);
+    }
+    // Allocate memory for the shares
+    uint256 numShares = _numShares(blobData.length); 
+    bytes[] memory _shares = new bytes[](numShares);
+    for (uint256 i = 0; i < _shares.length; i++) {
+        _shares[i] = new bytes(512);
+    }
+    // Get the namespace bytes
+    bytes29 namespaceBytes = namespace.toBytes();
+
+    // The first share is special, because it has startingSequence set to true and the 4-byte sequence length
+    _copyNamespace(_shares[0], namespaceBytes);
+    _writeInfoByteV0(_shares[0], true);
+    _writeSequenceLength(_shares[0], uint32(blobData.length));
+    uint32 cursor = 0;
+    cursor = _copyBytes(_shares[0], cursor, blobData, uint32(478)); //only 478 bytes, because 4 bytes are used for the sequence length
+
+    if (shares.length != 1) {
+        // The remaining shares are all the same
+        for (uint256 i = 1; i < _shares.length; i++) {
+            // Copy the namespace
+            _copyNamespace(_shares[i], namespaceBytes);
+            // Write the info byte
+            _writeInfoByteV0(_shares[i], false);
+            // Copy the data
+            cursor = _copyBytes(_shares[i], cursor, blobData, uint32(482)); // copy the full 482 bytes
+        }
+    }
+
+    shares = _shares;
+    err = false;
+}
+
+function _roundDownPowerOfTwo(uint256 x) pure returns (uint256) {
+    uint256 result = 1;
+    while (result * 2 <= x) {
+        result *= 2;
+    }
+    return result;
+}
+
+function _roundUpPowerOfTwo(uint256 x) pure returns (uint256) {
+    uint256 result = 1;
+    while (result < x) {
+        result *= 2;
+    }
+    return result;
+}
+
+function _blobMinSquareSize(uint256 shareCount) pure returns (uint256) {
+    return _roundUpPowerOfTwo(MathUpgradeable.sqrt(shareCount, MathUpgradeable.Rounding.Ceil));
+}
+
+function _subtreeWidth(uint256 shareCount, uint256 subtreeRootThreshold) pure returns (uint256) {
+    uint256 s = shareCount / subtreeRootThreshold;
+    if (s != 0) {
+        s++;
+    }
+    s = _roundUpPowerOfTwo(s);
+    return MathUpgradeable.min(s, _blobMinSquareSize(shareCount));
+}
+
+function _merkleMountainRangeSizes(uint256 totalSize, uint256 maxTreeSize) pure returns (uint256[] memory) {
+    // Overestimate size of array
+    // This is a workaround because Solidity doesn't support dynamic memory arrays like Go or Rust
+    uint256 bigTrees = totalSize / maxTreeSize;
+    uint256 leftovers = totalSize % maxTreeSize;
+    uint256 numTrees;
+    if (leftovers == 0) {
+        numTrees = bigTrees;
+    }
+    else {
+        numTrees = bigTrees + MathUpgradeable.log2(leftovers) + (leftovers%2);
+    }
+    uint256[] memory treeSizes = new uint256[](numTrees);
+    uint256 count = 0;
+    while (totalSize != 0) {
+        if (totalSize >= maxTreeSize) {
+            treeSizes[count] = maxTreeSize;
+            totalSize -= maxTreeSize;
+        }
+        else {
+            uint256 treeSize = _roundDownPowerOfTwo(totalSize);
+            treeSizes[count] = treeSize;
+            totalSize -= treeSize;
+        }
+        count++;
+    }
+    return treeSizes;
+}
+
+function _createCommitment(bytes[] memory shares, Namespace memory namespace) returns (bytes32 commitment) {
+    uint256 subtreeWidth = _subtreeWidth(shares.length, SUBTREE_ROOT_THRESHOLD);
+    uint256[] memory treeSizes = _merkleMountainRangeSizes(shares.length, subtreeWidth);
+    bytes[][] memory leafSets = new bytes[][](treeSizes.length);
+    uint256 cursor = 0;
+    // So much copying...
+    // This could likely be optimized, but I'm not an EVM expert
+    // Let's see if the gas is too high, optimize later.
+    // Stop when we hit 0, the delimeter indicating the end of the array
+    for (uint256 i = 0; i < treeSizes.length; i++) {
+        leafSets[i] = new bytes[](treeSizes[i]);
+        for (uint256 j = 0; j < treeSizes[i]; j++) {
+            //leafSets[i][j] = new bytes(512);
+            // Try with 512 + 29, prefixing with the 29 byte namespace
+            leafSets[i][j] = new bytes(512);
+            // copy the share
+            for (uint256 k = 0; k < 512; k++) {
+                leafSets[i][j][k] = shares[cursor][k];
+            }
+            cursor += treeSizes[i];
+        }
+    }
+
+    //NamespaceNode[] memory subtreeRoots = new NamespaceNode[](leafSets.length);
+    bytes32[] memory subtreeRoots = new bytes32[](leafSets.length);
+    // Fore each leafSet, compute the root using _computeRoot. Pass a null value for the "proof" parameter
+    //NamespaceMerkleMultiproof memory nullproof = NamespaceMerkleMultiproof(0, 0, new NamespaceNode[](0));
+    for (uint256 i = 0; i < leafSets.length; i++) {
+        NamespaceNode[] memory leafNamespaceNodes = new NamespaceNode[](leafSets[i].length);
+        for (uint256 j = 0; j < leafSets[i].length; j++) {
+            leafNamespaceNodes[j] = leafDigest(namespace, leafSets[i][j]);
+        }
+        //NamespaceMerkleMultiproof memory emptyProof = NamespaceMerkleMultiproof(0, leafSets[i].length, new NamespaceNode[](0));
+        NamespaceMerkleMultiproof memory emptyProof = NamespaceMerkleMultiproof(0, leafSets[i].length, leafNamespaceNodes);
+        (NamespaceNode memory root,,,) = NamespaceMerkleTree._computeRoot(emptyProof, leafNamespaceNodes, 0, leafNamespaceNodes.length, 0, 0);
+        subtreeRoots[i] = bLeafDigest(bytes(abi.encodePacked(root.min.toBytes(), root.max.toBytes(), root.digest)));
+    }
+    //BinaryMerkleMultiproof memory nullBinaryProof = BinaryMerkleMultiproof(new bytes32[](0), 0, 0);
+    BinaryMerkleMultiproof memory emptyBinaryProof = BinaryMerkleMultiproof(new bytes32[](0), 0, subtreeRoots.length);
+    (bytes32 binaryTreeRoot,,,) = BinaryMerkleTree._computeRootMulti(emptyBinaryProof, subtreeRoots, 0, subtreeRoots.length, 0, 0);
+    commitment = binaryTreeRoot;
+}

src/lib/commitment/test/Commitment.t.sol

@@ -0,0 +1,109 @@
+pragma solidity ^0.8.22;
+
+import "ds-test/test.sol";
+import "forge-std/Vm.sol";
+import "forge-std/console.sol";
+import {_bytesToSharesV0, _createCommitment, _bytesToHexString} from "../Commitment.sol";
+import {toNamespace, Namespace} from "../../tree/Types.sol";
+import "../../tree/namespace/NamespaceMerkleTree.sol";
+import "../../tree/namespace/NamespaceMerkleMultiproof.sol";
+
+contract CommitmentTest is DSTest {
+    Vm private constant vm = Vm(address(uint160(uint256(keccak256("hevm cheat code")))));
+
+    struct TestVector {
+        string commitment;
+        string data;
+        string namespace;
+        string shares;
+    }
+
+    function fromHexChar(uint8 c) public pure returns (uint8) {
+        if (bytes1(c) >= bytes1('0') && bytes1(c) <= bytes1('9')) {
+            return c - uint8(bytes1('0'));
+        }
+        if (bytes1(c) >= bytes1('a') && bytes1(c) <= bytes1('f')) {
+            return 10 + c - uint8(bytes1('a'));
+        }
+        if (bytes1(c) >= bytes1('A') && bytes1(c) <= bytes1('F')) {
+            return 10 + c - uint8(bytes1('A'));
+        }
+        revert("fail");
+    }
+
+    function fromHex(string memory s) public pure returns (bytes memory) {
+        bytes memory ss = bytes(s);
+        require(ss.length%2 == 0); // length must be even
+        bytes memory r = new bytes(ss.length/2);
+        for (uint i=0; i<ss.length/2; ++i) {
+            r[i] = bytes1(fromHexChar(uint8(ss[2*i])) * 16 + fromHexChar(uint8(ss[2*i+1])));
+        }
+        return r;
+    }
+
+    function compareStrings(string memory a, string memory b) public pure returns (bool) {
+        return (keccak256(abi.encodePacked((a))) == keccak256(abi.encodePacked((b))));
+    }
+
+    function testBytesToSharesV0() view external {
+
+        // test vectors were generated here: https://github.com/S1nus/share-test-vec-gen
+        string memory path = "./src/lib/commitment/test/testVectors.json";
+        string memory jsonData = vm.readFile(path);
+        bytes memory vecsData = vm.parseJson(jsonData);
+        TestVector[] memory vecs = abi.decode(vecsData, (TestVector[]));
+
+        for (uint i = 0; i < vecs.length; i++) {
+            bytes29 nsString = bytes29(fromHex(vecs[i].namespace));
+            Namespace memory ns = toNamespace(nsString);
+            bytes memory data = fromHex(vecs[i].data);
+            (bytes[] memory shares, bool err) = _bytesToSharesV0(data, ns);
+            string memory out = "";
+            for (uint j = 0; j < shares.length; j++) {
+                out = string.concat(out, _bytesToHexString(shares[j]));
+            }
+            // none of the test vectors should cause an error
+            //assert(!err);
+            //assert(compareStrings(out, vecs[i].shares));
+            if (!compareStrings(out, vecs[i].shares)) {
+                console.log("expected: ", vecs[i].shares);
+                console.log("got: ", out);
+            }
+        }
+    }
+
+    function testCreateCommitmentV0() external {
+        string memory path = "./src/lib/commitment/test/testVectors2.json";
+        string memory jsonData = vm.readFile(path);
+        bytes memory vecsData = vm.parseJson(jsonData);
+        TestVector[] memory vecs = abi.decode(vecsData, (TestVector[]));
+
+
+        //for (uint i = 0; i < vecs.length; i++) {
+            bytes29 nsString = bytes29(fromHex(vecs[0].namespace));
+            Namespace memory ns = toNamespace(nsString);
+            bytes memory data = fromHex(vecs[0].data);
+            (bytes[] memory shares, bool err) = _bytesToSharesV0(data, ns);
+            bytes32 commitment = _createCommitment(shares, ns);
+            if (!compareStrings(_bytesToHexString(abi.encodePacked(commitment)), vecs[0].commitment)) {
+                console.log("Commitment mismatch for vector ", 0);
+                console.log("expected: ", vecs[0].commitment);
+                console.log("got: ", _bytesToHexString(abi.encodePacked(commitment)));
+                return;
+            }
+        //}
+        console.log("All good :)");
+
+        /*bytes32 dummy = hex"000000000000000000000000000000000000005cfe5e6a0c8e6402fd5e010000";
+        NamespaceNode memory node = NamespaceNode(ns, ns, dummy);
+        NamespaceNode[] memory nodes = new NamespaceNode[](1);
+        nodes[0] = node;
+        NamespaceMerkleMultiproof memory nullproof = NamespaceMerkleMultiproof(0, 0, new NamespaceNode[](0));
+        NamespaceMerkleMultiproof memory populatedProof = NamespaceMerkleMultiproof(0, 1, nodes);
+        (NamespaceNode memory root,,,) = NamespaceMerkleTree._computeRoot(populatedProof, nodes, 0, 1, 0, 0);
+        console.log(_bytesToHexString(abi.encodePacked(node.digest)));
+        console.log(_bytesToHexString(abi.encodePacked(root.digest)));*/
+        //for (uint i = 0; i < vecs.length; i++) {
+        //}
+    }
+}

src/lib/tree/Types.sol

@@ -28,6 +28,12 @@ function toBytes(Namespace memory n) pure returns (bytes29) {
     return bytes29(abi.encodePacked(n.version, n.id));
 }
 
+function isReservedNamespace(Namespace memory n) pure returns (bool) {
+    bytes29 upper = hex"00000000000000000000000000000000000000000000000000000000FF";
+    bytes29 lower = hex"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00";
+    return n.toBytes() < upper || n.toBytes() > lower;
+}
+
 function toNamespace(bytes29 n) pure returns (Namespace memory) {
     bytes memory id = new bytes(28);
     for (uint256 i = 1; i < 29; i++) {

src/lib/tree/binary/BinaryMerkleTree.sol

@@ -126,7 +126,7 @@ library BinaryMerkleTree {
         uint256 end,
         uint256 headProof,
         uint256 headLeaves
-    ) private pure returns (bytes32, uint256, uint256, bool) {
+    ) public pure returns (bytes32, uint256, uint256, bool) {
         // reached a leaf
         if (end - begin == 1) {
             // if current range overlaps with proof range, pop and return a leaf

src/lib/tree/binary/TreeHasher.sol

@@ -20,4 +20,4 @@ function nodeDigest(bytes32 left, bytes32 right) pure returns (bytes32 digest) {
 // solhint-disable-next-line func-visibility
 function leafDigest(bytes memory data) pure returns (bytes32 digest) {
     digest = sha256(abi.encodePacked(Constants.LEAF_PREFIX, data));
-}
+}