NomosDA spec v1 updates (#117)

* Modify and rename hash_commitment method. Now we just hash the column commitment itself.

* Fix calls

* Remove certificate from verifier

* Update verifier

* Fix tests

* Fix verifier imports

* Fix more imports

* Fix dispersal

* Fix more imports

* Fix missing parameter in dispersal

* Fix tests

* Full flow renaming

* Disperse encoded data in full flow test

* Make da verification indempotent (#118)

---------

Co-authored-by: Gusto <[email protected]>
Co-authored-by: gusto <[email protected]>

da/api/common.py

@@ -2,7 +2,7 @@
 from dataclasses import dataclass
 from typing import Optional, List, Sequence
 
-from da.common import Certificate
+from da.common import BlobId
 from da.verifier import DABlob
 
 
@@ -15,16 +15,16 @@ class Metadata:
 
 
 @dataclass
-class VID:
-    # da certificate id
-    cert_id: bytes
+class BlobMetadata:
+    # da blob_id id
+    blob_id: BlobId
     # application + index information
     metadata: Metadata
 
 
 class BlobStore(ABC):
     @abstractmethod
-    def add(self, certificate: Certificate, metadata: Metadata):
+    def add(self, id: BlobId, metadata: Metadata):
         """
         Raises: ValueError if there is already a registered certificate fot the given metadata
         """
@@ -39,14 +39,14 @@ class DAApi:
     def __init__(self, bs: BlobStore):
         self.store = bs
 
-    def write(self, certificate: Certificate, metadata: Metadata):
+    def write(self, id: BlobId, metadata: Metadata):
         """
         Write method should be used by a service that is able to retrieve verified certificates
         from the latest Block. Once a certificate is retrieved, api creates a relation between
         the blob of an original data, certificate and index for the app_id of the certificate.
         Raises: ValueError if there is already a registered certificate for a given metadata
         """
-        self.store.add(certificate, metadata)
+        self.store.add(id, metadata)
 
     def read(self, app_id, indexes) -> List[Optional[DABlob]]:
         """

da/api/test_flow.py

@@ -14,8 +14,8 @@ def __init__(self):
         self.blob_store = {}
         self.app_id_store = defaultdict(dict)
 
-    def populate(self, blob, cert_id: bytes):
-        self.blob_store[cert_id] = blob
+    def populate(self, blob, blob_id: BlobId):
+        self.blob_store[blob_id] = blob
 
     # Implements `add` method from BlobStore abstract class.
     def add(self, cert_id: bytes, metadata: Metadata):
@@ -35,58 +35,58 @@ def get_multiple(self, app_id, indexes) -> List[Optional[DABlob]]:
 class TestFlow(TestCase):
     def test_api_write_read(self):
         expected_blob = "hello"
-        cert_id = b"11"*32
+        blob_id = b"11"*32
         app_id = 1
         idx = 1
         mock_meta = Metadata(1, 1)
 
         mock_store = MockStore()
-        mock_store.populate(expected_blob, cert_id)
+        mock_store.populate(expected_blob, blob_id)
 
         api = DAApi(mock_store)
 
-        api.write(cert_id, mock_meta)
+        api.write(blob_id, mock_meta)
         blobs = api.read(app_id, [idx])
 
         self.assertEqual([expected_blob], blobs)
 
     def test_same_index(self):
         expected_blob = "hello"
-        cert_id = b"11"*32
+        blob_id = b"11"*32
         app_id = 1
         idx = 1
         mock_meta = Metadata(1, 1)
 
         mock_store = MockStore()
-        mock_store.populate(expected_blob, cert_id)
+        mock_store.populate(expected_blob, blob_id)
 
         api = DAApi(mock_store)
 
-        api.write(cert_id, mock_meta)
+        api.write(blob_id, mock_meta)
         with self.assertRaises(ValueError):
-            api.write(cert_id, mock_meta)
+            api.write(blob_id, mock_meta)
 
         blobs = api.read(app_id, [idx])
 
         self.assertEqual([expected_blob], blobs)
 
     def test_multiple_indexes_same_data(self):
         expected_blob = "hello"
-        cert_id = b"11"*32
+        blob_id = b"11"*32
         app_id = 1
         idx1 = 1
         idx2 = 2
         mock_meta1 = Metadata(app_id, idx1)
         mock_meta2 = Metadata(app_id, idx2)
 
         mock_store = MockStore()
-        mock_store.populate(expected_blob, cert_id)
+        mock_store.populate(expected_blob, blob_id)
 
         api = DAApi(mock_store)
 
-        api.write(cert_id, mock_meta1)
-        mock_store.populate(expected_blob, cert_id)
-        api.write(cert_id, mock_meta2)
+        api.write(blob_id, mock_meta1)
+        mock_store.populate(expected_blob, blob_id)
+        api.write(blob_id, mock_meta2)
 
         blobs_idx1 = api.read(app_id, [idx1])
         blobs_idx2 = api.read(app_id, [idx2])

da/common.py

@@ -7,6 +7,8 @@
 from py_ecc.bls import G2ProofOfPossession
 
 
+type BlobId = bytes
+
 class NodeId(Bytes32):
     pass
 
@@ -34,43 +36,12 @@ def transposed(self) -> Self:
         return ChunksMatrix(self.columns)
 
 
-BLSPublicKey = bytes
-BLSPrivateKey = int
-BLSSignature = bytes
-
 
 class Bitfield(List[bool]):
     pass
 
 
-@dataclass
-class Attestation:
-    signature: BLSSignature
-
-
-@dataclass
-class Certificate:
-    aggregated_signatures: BLSSignature
-    signers: Bitfield
-    aggregated_column_commitment: Commitment
-    row_commitments: List[Commitment]
-
-    def id(self) -> bytes:
-        return build_attestation_message(self.aggregated_column_commitment, self.row_commitments)
-
-    def verify(self, nodes_public_keys: List[BLSPublicKey]) -> bool:
-        """
-        List of nodes public keys should be a trusted list of verified proof of possession keys.
-        Otherwise, we could fall under the Rogue Key Attack
-        `assert all(bls_pop.PopVerify(pk, proof) for pk, proof in zip(node_public_keys, pops))`
-        """
-        # we sort them as the signers bitfield is sorted by the public keys as well
-        signers_keys = list(compress(sorted(nodes_public_keys), self.signers))
-        message = build_attestation_message(self.aggregated_column_commitment, self.row_commitments)
-        return NomosDaG2ProofOfPossession.AggregateVerify(signers_keys, [message]*len(signers_keys), self.aggregated_signatures)
-
-
-def build_attestation_message(aggregated_column_commitment: Commitment, row_commitments: Sequence[Commitment]) -> bytes:
+def build_blob_id(aggregated_column_commitment: Commitment, row_commitments: Sequence[Commitment]) -> BlobId:
     hasher = sha3_256()
     hasher.update(bytes(aggregated_column_commitment))
     for c in row_commitments:

da/dispersal.py

@@ -1,26 +1,22 @@
 from dataclasses import dataclass
-from hashlib import sha3_256
-from typing import List, Optional, Generator, Sequence
+from typing import List, Generator
 
-from da.common import Certificate, NodeId, BLSPublicKey, Bitfield, build_attestation_message, NomosDaG2ProofOfPossession as bls_pop
+from da.common import NodeId
 from da.encoder import EncodedData
-from da.verifier import DABlob, Attestation
+from da.verifier import DABlob
 
 
 @dataclass
 class DispersalSettings:
     nodes_ids: List[NodeId]
-    nodes_pubkey: List[BLSPublicKey]
     threshold: int
 
 
 class Dispersal:
     def __init__(self, settings: DispersalSettings):
         self.settings = settings
         # sort over public keys
-        self.settings.nodes_ids, self.settings.nodes_pubkey = zip(
-            *sorted(zip(self.settings.nodes_ids, self.settings.nodes_pubkey), key=lambda x: x[1])
-        )
+        self.settings.nodes_ids.sort()
 
     def _prepare_data(self, encoded_data: EncodedData) -> Generator[DABlob, None, None]:
         assert len(encoded_data.column_commitments) == len(self.settings.nodes_ids)
@@ -32,9 +28,10 @@ def _prepare_data(self, encoded_data: EncodedData) -> Generator[DABlob, None, No
         aggregated_column_commitment = encoded_data.aggregated_column_commitment
         aggregated_column_proofs = encoded_data.aggregated_column_proofs
         blobs_data = zip(columns, column_commitments, zip(*rows_proofs), aggregated_column_proofs)
-        for (column, column_commitment, row_proofs, column_proof) in blobs_data:
+        for column_idx, (column, column_commitment, row_proofs, column_proof) in enumerate(blobs_data):
             blob = DABlob(
                 column,
+                column_idx,
                 column_commitment,
                 aggregated_column_commitment,
                 column_proof,
@@ -43,48 +40,14 @@ def _prepare_data(self, encoded_data: EncodedData) -> Generator[DABlob, None, No
             )
             yield blob
 
-    def _send_and_await_response(self, node: NodeId, blob: DABlob) -> Optional[Attestation]:
+    def _send_and_await_response(self, node: NodeId, blob: DABlob) -> bool:
         pass
 
-    def _build_certificate(
-            self,
-            encoded_data: EncodedData,
-            attestations: Sequence[Attestation],
-            signers: Bitfield
-    ) -> Certificate:
-        assert len(attestations) >= self.settings.threshold
-        assert len(attestations) == signers.count(True)
-        aggregated = bls_pop.Aggregate([attestation.signature for attestation in attestations])
-        return Certificate(
-            aggregated_signatures=aggregated,
-            signers=signers,
-            aggregated_column_commitment=encoded_data.aggregated_column_commitment,
-            row_commitments=encoded_data.row_commitments
-        )
-
-    @staticmethod
-    def _verify_attestation(public_key: BLSPublicKey, attested_message: bytes, attestation: Attestation) -> bool:
-        return bls_pop.Verify(public_key, attested_message, attestation.signature)
-
-    @staticmethod
-    def _build_attestation_message(encoded_data: EncodedData) -> bytes:
-        return build_attestation_message(encoded_data.aggregated_column_commitment, encoded_data.row_commitments)
-
-    def disperse(self, encoded_data: EncodedData) -> Optional[Certificate]:
-        attestations = []
-        attested_message = self._build_attestation_message(encoded_data)
-        signed = Bitfield(False for _ in range(len(self.settings.nodes_ids)))
+    def disperse(self, encoded_data: EncodedData):
         blob_data = zip(
-            range(len(self.settings.nodes_ids)),
             self.settings.nodes_ids,
-            self.settings.nodes_pubkey,
             self._prepare_data(encoded_data)
         )
-        for i, node, pk, blob in blob_data:
-            if attestation := self._send_and_await_response(node, blob):
-                if self._verify_attestation(pk, attested_message, attestation):
-                    # mark as received
-                    signed[i] = True
-                    attestations.append(attestation)
-            if len(attestations) >= self.settings.threshold:
-                return self._build_certificate(encoded_data, attestations, signed)
+        for node, blob in blob_data:
+            self._send_and_await_response(node, blob)
+

da/encoder.py

@@ -5,9 +5,9 @@
 
 from eth2spec.eip7594.mainnet import KZGCommitment as Commitment, KZGProof as Proof, BLSFieldElement
 
-from da.common import ChunksMatrix, Chunk, Row, Column
+from da.common import ChunksMatrix, Chunk, Row
 from da.kzg_rs import kzg, rs
-from da.kzg_rs.common import GLOBAL_PARAMETERS, ROOTS_OF_UNITY, BLS_MODULUS, BYTES_PER_FIELD_ELEMENT
+from da.kzg_rs.common import GLOBAL_PARAMETERS, ROOTS_OF_UNITY, BYTES_PER_FIELD_ELEMENT
 from da.kzg_rs.poly import Polynomial
 
 
@@ -86,11 +86,11 @@ def _compute_column_kzg_commitments(self, chunks_matrix: ChunksMatrix) -> List[T
 
     @staticmethod
     def _compute_aggregated_column_commitment(
-        chunks_matrix: ChunksMatrix, column_commitments: Sequence[Commitment]
+        column_commitments: Sequence[Commitment]
     ) -> Tuple[Polynomial, Commitment]:
         data = bytes(chain.from_iterable(
-            DAEncoder.hash_column_and_commitment(column, commitment)
-            for column, commitment in zip(chunks_matrix.columns, column_commitments)
+            DAEncoder.hash_commitment_blake2b31(commitment)
+            for commitment in column_commitments
         ))
         return kzg.bytes_to_commitment(data, GLOBAL_PARAMETERS)
 
@@ -111,7 +111,7 @@ def encode(self, data: bytes) -> EncodedData:
         row_proofs = self._compute_rows_proofs(extended_matrix, row_polynomials, row_commitments)
         column_polynomials, column_commitments = zip(*self._compute_column_kzg_commitments(extended_matrix))
         aggregated_column_polynomial, aggregated_column_commitment = (
-            self._compute_aggregated_column_commitment(extended_matrix, column_commitments)
+            self._compute_aggregated_column_commitment(column_commitments)
         )
         aggregated_column_proofs = self._compute_aggregated_column_proofs(
             aggregated_column_polynomial, column_commitments
@@ -129,8 +129,8 @@ def encode(self, data: bytes) -> EncodedData:
         return result
 
     @staticmethod
-    def hash_column_and_commitment(column: Column, commitment: Commitment) -> bytes:
+    def hash_commitment_blake2b31(commitment: Commitment) -> bytes:
         return (
             # digest size must be 31 bytes as we cannot encode 32 without risking overflowing the BLS_MODULUS
-            int.from_bytes(blake2b(column.as_bytes() + bytes(commitment), digest_size=31).digest())
-        ).to_bytes(32, byteorder="big")
+            int.from_bytes(blake2b(bytes(commitment), digest_size=31).digest())
+        ).to_bytes(32, byteorder="big") # rewrap into 32 padded bytes for the field elements, EC library dependant