feat: Add prehash_compare_key to allow proving nonexistence in sparse trees (cosmos#136)

* add prehash_compare_key

prehash_compare_key indicates whether to compare the keys lexicographically according to their _hashed_ values (implied by the hash function given by prehash_key). This is required for nonexistence proofs in proof specs that use prehashing.

* use keyForComparison in getNonExistProofForKey

* apply keyForComparison in typescript verifyNonExistence

* update smt spec to use prehash_sha256

* rename prehash_compare_key -> prehash_key_before_comparison

* fix: Make it compile in no_std by removing naming of Vec

* doc: Add comment describing keyForComparison/key_for_comparison

* doc: Elaborate more on why prehashing before comparison

* use updated smt test vectors

* Avoid cloning `leaf_spec` when prehashing keys for comparison

co-authored-by: @romac

Co-authored-by: Romain Ruetschi <[email protected]>

* Fix missing `Eq` derive on prost build (reverting)

* Revert "Fix missing `Eq` derive on prost build (reverting)"

This reverts commit 69a38e8.

* Fix fmt issue in CI, fix generated code to contain `Eq` impl

* add testing for SMT proofs in js

---------

Co-authored-by: Ava Howell <[email protected]>
Co-authored-by: Ava Howell <[email protected]>
Co-authored-by: Romain Ruetschi <[email protected]>

Author: 3 people

go/ics23.go

@@ -52,7 +52,7 @@ func VerifyMembership(spec *ProofSpec, root CommitmentRoot, proof *CommitmentPro
 func VerifyNonMembership(spec *ProofSpec, root CommitmentRoot, proof *CommitmentProof, key []byte) bool {
 	// decompress it before running code (no-op if not compressed)
 	proof = Decompress(proof)
-	np := getNonExistProofForKey(proof, key)
+	np := getNonExistProofForKey(spec, proof, key)
 	if np == nil {
 		return false
 	}
@@ -148,27 +148,27 @@ func getExistProofForKey(proof *CommitmentProof, key []byte) *ExistenceProof {
 	return nil
 }
 
-func getNonExistProofForKey(proof *CommitmentProof, key []byte) *NonExistenceProof {
+func getNonExistProofForKey(spec *ProofSpec, proof *CommitmentProof, key []byte) *NonExistenceProof {
 	switch p := proof.Proof.(type) {
 	case *CommitmentProof_Nonexist:
 		np := p.Nonexist
-		if isLeft(np.Left, key) && isRight(np.Right, key) {
+		if isLeft(spec, np.Left, key) && isRight(spec, np.Right, key) {
 			return np
 		}
 	case *CommitmentProof_Batch:
 		for _, sub := range p.Batch.Entries {
-			if np := sub.GetNonexist(); np != nil && isLeft(np.Left, key) && isRight(np.Right, key) {
+			if np := sub.GetNonexist(); np != nil && isLeft(spec, np.Left, key) && isRight(spec, np.Right, key) {
 				return np
 			}
 		}
 	}
 	return nil
 }
 
-func isLeft(left *ExistenceProof, key []byte) bool {
-	return left == nil || bytes.Compare(left.Key, key) < 0
+func isLeft(spec *ProofSpec, left *ExistenceProof, key []byte) bool {
+	return left == nil || bytes.Compare(keyForComparison(spec, left.Key), keyForComparison(spec, key)) < 0
 }
 
-func isRight(right *ExistenceProof, key []byte) bool {
-	return right == nil || bytes.Compare(right.Key, key) > 0
+func isRight(spec *ProofSpec, right *ExistenceProof, key []byte) bool {
+	return right == nil || bytes.Compare(keyForComparison(spec, right.Key), keyForComparison(spec, key)) > 0
 }

go/proof.go

@@ -47,7 +47,7 @@ var TendermintSpec = &ProofSpec{
 var SmtSpec = &ProofSpec{
 	LeafSpec: &LeafOp{
 		Hash:         HashOp_SHA256,
-		PrehashKey:   HashOp_NO_HASH,
+		PrehashKey:   HashOp_SHA256,
 		PrehashValue: HashOp_SHA256,
 		Length:       LengthOp_NO_PREFIX,
 		Prefix:       []byte{0},
@@ -60,7 +60,8 @@ var SmtSpec = &ProofSpec{
 		EmptyChild:      make([]byte, 32),
 		Hash:            HashOp_SHA256,
 	},
-	MaxDepth: 256,
+	MaxDepth:                   256,
+	PrehashKeyBeforeComparison: true,
 }
 
 func encodeVarintProto(l int) []byte {
@@ -204,6 +205,17 @@ func (p *ExistenceProof) CheckAgainstSpec(spec *ProofSpec) error {
 	return nil
 }
 
+// If we should prehash the key before comparison, do so; otherwise, return the key. Prehashing
+// changes lexical comparison, so we do so before comparison if the spec sets
+// `PrehashKeyBeforeComparison`.
+func keyForComparison(spec *ProofSpec, key []byte) []byte {
+	if !spec.PrehashKeyBeforeComparison {
+		return key
+	}
+	hash, _ := doHashOrNoop(spec.LeafSpec.PrehashKey, key)
+	return hash
+}
+
 // Verify does all checks to ensure the proof has valid non-existence proofs,
 // and they ensure the given key is not in the CommitmentState
 func (p *NonExistenceProof) Verify(spec *ProofSpec, root CommitmentRoot, key []byte) error {
@@ -229,13 +241,13 @@ func (p *NonExistenceProof) Verify(spec *ProofSpec, root CommitmentRoot, key []b
 
 	// Ensure in valid range
 	if rightKey != nil {
-		if bytes.Compare(key, rightKey) >= 0 {
+		if bytes.Compare(keyForComparison(spec, key), keyForComparison(spec, rightKey)) >= 0 {
 			return errors.New("key is not left of right proof")
 		}
 	}
 
 	if leftKey != nil {
-		if bytes.Compare(key, leftKey) <= 0 {
+		if bytes.Compare(keyForComparison(spec, key), keyForComparison(spec, leftKey)) <= 0 {
 			return errors.New("key is not right of left proof")
 		}
 	}

go/proofs.pb.go

@@ -703,6 +703,9 @@ export namespace ics23 {
 
     /** ProofSpec minDepth */
     minDepth?: number | null;
+
+    /** ProofSpec prehashKeyBeforeComparison */
+    prehashKeyBeforeComparison?: boolean | null;
   }
 
   /**
@@ -736,6 +739,9 @@ export namespace ics23 {
     /** ProofSpec minDepth. */
     public minDepth: number;
 
+    /** ProofSpec prehashKeyBeforeComparison. */
+    public prehashKeyBeforeComparison: boolean;
+
     /**
      * Creates a new ProofSpec instance using the specified properties.
      * @param [properties] Properties to set