From 6ed08b4d7a2b248a43e77b80828a480c473cdf92 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Tue, 24 Sep 2024 03:46:57 +0200 Subject: [PATCH] [SMST] feat: Use compact SMST proofs (#823) ## Summary This PR changes the proofs being sent in `MsgSubmitPRoof` and stored by the `SubmitProof` keeper from `SparseMerkleClosestProof` to `SparseCompactMerkleClosestProof`. ## Issue `Proofs` one of the most block-space consuming primitives. The SMST support compact proofs that could help reduce their sizes. - #758 ## Type of change Select one or more from the following: - [x] New feature, functionality or library - [x] Consensus breaking; add the `consensus-breaking` label if so. See #791 for details - [ ] Bug fix - [ ] Code health or cleanup - [ ] Documentation - [ ] Other (specify) ## Testing - [x] **Unit Tests**: `make go_develop_and_test` - [x] **LocalNet E2E Tests**: `make test_e2e` - [ ] **DevNet E2E Tests**: Add the `devnet-test-e2e` label to the PR. ## Sanity Checklist - [x] I have tested my changes using the available tooling - [x] I have commented my code - [x] I have performed a self-review of my own code; both comments & source code --- api/poktroll/proof/tx.pulsar.go | 2 +- api/poktroll/proof/types.pulsar.go | 2 +- pkg/client/interface.go | 4 +- pkg/relayer/interface.go | 5 +- pkg/relayer/session/sessiontree.go | 44 ++++++---- pkg/relayer/session/sessiontree_test.go | 104 +++++++++++++++++++++++- proto/poktroll/proof/tx.proto | 2 +- proto/poktroll/proof/types.proto | 2 +- testutil/testtree/tree.go | 8 +- x/proof/keeper/proof.go | 2 - x/proof/keeper/proof_validation.go | 15 +++- x/proof/keeper/proof_validation_test.go | 11 ++- x/proof/types/tx.pb.go | 2 +- x/proof/types/types.pb.go | 2 +- 14 files changed, 163 insertions(+), 42 deletions(-) diff --git a/api/poktroll/proof/tx.pulsar.go b/api/poktroll/proof/tx.pulsar.go index 0f1d33850..0b815b1ae 100644 --- a/api/poktroll/proof/tx.pulsar.go +++ b/api/poktroll/proof/tx.pulsar.go @@ -4542,7 +4542,7 @@ type MsgSubmitProof struct { SupplierOperatorAddress string `protobuf:"bytes,1,opt,name=supplier_operator_address,json=supplierOperatorAddress,proto3" json:"supplier_operator_address,omitempty"` SessionHeader *session.SessionHeader `protobuf:"bytes,2,opt,name=session_header,json=sessionHeader,proto3" json:"session_header,omitempty"` - // serialized version of *smt.SparseMerkleClosestProof + // serialized version of *smt.SparseCompactMerkleClosestProof Proof []byte `protobuf:"bytes,3,opt,name=proof,proto3" json:"proof,omitempty"` } diff --git a/api/poktroll/proof/types.pulsar.go b/api/poktroll/proof/types.pulsar.go index b1eddf0fd..cace1d1e4 100644 --- a/api/poktroll/proof/types.pulsar.go +++ b/api/poktroll/proof/types.pulsar.go @@ -1272,7 +1272,7 @@ type Proof struct { SupplierOperatorAddress string `protobuf:"bytes,1,opt,name=supplier_operator_address,json=supplierOperatorAddress,proto3" json:"supplier_operator_address,omitempty"` // The session header of the session that this claim is for. SessionHeader *session.SessionHeader `protobuf:"bytes,2,opt,name=session_header,json=sessionHeader,proto3" json:"session_header,omitempty"` - // The serialized SMST proof from the `#ClosestProof()` method. + // The serialized SMST compacted proof from the `#ClosestProof()` method. ClosestMerkleProof []byte `protobuf:"bytes,3,opt,name=closest_merkle_proof,json=closestMerkleProof,proto3" json:"closest_merkle_proof,omitempty"` } diff --git a/pkg/client/interface.go b/pkg/client/interface.go index c003b36a1..502773709 100644 --- a/pkg/client/interface.go +++ b/pkg/client/interface.go @@ -65,11 +65,9 @@ type SupplierClient interface { ctx context.Context, claimMsgs ...MsgCreateClaim, ) error - // SubmitProof sends proof messages which contain the smt.SparseMerkleClosestProof, + // SubmitProof sends proof messages which contain the smt.SparseCompactMerkleClosestProof, // corresponding to some previously created claim for the same session. // The proof is validated on-chain as part of the pocket protocol. - // TODO_MAINNET(#427): Use SparseCompactClosestProof here to reduce - // the amount of data stored on-chain. SubmitProofs( ctx context.Context, sessionProofs ...MsgSubmitProof, diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 7f4c953cf..1f231e7f1 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -128,7 +128,7 @@ type SessionTree interface { // ProveClosest is a wrapper for the SMST's ProveClosest function. It returns the // proof for the given path. // This function should be called several blocks after a session has been claimed and needs to be proven. - ProveClosest(path []byte) (proof *smt.SparseMerkleClosestProof, err error) + ProveClosest(path []byte) (proof *smt.SparseCompactMerkleClosestProof, err error) // GetClaimRoot returns the root hash of the SMST needed for creating the claim. GetClaimRoot() []byte @@ -158,4 +158,7 @@ type SessionTree interface { // GetSupplierOperatorAddress returns the supplier operator address building this tree. GetSupplierOperatorAddress() *cosmostypes.AccAddress + + // GetTrieSpec returns the trie spec of the SMST. + GetTrieSpec() smt.TrieSpec } diff --git a/pkg/relayer/session/sessiontree.go b/pkg/relayer/session/sessiontree.go index 18e3ec607..b55340224 100644 --- a/pkg/relayer/session/sessiontree.go +++ b/pkg/relayer/session/sessiontree.go @@ -50,11 +50,11 @@ type sessionTree struct { // proofPath is the path for which the proof was generated. proofPath []byte - // proof is the generated proof for the session given a proofPath. - proof *smt.SparseMerkleClosestProof + // compactProof is the generated compactProof for the session given a proofPath. + compactProof *smt.SparseCompactMerkleClosestProof - // proofBz is the marshaled proof for the session. - proofBz []byte + // compactProofBz is the marshaled proof for the session. + compactProofBz []byte // treeStore is the KVStore used to store the SMST. treeStore pebble.PebbleKVStore @@ -154,9 +154,7 @@ func (st *sessionTree) Update(key, value []byte, weight uint64) error { // This function is intended to be called after a session has been claimed and needs to be proven. // If the proof has already been generated, it returns the cached proof. // It returns an error if the SMST has not been flushed yet (the claim has not been generated) -// TODO_IMPROVE(#427): Compress the proof into a SparseCompactClosestMerkleProof -// prior to submitting to chain to reduce on-chain storage requirements for proofs. -func (st *sessionTree) ProveClosest(path []byte) (proof *smt.SparseMerkleClosestProof, err error) { +func (st *sessionTree) ProveClosest(path []byte) (compactProof *smt.SparseCompactMerkleClosestProof, err error) { st.sessionMu.Lock() defer st.sessionMu.Unlock() @@ -166,13 +164,13 @@ func (st *sessionTree) ProveClosest(path []byte) (proof *smt.SparseMerkleClosest } // If the proof has already been generated, return the cached proof. - if st.proof != nil { + if st.compactProof != nil { // Make sure the path is the same as the one for which the proof was generated. if !bytes.Equal(path, st.proofPath) { return nil, ErrSessionTreeProofPathMismatch } - return st.proof, nil + return st.compactProof, nil } // Restore the KVStore from disk since it has been closed after the claim has been generated. @@ -184,12 +182,19 @@ func (st *sessionTree) ProveClosest(path []byte) (proof *smt.SparseMerkleClosest sessionSMT := smt.ImportSparseMerkleSumTrie(st.treeStore, sha256.New(), st.claimedRoot, smt.WithValueHasher(nil)) // Generate the proof and cache it along with the path for which it was generated. - proof, err = sessionSMT.ProveClosest(path) + // There is no ProveClosest variant that generates a compact proof directly. + // Generate a regular SparseMerkleClosestProof then compact it. + proof, err := sessionSMT.ProveClosest(path) if err != nil { return nil, err } - proofBz, err := proof.Marshal() + compactProof, err = smt.CompactClosestProof(proof, &sessionSMT.TrieSpec) + if err != nil { + return nil, err + } + + compactProofBz, err := compactProof.Marshal() if err != nil { return nil, err } @@ -197,20 +202,25 @@ func (st *sessionTree) ProveClosest(path []byte) (proof *smt.SparseMerkleClosest // If no error occurred, cache the proof and the path for which it was generated. st.sessionSMT = sessionSMT st.proofPath = path - st.proof = proof - st.proofBz = proofBz + st.compactProof = compactProof + st.compactProofBz = compactProofBz - return st.proof, nil + return st.compactProof, nil } // GetProofBz returns the marshaled proof for the session. func (st *sessionTree) GetProofBz() []byte { - return st.proofBz + return st.compactProofBz +} + +// GetTrieSpec returns the trie spec of the SMST. +func (st *sessionTree) GetTrieSpec() smt.TrieSpec { + return *st.sessionSMT.Spec() } // GetProof returns the proof for the SMST if it has been generated or nil otherwise. -func (st *sessionTree) GetProof() *smt.SparseMerkleClosestProof { - return st.proof +func (st *sessionTree) GetProof() *smt.SparseCompactMerkleClosestProof { + return st.compactProof } // Flush gets the root hash of the SMST needed for submitting the claim; diff --git a/pkg/relayer/session/sessiontree_test.go b/pkg/relayer/session/sessiontree_test.go index 4e199dcfe..762ebcb1b 100644 --- a/pkg/relayer/session/sessiontree_test.go +++ b/pkg/relayer/session/sessiontree_test.go @@ -1,3 +1,105 @@ package session_test -// TODO: Add tests to the sessionTree logic +import ( + "bytes" + "compress/gzip" + "crypto/rand" + "testing" + + "github.com/pokt-network/poktroll/pkg/crypto/protocol" + "github.com/pokt-network/smt" + "github.com/pokt-network/smt/kvstore/pebble" + "github.com/stretchr/testify/require" +) + +const ( + // Test multiple SMST sizes to see how the compaction ratio changes when the number + // of leaves increases. + // maxLeafs is the maximum number of leaves to test, after which the test stops. + maxLeafs = 10000 + // Since the inserted leaves are random, we run the test for a given leaf count + // multiple times to remove the randomness bias. + numIterations = 100 +) + +// No significant performance gains were observed when using compact proofs compared +// to non-compact proofs. +// In fact, compact proofs appear to be less efficient than gzipped proofs, even +// without considering the "proof closest value" compression. +// For a sample comparison between compression and compaction ratios, see: +// https://github.com/pokt-network/poktroll/pull/823#issuecomment-2363987920 +func TestSessionTree_CompactProofsAreSmallerThanNonCompactProofs(t *testing.T) { + // Run the test for different number of leaves. + for numLeafs := 10; numLeafs <= maxLeafs; numLeafs *= 10 { + cumulativeProofSize := 0 + cumulativeCompactProofSize := 0 + cumulativeGzippedProofSize := 0 + // We run the test numIterations times for each number of leaves to remove the randomness bias. + for iteration := 0; iteration <= numIterations; iteration++ { + kvStore, err := pebble.NewKVStore("") + require.NoError(t, err) + + trie := smt.NewSparseMerkleSumTrie(kvStore, protocol.NewTrieHasher(), smt.WithValueHasher(nil)) + + // Insert numLeaf random leaves. + for i := 0; i < numLeafs; i++ { + key := make([]byte, 32) + _, err = rand.Read(key) + require.NoError(t, err) + // Insert an empty value since this does not get affected by the compaction, + // this is also to not favor proof compression that compresses the value too. + trie.Update(key, []byte{}, 1) + } + + // Generate a random path. + var path = make([]byte, 32) + _, err = rand.Read(path) + require.NoError(t, err) + + // Create the proof. + proof, err := trie.ProveClosest(path) + require.NoError(t, err) + + proofBz, err := proof.Marshal() + require.NoError(t, err) + + // Accumulate the proof size over numIterations runs. + cumulativeProofSize += len(proofBz) + + // Generate the compacted proof. + compactProof, err := smt.CompactClosestProof(proof, &trie.TrieSpec) + require.NoError(t, err) + + compactProofBz, err := compactProof.Marshal() + require.NoError(t, err) + + // Accumulate the compact proof size over numIterations runs. + cumulativeCompactProofSize += len(compactProofBz) + + // Gzip the non compacted proof. + var buf bytes.Buffer + gzipWriter := gzip.NewWriter(&buf) + _, err = gzipWriter.Write(proofBz) + require.NoError(t, err) + err = gzipWriter.Close() + require.NoError(t, err) + + // Accumulate the gzipped proof size over numIterations runs. + cumulativeGzippedProofSize += len(buf.Bytes()) + } + + // Calculate how much more efficient compact SMT proofs are compared to non-compact proofs. + compactionRatio := float32(cumulativeProofSize) / float32(cumulativeCompactProofSize) + + // Claculate how much more efficient gzipped proofs are compared to non-compact proofs. + compressionRatio := float32(cumulativeProofSize) / float32(cumulativeGzippedProofSize) + + // Gzip compression is more efficient than SMT compaction. + require.Greater(t, compressionRatio, compactionRatio) + + t.Logf( + "numLeaf=%d: compactionRatio: %f, compressionRatio: %f", + numLeafs, compactionRatio, compressionRatio, + ) + } +} diff --git a/proto/poktroll/proof/tx.proto b/proto/poktroll/proof/tx.proto index 6ac2a595e..4264f0daa 100644 --- a/proto/poktroll/proof/tx.proto +++ b/proto/poktroll/proof/tx.proto @@ -91,7 +91,7 @@ message MsgSubmitProof { string supplier_operator_address = 1 [(cosmos_proto.scalar) = "cosmos.AddressString"]; poktroll.session.SessionHeader session_header = 2; - // serialized version of *smt.SparseMerkleClosestProof + // serialized version of *smt.SparseCompactMerkleClosestProof bytes proof = 3; } diff --git a/proto/poktroll/proof/types.proto b/proto/poktroll/proof/types.proto index f968e4cb8..5a67772e2 100644 --- a/proto/poktroll/proof/types.proto +++ b/proto/poktroll/proof/types.proto @@ -18,7 +18,7 @@ message Proof { string supplier_operator_address = 1 [(cosmos_proto.scalar) = "cosmos.AddressString"]; // The session header of the session that this claim is for. poktroll.session.SessionHeader session_header = 2; - // The serialized SMST proof from the `#ClosestProof()` method. + // The serialized SMST compacted proof from the `#ClosestProof()` method. bytes closest_merkle_proof = 3; } diff --git a/testutil/testtree/tree.go b/testutil/testtree/tree.go index 5774de7c8..6743d753f 100644 --- a/testutil/testtree/tree.go +++ b/testutil/testtree/tree.go @@ -121,18 +121,18 @@ func NewProof( t.Helper() // Generate a closest proof from the session tree using closestProofPath. - merkleProof, err := sessionTree.ProveClosest(closestProofPath) + merkleCompactProof, err := sessionTree.ProveClosest(closestProofPath) require.NoError(t, err) - require.NotNil(t, merkleProof) + require.NotNil(t, merkleCompactProof) // Serialize the closest merkle proof. - merkleProofBz, err := merkleProof.Marshal() + merkleCompactProofBz, err := merkleCompactProof.Marshal() require.NoError(t, err) return &prooftypes.Proof{ SupplierOperatorAddress: supplierOperatorAddr, SessionHeader: sessionHeader, - ClosestMerkleProof: merkleProofBz, + ClosestMerkleProof: merkleCompactProofBz, } } diff --git a/x/proof/keeper/proof.go b/x/proof/keeper/proof.go index 1309920a0..915803199 100644 --- a/x/proof/keeper/proof.go +++ b/x/proof/keeper/proof.go @@ -15,8 +15,6 @@ import ( func (k Keeper) UpsertProof(ctx context.Context, proof types.Proof) { logger := k.Logger().With("method", "UpsertProof") - // TODO_MAINNET(#427): Use the marshal method on the SparseCompactClosestProof - // type here instead in order to reduce space stored on chain. proofBz := k.cdc.MustMarshal(&proof) storeAdapter := runtime.KVStoreAdapter(k.storeService.OpenKVStore(ctx)) diff --git a/x/proof/keeper/proof_validation.go b/x/proof/keeper/proof_validation.go index 638e01d0a..f771ca626 100644 --- a/x/proof/keeper/proof_validation.go +++ b/x/proof/keeper/proof_validation.go @@ -91,16 +91,23 @@ func (k Keeper) EnsureValidProof( } // Unmarshal the closest merkle proof from the message. - sparseMerkleClosestProof := &smt.SparseMerkleClosestProof{} - if err = sparseMerkleClosestProof.Unmarshal(proof.ClosestMerkleProof); err != nil { + sparseCompactMerkleClosestProof := &smt.SparseCompactMerkleClosestProof{} + if err = sparseCompactMerkleClosestProof.Unmarshal(proof.ClosestMerkleProof); err != nil { return types.ErrProofInvalidProof.Wrapf( "failed to unmarshal closest merkle proof: %s", err, ) } - // TODO_MAINNET(#427): Utilize smt.VerifyCompactClosestProof here to - // reduce on-chain storage requirements for proofs. + // SparseCompactMerkeClosestProof does not implement GetValueHash, so we need to decompact it. + sparseMerkleClosestProof, err := smt.DecompactClosestProof(sparseCompactMerkleClosestProof, &protocol.SmtSpec) + if err != nil { + return types.ErrProofInvalidProof.Wrapf( + "failed to decompact closest merkle proof: %s", + err, + ) + } + // Get the relay request and response from the proof.GetClosestMerkleProof. relayBz := sparseMerkleClosestProof.GetValueHash(&protocol.SmtSpec) relay := &servicetypes.Relay{} diff --git a/x/proof/keeper/proof_validation_test.go b/x/proof/keeper/proof_validation_test.go index 7c7ba667f..13222338d 100644 --- a/x/proof/keeper/proof_validation_test.go +++ b/x/proof/keeper/proof_validation_test.go @@ -156,8 +156,8 @@ func TestEnsureValidProof_Error(t *testing.T) { // Store the expected error returned during deserialization of the invalid // closest Merkle proof bytes. - sparseMerkleClosestProof := &smt.SparseMerkleClosestProof{} - expectedInvalidProofUnmarshalErr := sparseMerkleClosestProof.Unmarshal(invalidClosestProofBytes) + sparseCompactMerkleClosestProof := &smt.SparseCompactMerkleClosestProof{} + expectedInvalidProofUnmarshalErr := sparseCompactMerkleClosestProof.Unmarshal(invalidClosestProofBytes) // Construct a relay to be mangled such that it fails to deserialize in order // to set the error expectation for the relevant test case. @@ -611,9 +611,12 @@ func TestEnsureValidProof_Error(t *testing.T) { ) // Extract relayHash to check below that it's difficulty is insufficient - sparseMerkleClosestProof := &smt.SparseMerkleClosestProof{} - err = sparseMerkleClosestProof.Unmarshal(proof.ClosestMerkleProof) + err = sparseCompactMerkleClosestProof.Unmarshal(proof.ClosestMerkleProof) require.NoError(t, err) + var sparseMerkleClosestProof *smt.SparseMerkleClosestProof + sparseMerkleClosestProof, err = smt.DecompactClosestProof(sparseCompactMerkleClosestProof, &protocol.SmtSpec) + require.NoError(t, err) + relayBz := sparseMerkleClosestProof.GetValueHash(&protocol.SmtSpec) relayHashArr := protocol.GetRelayHashFromBytes(relayBz) relayHash := relayHashArr[:] diff --git a/x/proof/types/tx.pb.go b/x/proof/types/tx.pb.go index a98755961..8d5376a58 100644 --- a/x/proof/types/tx.pb.go +++ b/x/proof/types/tx.pb.go @@ -403,7 +403,7 @@ func (m *MsgCreateClaimResponse) GetClaim() *Claim { type MsgSubmitProof struct { SupplierOperatorAddress string `protobuf:"bytes,1,opt,name=supplier_operator_address,json=supplierOperatorAddress,proto3" json:"supplier_operator_address,omitempty"` SessionHeader *types1.SessionHeader `protobuf:"bytes,2,opt,name=session_header,json=sessionHeader,proto3" json:"session_header,omitempty"` - // serialized version of *smt.SparseMerkleClosestProof + // serialized version of *smt.SparseCompactMerkleClosestProof Proof []byte `protobuf:"bytes,3,opt,name=proof,proto3" json:"proof,omitempty"` } diff --git a/x/proof/types/types.pb.go b/x/proof/types/types.pb.go index 52230cff6..a316197dc 100644 --- a/x/proof/types/types.pb.go +++ b/x/proof/types/types.pb.go @@ -89,7 +89,7 @@ type Proof struct { SupplierOperatorAddress string `protobuf:"bytes,1,opt,name=supplier_operator_address,json=supplierOperatorAddress,proto3" json:"supplier_operator_address,omitempty"` // The session header of the session that this claim is for. SessionHeader *types.SessionHeader `protobuf:"bytes,2,opt,name=session_header,json=sessionHeader,proto3" json:"session_header,omitempty"` - // The serialized SMST proof from the `#ClosestProof()` method. + // The serialized SMST compacted proof from the `#ClosestProof()` method. ClosestMerkleProof []byte `protobuf:"bytes,3,opt,name=closest_merkle_proof,json=closestMerkleProof,proto3" json:"closest_merkle_proof,omitempty"` }