improving ledger docs

Author: ramtinms

ledger/README.md

@@ -1,9 +1,9 @@
-# Flow Ledger Package**
+# Flow Ledger Package
 
 **Ledger** is a stateful fork-aware key/value storage. Any update (value change for a key) to the ledger generates a new ledger state. Updates can be applied to any recent state. These changes don't have to be sequential and ledger supports a tree of states. Ledger provides value lookup by key at a particular state (historic lookups) and can prove the existence/non-existence of a key-value pair at the given state. Ledger assumes the initial state includes all keys with an empty bytes slice as value.
 
-This package provides two various implementation of ledger:
+This package provides two ledger implementations:
 
-- **Complete Ledger** implements a fast, memory-efficient and reliable ledger. It holds a limited number of recently active states in memory (for speed) and uses write-ahead logs and checkpointing to provide reliability. Under the hood complete ledger uses a collection of MTries (forest). MTrie is a customized in-memory binary Patricia Merkle trie storing payloads at specific storage paths. The payload includes both key-value pair and storage paths are determined by the PathFinder. Forest utilizes unchanged sub-trie sharing between tries to save memory.
+- **Complete Ledger** implements a fast, memory-efficient and reliable ledger. It holds a limited number of recently used states in memory (for speed) and uses write-ahead logs and checkpointing to provide reliability. Under the hood complete ledger uses a collection of MTries(forest). MTrie is a customized in-memory binary Patricia Merkle trie storing payloads at specific storage paths. The payload includes both key-value pair and storage paths are determined by the PathFinder. Forest utilizes unchanged sub-trie sharing between tries to save memory.
 
-- **Partial Ledger** implements the ledger functionality for a limited subset of keys. Partial ledgers are designed to be constructed and verified by a collection of proofs from a complete ledger. The partial ledger uses a partial binary Merkle trie which holds intermediate hash value for the pruned branched and prevents updates to keys that were not part of proofs.
+- **Partial Ledger** implements the ledger functionality for a limited subset of keys. Partial ledgers are designed to be constructed and verified by a collection of proofs from a complete ledger. The partial ledger uses a partial binary Merkle trie which holds intermediate hash value for the pruned branched and prevents updates to keys that were not part of proofs.

ledger/common/encoding/encoding.go

@@ -1,3 +1,4 @@
+// Package encoding provides byte serialization and deserialization of trie and ledger structs.
 package encoding
 
 import (
@@ -7,15 +8,14 @@ import (
 	"github.com/dapperlabs/flow-go/ledger/common/utils"
 )
 
-// Version encoder/decoder code only supports
-// decoding data with version smaller or equal to this value
-// bumping this number prevents older versions of the code
-// to deal with the newer version of data
+// Version captures the maximum version of encoding that this code supports
+// in other words this code encodes the data with the latest version and
+// can only decode data with version smaller or equal to this value
+// bumping this number prevents older versions of the code to deal with the newer version of data
 // codes should be updated with backward compatibility if needed
-// and act differently based on the version
 const Version = uint16(0)
 
-// Type capture the type of encoded entity
+// Type capture the type of encoded entity (e.g. State, Key, Value, Path)
 type Type uint8
 
 const (
@@ -49,7 +49,7 @@ const (
 )
 
 func (e Type) String() string {
-	return [...]string{"Unknown", "State", "KeyPart", "Key", "Value", "Path", "Payload", "Proof", "BatchProof"}[e]
+	return [...]string{"Unknown", "State", "KeyPart", "Key", "Value", "Path", "Payload", "Proof", "BatchProof", "Query", "Update", "Trie Update"}[e]
 }
 
 // CheckVersion extracts encoding bytes from a raw encoded message
@@ -545,7 +545,7 @@ func decodeTrieUpdate(inp []byte) (*ledger.TrieUpdate, error) {
 	return &ledger.TrieUpdate{RootHash: rh, Paths: paths, Payloads: payloads}, nil
 }
 
-// EncodeProof encodes the content of a proof into a byte slice
+// EncodeTrieProof encodes the content of a proof into a byte slice
 func EncodeTrieProof(p *ledger.TrieProof) []byte {
 	if p == nil {
 		return []byte{}

ledger/common/hash.go

@@ -6,6 +6,7 @@ import (
 	"github.com/dapperlabs/flow-go/ledger/common/utils"
 )
 
+// default value and default hash value for a default node
 var emptySlice []byte
 var defaultLeafHash = HashLeaf([]byte("default:"), emptySlice)
 

ledger/common/pathfinder/pathfinder.go

@@ -1,3 +1,4 @@
+// Package pathfinder computes the trie storage path for any given key/value pair
 package pathfinder
 
 import (

ledger/common/proof.go

@@ -8,6 +8,7 @@ import (
 )
 
 // TODO move this to proof itself
+
 // VerifyTrieProof verifies the proof, by constructing all the
 // hash from the leaf to the root and comparing the rootHash
 func VerifyTrieProof(p *ledger.TrieProof, expectedState ledger.State) bool {

ledger/common/utils/test.go

@@ -21,24 +21,28 @@ func TwoBytesPath(inp uint16) ledger.Path {
 	return ledger.Path(b)
 }
 
+// LightPayload returns a payload with 2 byte key and 2 byte value
 func LightPayload(key uint16, value uint16) *ledger.Payload {
 	k := ledger.Key{KeyParts: []ledger.KeyPart{ledger.KeyPart{Type: 0, Value: Uint16ToBinary(key)}}}
 	v := ledger.Value(Uint16ToBinary(value))
 	return &ledger.Payload{Key: k, Value: v}
 }
 
+// LightPayload8 returns a payload with 1 byte key and 1 byte value
 func LightPayload8(key uint8, value uint8) *ledger.Payload {
 	k := ledger.Key{KeyParts: []ledger.KeyPart{ledger.KeyPart{Type: 0, Value: []byte{key}}}}
 	v := ledger.Value([]byte{value})
 	return &ledger.Payload{Key: k, Value: v}
 }
 
+// KeyPartFixture returns a key part fixture
 func KeyPartFixture(typ uint16, val string) *ledger.KeyPart {
 	kp1t := uint16(typ)
 	kp1v := []byte(val)
 	return ledger.NewKeyPart(kp1t, kp1v)
 }
 
+// QueryFixture returns a query fixture
 func QueryFixture() *ledger.Query {
 	sc, _ := hex.DecodeString("6a7a565add94fb36069d79e8725c221cd1e5740742501ef014ea6db999fd98ad")
 	k1p1 := ledger.NewKeyPart(uint16(1), []byte("1"))
@@ -53,6 +57,7 @@ func QueryFixture() *ledger.Query {
 	return u
 }
 
+// UpdateFixture returns an update fixture
 func UpdateFixture() *ledger.Update {
 	sc, _ := hex.DecodeString("6a7a565add94fb36069d79e8725c221cd1e5740742501ef014ea6db999fd98ad")
 	k1p1 := ledger.NewKeyPart(uint16(1), []byte("1"))
@@ -69,11 +74,13 @@ func UpdateFixture() *ledger.Update {
 	return u
 }
 
+// RootHashFixture returns a root hash fixture
 func RootHashFixture() ledger.RootHash {
 	sc, _ := hex.DecodeString("6a7a565add94fb36069d79e8725c221cd1e5740742501ef014ea6db999fd98ad")
 	return ledger.RootHash(sc)
 }
 
+// TrieProofFixture returns a trie proof fixture
 func TrieProofFixture() (*ledger.TrieProof, ledger.State) {
 	p := ledger.NewTrieProof()
 	p.Path = TwoBytesPath(330)
@@ -90,6 +97,7 @@ func TrieProofFixture() (*ledger.TrieProof, ledger.State) {
 	return p, ledger.State(sc)
 }
 
+// TrieBatchProofFixture returns a trie batch proof fixture
 func TrieBatchProofFixture() (*ledger.TrieBatchProof, ledger.State) {
 	p1 := ledger.NewTrieProof()
 	p1.Path = TwoBytesPath(330)
@@ -170,6 +178,7 @@ func RandomPayloads(n int, minByteSize int, maxByteSize int) []*ledger.Payload {
 	return res
 }
 
+// RandomValues returns n random values with variable sizes (minByteSize <= size < maxByteSize)
 func RandomValues(n int, minByteSize, maxByteSize int) []ledger.Value {
 	if minByteSize > maxByteSize {
 		panic("minByteSize cannot be smaller then maxByteSize")

ledger/common/utils/utils.go

@@ -8,7 +8,7 @@ import (
 )
 
 // IsBitSet returns if the bit at index `idx` in the byte array `b` is set to 1 (big endian)
-// TODO: remove error return
+// TODO: remove error return to improve performance
 func IsBitSet(b []byte, idx int) (bool, error) {
 	if idx >= len(b)*8 {
 		return false, fmt.Errorf("input (%v) only has %d bits, can't look up bit %d", b, len(b)*8, idx)
@@ -17,7 +17,7 @@ func IsBitSet(b []byte, idx int) (bool, error) {
 }
 
 // SetBit sets the bit at position i in the byte array b to 1
-// TODO: remove error return
+// TODO: remove error return to improve performance
 func SetBit(b []byte, i int) error {
 	if i >= len(b)*8 {
 		return fmt.Errorf("input (%v) only has %d bits, can't set bit %d", b, len(b)*8, i)
@@ -48,22 +48,26 @@ func Uint64ToBinary(integer uint64) []byte {
 	return b
 }
 
+// AppendUint8 appends the value byte to the input slice
 func AppendUint8(input []byte, value uint8) []byte {
 	return append(input, byte(value))
 }
 
+// AppendUint16 appends the value bytes to the input slice (big endian)
 func AppendUint16(input []byte, value uint16) []byte {
 	buffer := make([]byte, 2)
 	binary.BigEndian.PutUint16(buffer, value)
 	return append(input, buffer...)
 }
 
+// AppendUint32 appends the value bytes to the input slice (big endian)
 func AppendUint32(input []byte, value uint32) []byte {
 	buffer := make([]byte, 4)
 	binary.BigEndian.PutUint32(buffer, value)
 	return append(input, buffer...)
 }
 
+// AppendUint64 appends the value bytes to the input slice (big endian)
 func AppendUint64(input []byte, value uint64) []byte {
 	buffer := make([]byte, 8)
 	binary.BigEndian.PutUint64(buffer, value)
@@ -90,34 +94,39 @@ func AppendLongData(input []byte, data []byte) []byte {
 	return input
 }
 
+// ReadSlice reads `size` bytes from the input
 func ReadSlice(input []byte, size int) (value []byte, rest []byte, err error) {
 	if len(input) < size {
 		return nil, input, fmt.Errorf("input size is too small to be splited %d < %d ", len(input), size)
 	}
 	return input[:size], input[size:], nil
 }
 
+// ReadUint8 reads a uint8 from the input and returns the rest
 func ReadUint8(input []byte) (value uint8, rest []byte, err error) {
 	if len(input) < 1 {
 		return 0, input, fmt.Errorf("input size (%d) is too small to read a uint8", len(input))
 	}
 	return uint8(input[0]), input[1:], nil
 }
 
+// ReadUint16 reads a uint16 from the input and returns the rest
 func ReadUint16(input []byte) (value uint16, rest []byte, err error) {
 	if len(input) < 2 {
 		return 0, input, fmt.Errorf("input size (%d) is too small to read a uint16", len(input))
 	}
 	return binary.BigEndian.Uint16(input[:2]), input[2:], nil
 }
 
+// ReadUint32 reads a uint32 from the input and returns the rest
 func ReadUint32(input []byte) (value uint32, rest []byte, err error) {
 	if len(input) < 4 {
 		return 0, input, fmt.Errorf("input size (%d) is too small to read a uint32", len(input))
 	}
 	return binary.BigEndian.Uint32(input[:4]), input[4:], nil
 }
 
+// ReadUint64 reads a uint64 from the input and returns the rest
 func ReadUint64(input []byte) (value uint64, rest []byte, err error) {
 	if len(input) < 8 {
 		return 0, input, fmt.Errorf("input size (%d) is too small to read a uint64", len(input))
@@ -185,6 +194,7 @@ func ReadLongDataFromReader(reader io.Reader) ([]byte, error) {
 	return buf, nil
 }
 
+// ReadFromBuffer reads 'length' bytes from the input
 func ReadFromBuffer(reader io.Reader, length int) ([]byte, error) {
 	if length == 0 {
 		return nil, nil

ledger/complete/ledger.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/rs/zerolog"
 
 	"github.com/dapperlabs/flow-go/ledger"
 	"github.com/dapperlabs/flow-go/ledger/common/encoding"
@@ -16,26 +17,29 @@ import (
 )
 
 // Ledger (complete) is a fast memory-efficient fork-aware thread-safe trie-based key/value storage.
-// Ledger holds an array of registers (key value pairs) and keep tracks of changes over a limited time.
+// Ledger holds an array of registers (key-value pairs) and keeps tracks of changes over a limited time.
 // Each register is referenced by an ID (key) and holds a value (byte slice).
-// Ledger provides atomic batched update and read (with or without proofs) operation given a list of keys.
+// Ledger provides atomic batched updates and read (with or without proofs) operation given a list of keys.
 // Every update to the Ledger creates a new state which captures the state of the storage.
-// Under the hood, it uses binary merkle tries to generate inclusion and noninclusion proofs.
-// Ledger is fork-aware that means any update can applied at any previous state which forms a tree of tries (forest).
-// The forrest is in memory but all changes (e.g. register updates) are captured inside write-ahead-logs for crash recovery reasons.
-// In order to limit the memory usage and maintain the performance storage only keeps limited number of
-// tries and purge the old ones (LRU-based); in other words Ledger is not designed to be used
-// for archival use but make it possible for other software components to reconstruct very old tries using write-ahead logs.
+// Under the hood, it uses binary Merkle tries to generate inclusion and non-inclusion proofs.
+// Ledger is fork-aware which means any update can be applied at any previous state which forms a tree of tries (forest).
+// The forest is in memory but all changes (e.g. register updates) are captured inside write-ahead-logs for crash recovery reasons.
+// In order to limit the memory usage and maintain the performance storage only keeps a limited number of
+// tries and purge the old ones (LRU-based); in other words, Ledger is not designed to be used
+// for archival usage but make it possible for other software components to reconstruct very old tries using write-ahead logs.
 type Ledger struct {
 	forest  *mtrie.Forest
 	wal     *wal.LedgerWAL
 	metrics module.LedgerMetrics
+	logger  zerolog.Logger
 }
 
-const CacheSize = 1000
-
 // NewLedger creates a new in-memory trie-backed ledger storage with persistence.
-func NewLedger(dbDir string, capacity int, metrics module.LedgerMetrics, reg prometheus.Registerer) (*Ledger, error) {
+func NewLedger(dbDir string,
+	capacity int,
+	metrics module.LedgerMetrics,
+	log zerolog.Logger,
+	reg prometheus.Registerer) (*Ledger, error) {
 
 	w, err := wal.NewWAL(nil, reg, dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize)
 	if err != nil {
@@ -48,10 +52,14 @@ func NewLedger(dbDir string, capacity int, metrics module.LedgerMetrics, reg pro
 	if err != nil {
 		return nil, fmt.Errorf("cannot create forest: %w", err)
 	}
+
+	logger := log.With().Str("ledger", "complete").Logger()
+
 	storage := &Ledger{
 		forest:  forest,
 		wal:     w,
 		metrics: metrics,
+		logger:  logger,
 	}
 
 	err = w.ReplayOnForest(forest)
@@ -157,7 +165,10 @@ func (l *Ledger) Set(update *ledger.Update) (newState ledger.State, err error) {
 		l.metrics.UpdateDurationPerItem(durationPerValue)
 	}
 
-	// TODO log info state
+	l.logger.Info().Hex("from", update.State()).
+		Hex("to", newRootHash[:]).
+		Int("update_size", update.Size()).
+		Msg("ledger updated")
 	return ledger.State(newRootHash), nil
 }
 
@@ -189,6 +200,7 @@ func (l *Ledger) CloseStorage() {
 	_ = l.wal.Close()
 }
 
+// MemSize return the amount of memory used by ledger
 // TODO implement an approximate MemSize method
 func (l *Ledger) MemSize() (int64, error) {
 	return 0, nil
@@ -204,6 +216,7 @@ func (l *Ledger) ForestSize() int {
 	return l.forest.Size()
 }
 
+// Checkpointer returns a checkpointer instance
 func (l *Ledger) Checkpointer() (*wal.Checkpointer, error) {
 	checkpointer, err := l.wal.NewCheckpointer()
 	if err != nil {

ledger/complete/ledger_benchmark_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/rs/zerolog"
+
 	"github.com/dapperlabs/flow-go/ledger"
 	"github.com/dapperlabs/flow-go/ledger/common/encoding"
 	"github.com/dapperlabs/flow-go/ledger/common/utils"
@@ -38,7 +40,7 @@ func benchmarkStorage(steps int, b *testing.B) {
 		b.Fatal(err)
 	}
 
-	led, err := complete.NewLedger(dir, steps+1, &metrics.NoopCollector{}, nil)
+	led, err := complete.NewLedger(dir, steps+1, &metrics.NoopCollector{}, zerolog.Logger{}, nil)
 	defer led.Done()
 	if err != nil {
 		b.Fatal("can't create a new complete ledger")