radixdb: scaffold a proper blobStore mechanism

blob.go

@@ -0,0 +1,73 @@
+package radixdb
+
+import "crypto/sha256"
+
+const (
+	// Length of the record value hash in bytes.
+	blobIDLen = 32
+)
+
+// blobID is a 32-byte fixed length byte array representing the SHA-256 hash of
+// a record value. It is an array instead of a slice for map key compatibility.
+type blobID [blobIDLen]byte
+
+// blobStoreEntry represents a value and its reference count within the
+// blobStore. The value field stores the actual value data, and refCount
+// tracks the number of active references to the value.
+type blobStoreEntry struct {
+	value    []byte
+	refCount int
+}
+
+// blobStore maps blobIDs to their corresponding byte slices. This type is used
+// to store values that exceed the 32-byte length threshold.
+type blobStore map[blobID]*blobStoreEntry
+
+// buildBlobID builds a blobID from the given byte slice. It requires that the
+// given byte slice length matches the blobID length (32-bytes).
+func buildBlobID(src []byte) (blobID, error) {
+	var ret blobID
+
+	if len(src) != blobIDLen {
+		return ret, ErrInvalidBlobID
+	}
+
+	copy(ret[:], src)
+
+	return ret, nil
+}
+
+// toSlice returns the given blobID as a byte slice.
+func (id blobID) toSlice() []byte {
+	return id[:]
+}
+
+// getValue returns the blob value that matches the given blobID.
+func (b blobStore) getValue(id blobID) []byte {
+	blob, found := b[id]
+
+	if !found {
+		return nil
+	}
+
+	// Create a copy of the value since returning a pointer to the underlying
+	// value can have serious implications, such as breaking data integrity.
+	ret := make([]byte, len(blob.value))
+	copy(ret, blob.value)
+
+	return ret
+}
+
+// put adds a new value to the blobStore or increments the reference count
+// of an existing value. It returns the blobID of the stored value.
+func (b blobStore) put(value []byte) blobID {
+	k := blobID(sha256.Sum256(value))
+
+	if entry, found := b[k]; found {
+		entry.refCount++
+	} else {
+		b[k] = &blobStoreEntry{value: value, refCount: 1}
+	}
+
+	return k
+}

blob_test.go

@@ -0,0 +1,39 @@
+package radixdb
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"testing"
+)
+
+func TestBlobStorePut(t *testing.T) {
+	store := blobStore{}
+
+	tests := []struct {
+		value            []byte
+		expectedBlobID   blobID
+		expectedRefCount int
+	}{
+		{[]byte("apple"), sha256.Sum256([]byte("apple")), 1},
+		{[]byte("apple"), sha256.Sum256([]byte("apple")), 2},
+		{[]byte("apple"), sha256.Sum256([]byte("apple")), 3},
+	}
+
+	for _, test := range tests {
+		blobID := store.put(test.value)
+
+		if !bytes.Equal(blobID.toSlice(), test.expectedBlobID.toSlice()) {
+			t.Errorf("unexpected blobID, got:%q, want:%q", blobID, test.expectedBlobID)
+		}
+
+		value := store.getValue(blobID)
+
+		if !bytes.Equal(value, test.value) {
+			t.Errorf("unexpected blob, got:%q, want:%q", value, test.value)
+		}
+
+		if got := store[blobID].refCount; got != test.expectedRefCount {
+			t.Errorf("unexpected refCount, got:%d, want:%d", got, test.expectedRefCount)
+		}
+	}
+}

debug.go

@@ -20,8 +20,8 @@ func printTree(node *node, prefix string, isLast bool, isRoot bool, treeSize uin
 
 	var val string
 
-	if node.value != nil {
-		val = string(node.value)
+	if node.data != nil {
+		val = string(node.data)
 	} else {
 		val = "<nil>"
 	}

node.go

@@ -2,7 +2,6 @@ package radixdb
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"encoding/binary"
 	"hash/crc32"
 	"sort"
@@ -12,13 +11,19 @@ import (
 // is designed to be memory-efficient by using a minimal set of fields to
 // represent each node. In a Radix tree, the node's key inherently carries
 // significant information, hence reducing the need to maintain metadata.
+// Adding fields to this struct can significantly increase memory overhead.
+// Think carefully before adding anything to the struct.
 type node struct {
 	key      []byte  // Path segment of the node.
-	value    []byte  // Data associated with this node, if any.
 	isRecord bool    // True if node is a record; false if path component.
 	isBlob   bool    // True if the value is stored in the blob store.
 	children []*node // Pointers to child nodes.
 	checksum uint32  // CRC32 checksum of the node content.
+
+	// Holds the content of the node. Values less than or equal to 32-bytes
+	// are stored directly in this byte slice. Otherwise, it holds the blobID
+	// that points to the content in the blobStore.
+	data []byte
 }
 
 // hasChidren returns true if the receiver node has children.
@@ -31,6 +36,24 @@ func (n node) isLeaf() bool {
 	return len(n.children) == 0
 }
 
+// value retrieves the record value of the node. If the value is stored in the
+// blobStore, it fetches the value using the blobID stored in the data field.
+func (n node) value(blobs blobStore) []byte {
+	ret := n.data
+
+	if n.isBlob {
+		blobID, err := buildBlobID(n.data)
+
+		if err != nil {
+			return nil
+		}
+
+		ret = blobs.getValue(blobID)
+	}
+
+	return ret
+}
+
 // findCompatibleChild searches through the child nodes of the receiver node.
 // It returns the first child node that shares a common prefix. If no child is
 // found, the function returns nil.
@@ -107,7 +130,7 @@ func (n node) calculateChecksum() (uint32, error) {
 		return 0, err
 	}
 
-	if _, err := h.Write(n.value); err != nil {
+	if _, err := h.Write(n.data); err != nil {
 		return 0, err
 	}
 
@@ -163,7 +186,7 @@ func (n node) verifyChecksum() bool {
 // is intended for cases where sustaining the receiver's address is necessary.
 func (n *node) shallowCopyFrom(src *node) {
 	n.key = src.key
-	n.value = src.value
+	n.data = src.data
 	n.isBlob = src.isBlob
 	n.isRecord = src.isRecord
 	n.children = src.children
@@ -181,13 +204,12 @@ func (n *node) setKey(key []byte) {
 // setValue sets the given value to the node.
 func (n *node) setValue(blobs blobStore, value []byte) {
 	if len(value) <= inlineValueThreshold {
-		n.value = value
+		n.data = value
 		n.isBlob = false
 	} else {
-		k := blobID(sha256.Sum256(value))
-		n.value = k.toSlice()
+		id := blobs.put(value)
+		n.data = id.toSlice()
 		n.isBlob = true
-		blobs[k] = value
 	}
 }
 
@@ -233,14 +255,14 @@ func (n node) serialize() ([]byte, error) {
 
 	// Step 3: Serialize the value and its length, if the node holds a record.
 	if n.isRecord {
-		valLen := uint64(len(n.value))
+		valLen := uint64(len(n.data))
 
 		if err := binary.Write(&buf, binary.LittleEndian, valLen); err != nil {
 			return nil, err
 		}
 
 		if valLen > 0 {
-			if _, err := buf.Write(n.value); err != nil {
+			if _, err := buf.Write(n.data); err != nil {
 				return nil, err
 			}
 		}

node_test.go

@@ -262,7 +262,7 @@ func TestRemoveChild(t *testing.T) {
 func TestUpdateChecksum(t *testing.T) {
 	n := &node{
 		key:      []byte("apple"),
-		value:    []byte("sauce"),
+		data:     []byte("sauce"),
 		isRecord: true,
 	}
 
@@ -334,21 +334,21 @@ func TestSetValue(t *testing.T) {
 
 		// Test that the blobID is stored in the value slice.
 		if test.isBlob {
-			blobID, err := buildBlobID(n.value)
+			blobID, err := buildBlobID(n.data)
 
 			if err != nil {
 				t.Errorf("failed to buildBlobID: %v", err)
 			}
 
-			val, found := rdb.blobs[blobID]
+			blob, found := rdb.blobs[blobID]
 
 			if !found {
 				t.Error("cound not find blob")
 				return
 			}
 
-			if !bytes.Equal(val, test.value) {
-				t.Errorf("value mismatch, got:%q, want:%q", val, test.value)
+			if !bytes.Equal(blob.value, test.value) {
+				t.Errorf("value mismatch, got:%q, want:%q", blob.value, test.value)
 			}
 		}
 	}
@@ -357,7 +357,7 @@ func TestSetValue(t *testing.T) {
 func TestSerialize(t *testing.T) {
 	subject := node{
 		key:      []byte("apple"),
-		value:    []byte("sauce"),
+		data:     []byte("sauce"),
 		isRecord: true,
 		children: nil,
 	}
@@ -415,7 +415,7 @@ func TestSerialize(t *testing.T) {
 		t.Fatalf("failed to read value length: %v", err)
 	}
 
-	if want := uint64(len(subject.value)); want != valLen {
+	if want := uint64(len(subject.data)); want != valLen {
 		t.Errorf("unexpected value length, got:%d, want:%d", valLen, want)
 	}
 
@@ -425,8 +425,8 @@ func TestSerialize(t *testing.T) {
 		t.Fatalf("failed to read value data: %v", err)
 	}
 
-	if !bytes.Equal(valData, subject.value) {
-		t.Errorf("unexpected value data, got:%q, want:%q", valData, subject.value)
+	if !bytes.Equal(valData, subject.data) {
+		t.Errorf("unexpected value data, got:%q, want:%q", valData, subject.data)
 	}
 
 	// Reconstruct the child count.
@@ -451,7 +451,7 @@ func TestSerialize(t *testing.T) {
 	{
 		subject := node{
 			key:      []byte("banana"),
-			value:    []byte("smoothie"),
+			data:     []byte("smoothie"),
 			isRecord: true,
 			children: nil,
 		}

radixdb.go

@@ -26,20 +26,9 @@ var (
 )
 
 const (
-	// Length of the record value hash in bytes.
-	blobIDLen = 32
-
 	inlineValueThreshold = blobIDLen
 )
 
-// blobID is a 32-byte fixed length byte array representing the SHA-256 hash of
-// a record value. It is an array instead of a slice for map key compatibility.
-type blobID [blobIDLen]byte
-
-// blobStore maps blobIDs to their corresponding byte slices. This type is used
-// to store values that exceed the 32-byte length threshold.
-type blobStore map[blobID][]byte
-
 // RadixDB represents an in-memory Radix tree, providing concurrency-safe read
 // and write APIs. It maintains a reference to the root node and tracks various
 // metadata such as the total number of nodes.
@@ -58,7 +47,7 @@ type RadixDB struct {
 // New initializes and returns a new instance of RadixDB.
 func New() *RadixDB {
 	ret := &RadixDB{
-		blobs: map[blobID][]byte{},
+		blobs: map[blobID]*blobStoreEntry{},
 	}
 
 	ret.initFileHeader()
@@ -237,7 +226,7 @@ func (rdb *RadixDB) Get(key []byte) ([]byte, error) {
 		return nil, ErrInvalidChecksum
 	}
 
-	return node.value, nil
+	return node.value(rdb.blobs), nil
 }
 
 // Delete removes the node that matches the given key.
@@ -334,18 +323,12 @@ func (rdb *RadixDB) Delete(key []byte) error {
 	// children are guaranteed to share the node's key as their prefix, we
 	// can simply convert the node to a path compression node.
 	if node.isBlob {
-		blobID, err := buildBlobID(node.value)
-
-		if err != nil {
-			return err
-		}
-
-		delete(rdb.blobs, blobID)
+		// TODO(toru): Implement blobStore.release() and call it here.
 	}
 
 	node.isBlob = false
 	node.isRecord = false
-	node.value = nil
+	node.data = nil
 	rdb.numRecords--
 
 	return nil
@@ -518,22 +501,3 @@ func (rdb *RadixDB) traverse(cb func(*node) error) error {
 func (rdb *RadixDB) initFileHeader() {
 	rdb.header = newFileHeader()
 }
-
-// toSlice returns the given blobID as a byte slice.
-func (id blobID) toSlice() []byte {
-	return id[:]
-}
-
-// buildBlobID builds a blobID from the given byte slice. It requires that the
-// given byte slice length matches the blobID length (32-bytes).
-func buildBlobID(src []byte) (blobID, error) {
-	var ret blobID
-
-	if len(src) != blobIDLen {
-		return ret, ErrInvalidBlobID
-	}
-
-	copy(ret[:], src)
-
-	return ret, nil
-}