perf: optimise listpack perf

go.mod

@@ -4,6 +4,7 @@ go 1.22
 
 require (
 	github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964
+	github.com/klauspost/compress v1.17.9
 	github.com/redis/go-redis/v9 v9.5.2
 	github.com/rs/zerolog v1.33.0
 	github.com/sakeven/RbTree v1.1.1

go.sum

@@ -17,6 +17,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu
 github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=

internal/list/bench_test.go

@@ -17,36 +17,32 @@ func BenchmarkList(b *testing.B) {
 			ls.RPush(genKey(i))
 		}
 	})
-}
-
-func BenchmarkListPack(b *testing.B) {
-	const N = 1000
-	b.Run("next", func(b *testing.B) {
-		ls := genListPack(0, N)
-		it := ls.NewIterator()
+	b.Run("lpop", func(b *testing.B) {
+		ls := genList(0, b.N)
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			it.SeekBegin().Next()
+			ls.LPop()
 		}
 	})
-	b.Run("prev", func(b *testing.B) {
-		ls := genListPack(0, N)
-		it := ls.NewIterator()
+	b.Run("rpop", func(b *testing.B) {
+		ls := genList(0, b.N)
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			it.SeekEnd().Prev()
+			ls.RPop()
 		}
 	})
-	b.Run("lpush", func(b *testing.B) {
-		lp := NewListPack()
-		for i := 0; i < 10*10000; i++ {
-			lp.LPush("A")
+	b.Run("range", func(b *testing.B) {
+		ls := genList(0, 1000)
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			ls.Range(0, -1, func([]byte) {})
 		}
 	})
-	b.Run("rpush", func(b *testing.B) {
-		lp := NewListPack()
-		for i := 0; i < 10*10000; i++ {
-			lp.RPush("A")
+	b.Run("revrange", func(b *testing.B) {
+		ls := genList(0, 1000)
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			ls.RevRange(0, -1, func([]byte) {})
 		}
 	})
 }

internal/list/list.go

@@ -93,26 +93,24 @@ func (ls *QuickList) Size() (n int) {
 	return
 }
 
-type lsIterator func(data []byte)
-
-func (ls *QuickList) Range(start, end int, f lsIterator) {
+func (ls *QuickList) Range(start, end int, f func(data []byte)) {
 	if end == -1 {
 		end = math.MaxInt
 	}
 	for lp := ls.head; lp != nil; lp = lp.next {
-		it := lp.NewIterator().SeekBegin()
+		it := lp.Iterator().SeekBegin()
 		for !it.IsEnd() {
 			f(it.Next())
 		}
 	}
 }
 
-func (ls *QuickList) RevRange(start, end int, f lsIterator) {
+func (ls *QuickList) RevRange(start, end int, f func(data []byte)) {
 	if end == -1 {
 		end = math.MaxInt
 	}
 	for lp := ls.tail; lp != nil; lp = lp.prev {
-		it := lp.NewIterator().SeekEnd()
+		it := lp.Iterator().SeekEnd()
 		for !it.IsBegin() {
 			f(it.Prev())
 		}

internal/list/list_test.go

@@ -86,6 +86,7 @@ func TestList(t *testing.T) {
 			assert.Equal(string(data), genKey(i))
 			i++
 		})
+		assert.Equal(i, N)
 	})
 
 	t.Run("revrange", func(t *testing.T) {
@@ -95,5 +96,6 @@ func TestList(t *testing.T) {
 			assert.Equal(string(data), genKey(N-i-1))
 			i++
 		})
+		assert.Equal(i, N)
 	})
 }

internal/list/listpack.go

@@ -4,13 +4,17 @@ import (
 	"encoding/binary"
 	"slices"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/xgzlucario/rotom/internal/pkg"
 )
 
 var (
 	maxListPackSize = 8 * 1024
 
 	bpool = pkg.NewBufferPool()
+
+	encoder, _ = zstd.NewWriter(nil)
+	decoder, _ = zstd.NewReader(nil)
 )
 
 // ListPack is a lists of strings serialization format on Redis.
@@ -30,8 +34,9 @@ var (
 	Using this structure, it is fast to iterate from both sides.
 */
 type ListPack struct {
-	size uint16
-	data []byte
+	compress bool
+	size     uint16
+	data     []byte
 }
 
 func NewListPack() *ListPack {
@@ -43,27 +48,41 @@ func (lp *ListPack) Size() int {
 }
 
 func (lp *ListPack) LPush(data ...string) {
-	lp.NewIterator().Insert(data...)
+	lp.Iterator().Insert(data...)
 }
 
 func (lp *ListPack) RPush(data ...string) {
-	lp.NewIterator().SeekEnd().Insert(data...)
+	lp.Iterator().SeekEnd().Insert(data...)
 }
 
 func (lp *ListPack) LPop() (string, bool) {
-	return lp.NewIterator().RemoveNext()
+	return lp.Iterator().RemoveNext()
 }
 
 func (lp *ListPack) RPop() (string, bool) {
-	return lp.NewIterator().SeekEnd().RemovePrev()
+	return lp.Iterator().SeekEnd().RemovePrev()
+}
+
+func (lp *ListPack) Compress() {
+	if !lp.compress {
+		lp.data = encoder.EncodeAll(lp.data, make([]byte, 0, len(lp.data)/4))
+		lp.compress = true
+	}
+}
+
+func (lp *ListPack) Decompress() {
+	if lp.compress {
+		lp.data, _ = decoder.DecodeAll(lp.data, nil)
+		lp.compress = false
+	}
 }
 
 type lpIterator struct {
 	*ListPack
 	index int
 }
 
-func (lp *ListPack) NewIterator() *lpIterator {
+func (lp *ListPack) Iterator() *lpIterator {
 	return &lpIterator{ListPack: lp}
 }
 
@@ -126,6 +145,13 @@ func (it *lpIterator) Prev() []byte {
 }
 
 func (it *lpIterator) Insert(datas ...string) {
+	if it.IsEnd() {
+		for _, data := range datas {
+			it.data = appendEntry(it.data, data)
+			it.size++
+		}
+		return
+	}
 	var alloc []byte
 	for _, data := range datas {
 		alloc = appendEntry(alloc, data)
@@ -140,10 +166,10 @@ func (it *lpIterator) RemoveNext() (string, bool) {
 		return "", false
 	}
 	before := it.index
-	data := string(it.Next()) // seek to next
+	data := string(it.Next())
 	after := it.index
 	it.data = slices.Delete(it.data, before, after)
-	it.index = before // back to prev
+	it.index = min(before, after)
 	it.size--
 	return data, true
 }
@@ -153,7 +179,7 @@ func (it *lpIterator) RemovePrev() (string, bool) {
 		return "", false
 	}
 	before := it.index
-	data := string(it.Prev()) // seek to prev
+	data := string(it.Prev())
 	after := it.index
 	it.data = slices.Delete(it.data, after, before)
 	it.size--

internal/list/listpack_test.go

@@ -7,20 +7,12 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func genListPack(start, end int) *ListPack {
-	lp := NewListPack()
-	for i := start; i < end; i++ {
-		lp.RPush(genKey(i))
-	}
-	return lp
-}
-
 func genKey(i int) string {
 	return fmt.Sprintf("%06x", i)
 }
 
 func lp2list(lp *ListPack) (res []string) {
-	it := lp.NewIterator()
+	it := lp.Iterator()
 	for !it.IsEnd() {
 		res = append(res, string(it.Next()))
 	}
@@ -89,4 +81,12 @@ func TestListpack(t *testing.T) {
 		assert.Equal(val, "")
 		assert.False(ok)
 	})
+
+	t.Run("compress", func(t *testing.T) {
+		lp := NewListPack()
+		lp.LPush("A", "B", "C", "D", "E")
+		lp.Compress()
+		lp.Decompress()
+		assert.Equal(lp2list(lp), []string{"A", "B", "C", "D", "E"})
+	})
 }