perf: optimize zipmap iter performance

command_test.go

@@ -316,7 +316,7 @@ func TestCommand(t *testing.T) {
 
 	t.Run("concurrency", func(t *testing.T) {
 		var wg sync.WaitGroup
-		for i := 0; i < 10000; i++ {
+		for i := 0; i < 2000; i++ {
 			wg.Add(1)
 			go func() {
 				key := fmt.Sprintf("key-%08x", rand.Int())

go.mod

@@ -11,7 +11,8 @@ require (
 	github.com/rs/zerolog v1.33.0
 	github.com/stretchr/testify v1.9.0
 	github.com/tidwall/mmap v0.3.0
-	golang.org/x/sys v0.22.0
+	github.com/zeebo/xxh3 v1.0.2
+	golang.org/x/sys v0.23.0
 )
 
 require (
@@ -20,6 +21,7 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/edsrzf/mmap-go v1.1.0 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect

go.sum

@@ -26,6 +26,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY=
 github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y=
+github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
+github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -54,15 +56,20 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tidwall/mmap v0.3.0 h1:XXt1YsiXCF5/UAu3pLbu6g7iulJ9jsbs6vt7UpiV0sY=
 github.com/tidwall/mmap v0.3.0/go.mod h1:2/dNzF5zA+te/JVHfrqNLcRkb8LjdH3c80vYHFQEZRk=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
+golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
 gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=

internal/hash/zipmap.go

@@ -5,46 +5,57 @@ import (
 	"unsafe"
 
 	"github.com/xgzlucario/rotom/internal/list"
+	"github.com/zeebo/xxh3"
 )
 
 var _ MapI = (*ZipMap)(nil)
 
 // ZipMap store datas as [entry1, entry2, entry3...] in listpack.
 type ZipMap struct {
-	m *list.ListPack
+	data *list.ListPack
 }
 
-func NewZipMap() *ZipMap {
-	return &ZipMap{list.NewListPack()}
-}
-
-// encodeEntry encode data to [vlen, val, key].
-func encodeEntry(key string, val []byte) []byte {
-	buf := make([]byte, 0, len(key)+len(val)+1)
+// zipmapEntry is data format in zipmap.
+/*
+	entry format:
+	+-----------------+-----+-----+--------------+
+	| val_len(varint) | val | key | hash(1 Byte) |
+	+-----------------+-----+-----+--------------+
+*/
+type zipmapEntry struct{}
+
+func (zipmapEntry) encode(key string, val []byte) []byte {
+	buf := make([]byte, len(key)+len(val)+2)[:0]
 	buf = binary.AppendUvarint(buf, uint64(len(val)))
 	buf = append(buf, val...)
-	return append(buf, key...)
+	buf = append(buf, key...)
+	buf = append(buf, byte(xxh3.HashString(key)))
+	return buf
 }
 
-func decodeEntry(buf []byte) (key string, val []byte) {
+func (zipmapEntry) decode(buf []byte) (key string, val []byte) {
 	vlen, n := binary.Uvarint(buf)
 	val = buf[n : n+int(vlen)]
-	key = b2s(buf[n+int(vlen):])
+	key = b2s(buf[n+int(vlen) : len(buf)-1])
 	return
 }
 
-func (zm *ZipMap) seek(key string) (*list.LpIterator, []byte) {
-	it := zm.m.Iterator().SeekLast()
-	b := key[len(key)-1]
+func NewZipMap() *ZipMap {
+	return &ZipMap{list.NewListPack()}
+}
+
+func (zm *ZipMap) find(key string) (it *list.LpIterator, val []byte) {
+	it = zm.data.Iterator().SeekLast()
+	hash := byte(xxh3.HashString(key))
 
 	for !it.IsFirst() {
 		entry := it.Prev()
-		// fast equal
-		if entry[len(entry)-1] != b {
+
+		if entry[len(entry)-1] != hash {
 			continue
 		}
 
-		kb, vb := decodeEntry(entry)
+		kb, vb := zipmapEntry{}.decode(entry)
 		if key == kb {
 			return it, vb
 		}
@@ -53,26 +64,26 @@ func (zm *ZipMap) seek(key string) (*list.LpIterator, []byte) {
 }
 
 func (zm *ZipMap) Set(key string, val []byte) (newField bool) {
-	entry := b2s(encodeEntry(key, val))
-	it, _ := zm.seek(key)
+	entry := zipmapEntry{}.encode(key, val)
+	it, _ := zm.find(key)
 	if it != nil {
-		it.ReplaceNext(entry)
+		it.ReplaceNext(b2s(entry))
 		return false
 	}
-	zm.m.RPush(entry)
+	zm.data.RPush(b2s(entry))
 	return true
 }
 
 func (zm *ZipMap) Get(key string) ([]byte, bool) {
-	_, val := zm.seek(key)
+	_, val := zm.find(key)
 	if val != nil {
 		return val, true
 	}
 	return nil, false
 }
 
 func (zm *ZipMap) Remove(key string) bool {
-	it, _ := zm.seek(key)
+	it, _ := zm.find(key)
 	if it != nil {
 		it.RemoveNexts(1, nil)
 		return true
@@ -81,9 +92,9 @@ func (zm *ZipMap) Remove(key string) bool {
 }
 
 func (zm *ZipMap) Scan(fn func(string, []byte)) {
-	it := zm.m.Iterator().SeekLast()
+	it := zm.data.Iterator().SeekLast()
 	for !it.IsFirst() {
-		kb, vb := decodeEntry(it.Prev())
+		kb, vb := zipmapEntry{}.decode(it.Prev())
 		fn(kb, vb)
 	}
 }
@@ -96,7 +107,7 @@ func (zm *ZipMap) ToMap() *Map {
 	return m
 }
 
-func (zm *ZipMap) Len() int { return zm.m.Size() }
+func (zm *ZipMap) Len() int { return zm.data.Size() }
 
 func b2s(b []byte) string {
 	return *(*string)(unsafe.Pointer(&b))

internal/hash/zipset.go

@@ -2,13 +2,34 @@ package hash
 
 import (
 	"github.com/xgzlucario/rotom/internal/list"
+	"github.com/zeebo/xxh3"
 )
 
 var _ SetI = (*ZipSet)(nil)
 
 // ZipSet store datas as [key1, key2, key3...] in listpack.
 type ZipSet struct {
-	m *list.ListPack
+	data *list.ListPack
+}
+
+// zipsetEntry is data format in zipset.
+/*
+	entry format:
+	+-----+--------------+
+	| key | hash(1 Byte) |
+	+-----+--------------+
+*/
+type zipsetEntry struct{}
+
+func (zipsetEntry) encode(key string) []byte {
+	buf := make([]byte, len(key)+1)[:0]
+	buf = append(buf, key...)
+	buf = append(buf, byte(xxh3.HashString(key)))
+	return buf
+}
+
+func (zipsetEntry) decode(buf []byte) (key string) {
+	return b2s(buf[:len(buf)-1])
 }
 
 func NewZipSet() *ZipSet {
@@ -19,24 +40,39 @@ func (zs *ZipSet) Add(key string) (newField bool) {
 	if zs.Exist(key) {
 		return false
 	}
-	zs.m.RPush(key)
+	entry := zipsetEntry{}.encode(key)
+	zs.data.RPush(b2s(entry))
 	return true
 }
 
 func (zs *ZipSet) Exist(key string) bool {
-	it := zs.m.Iterator().SeekLast()
+	it := zs.data.Iterator().SeekLast()
+	hash := byte(xxh3.HashString(key))
+
 	for !it.IsFirst() {
-		if key == b2s(it.Prev()) {
+		entry := it.Prev()
+		if entry[len(entry)-1] != hash {
+			continue
+		}
+		kb := zipsetEntry{}.decode(entry)
+		if key == kb {
 			return true
 		}
 	}
 	return false
 }
 
 func (zs *ZipSet) Remove(key string) bool {
-	it := zs.m.Iterator().SeekLast()
+	it := zs.data.Iterator().SeekLast()
+	hash := byte(xxh3.HashString(key))
+
 	for !it.IsFirst() {
-		if key == b2s(it.Prev()) {
+		entry := it.Prev()
+		if entry[len(entry)-1] != hash {
+			continue
+		}
+		kb := zipsetEntry{}.decode(entry)
+		if key == kb {
 			it.RemoveNexts(1, nil)
 			return true
 		}
@@ -45,15 +81,23 @@ func (zs *ZipSet) Remove(key string) bool {
 }
 
 func (zs *ZipSet) Scan(fn func(string)) {
-	it := zs.m.Iterator().SeekLast()
+	it := zs.data.Iterator().SeekLast()
 	for !it.IsFirst() {
-		fn(b2s(it.Prev()))
+		entry := it.Prev()
+		key := zipsetEntry{}.decode(entry)
+		fn(key)
 	}
 }
 
-func (zs *ZipSet) Pop() (string, bool) { return zs.m.RPop() }
+func (zs *ZipSet) Pop() (string, bool) {
+	key, ok := zs.data.RPop()
+	if ok {
+		return key[:len(key)-1], true
+	}
+	return "", false
+}
 
-func (zs *ZipSet) Len() int { return zs.m.Size() }
+func (zs *ZipSet) Len() int { return zs.data.Size() }
 
 func (zs *ZipSet) ToSet() *Set {
 	s := NewSet()

internal/list/list.go

@@ -1,7 +1,5 @@
 package list
 
-import "math"
-
 //	 +------------------------------ QuickList -----------------------------+
 //	 |	     +-----------+     +-----------+             +-----------+      |
 //	head --- | listpack0 | <-> | listpack1 | <-> ... <-> | listpackN | --- tail
@@ -87,15 +85,10 @@ func (ls *QuickList) free(n *Node) {
 	}
 }
 
-func (ls *QuickList) Size() int {
-	return ls.size
-}
+func (ls *QuickList) Size() int { return ls.size }
 
-func (ls *QuickList) Range(start, end int, f func(data []byte)) {
-	if end == -1 {
-		end = math.MaxInt
-	}
-	count := end - start
+func (ls *QuickList) Range(start, stop int, f func(data []byte)) {
+	count := stop - start
 
 	lp := ls.head
 	for lp != nil && start > lp.Size() {
@@ -120,11 +113,8 @@ func (ls *QuickList) Range(start, end int, f func(data []byte)) {
 	}
 }
 
-func (ls *QuickList) RevRange(start, end int, f func(data []byte)) {
-	if end == -1 {
-		end = math.MaxInt
-	}
-	count := end - start
+func (ls *QuickList) RevRange(start, stop int, f func(data []byte)) {
+	count := stop - start
 
 	lp := ls.tail
 	for lp != nil && start > lp.Size() {

internal/list/list_test.go

@@ -7,16 +7,16 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func genList(start, end int) *QuickList {
+func genList(start, stop int) *QuickList {
 	lp := New()
-	for i := start; i < end; i++ {
+	for i := start; i < stop; i++ {
 		lp.RPush(genKey(i))
 	}
 	return lp
 }
 
 func list2slice(ls *QuickList) (res []string) {
-	ls.Range(0, -1, func(data []byte) {
+	ls.Range(0, ls.Size(), func(data []byte) {
 		res = append(res, string(data))
 	})
 	return
@@ -81,7 +81,7 @@ func TestList(t *testing.T) {
 	t.Run("range", func(t *testing.T) {
 		ls := genList(0, N)
 		i := 0
-		ls.Range(0, -1, func(data []byte) {
+		ls.Range(0, N, func(data []byte) {
 			assert.Equal(string(data), genKey(i))
 			i++
 		})
@@ -100,7 +100,7 @@ func TestList(t *testing.T) {
 	t.Run("revrange", func(t *testing.T) {
 		ls := genList(0, N)
 		i := 0
-		ls.RevRange(0, -1, func(data []byte) {
+		ls.RevRange(0, N, func(data []byte) {
 			assert.Equal(string(data), genKey(N-i-1))
 			i++
 		})