Skip to content

Commit

Permalink
perf: optimize zipmap iter performance
Browse files Browse the repository at this point in the history
  • Loading branch information
satoshi-099 committed Aug 5, 2024
1 parent 5c7a712 commit 0b20fb1
Show file tree
Hide file tree
Showing 11 changed files with 144 additions and 66 deletions.
2 changes: 1 addition & 1 deletion command_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ func TestCommand(t *testing.T) {

t.Run("concurrency", func(t *testing.T) {
var wg sync.WaitGroup
for i := 0; i < 10000; i++ {
for i := 0; i < 2000; i++ {
wg.Add(1)
go func() {
key := fmt.Sprintf("key-%08x", rand.Int())
Expand Down
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ require (
github.com/rs/zerolog v1.33.0
github.com/stretchr/testify v1.9.0
github.com/tidwall/mmap v0.3.0
golang.org/x/sys v0.22.0
github.com/zeebo/xxh3 v1.0.2
golang.org/x/sys v0.23.0
)

require (
Expand All @@ -20,6 +21,7 @@ require (
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
Expand Down
11 changes: 9 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY=
github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
Expand Down Expand Up @@ -54,15 +56,20 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tidwall/mmap v0.3.0 h1:XXt1YsiXCF5/UAu3pLbu6g7iulJ9jsbs6vt7UpiV0sY=
github.com/tidwall/mmap v0.3.0/go.mod h1:2/dNzF5zA+te/JVHfrqNLcRkb8LjdH3c80vYHFQEZRk=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
Expand Down
63 changes: 37 additions & 26 deletions internal/hash/zipmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,46 +5,57 @@ import (
"unsafe"

"github.com/xgzlucario/rotom/internal/list"
"github.com/zeebo/xxh3"
)

var _ MapI = (*ZipMap)(nil)

// ZipMap store datas as [entry1, entry2, entry3...] in listpack.
type ZipMap struct {
m *list.ListPack
data *list.ListPack
}

func NewZipMap() *ZipMap {
return &ZipMap{list.NewListPack()}
}

// encodeEntry encode data to [vlen, val, key].
func encodeEntry(key string, val []byte) []byte {
buf := make([]byte, 0, len(key)+len(val)+1)
// zipmapEntry is data format in zipmap.
/*
entry format:
+-----------------+-----+-----+--------------+
| val_len(varint) | val | key | hash(1 Byte) |
+-----------------+-----+-----+--------------+
*/
type zipmapEntry struct{}

func (zipmapEntry) encode(key string, val []byte) []byte {
buf := make([]byte, len(key)+len(val)+2)[:0]
buf = binary.AppendUvarint(buf, uint64(len(val)))
buf = append(buf, val...)
return append(buf, key...)
buf = append(buf, key...)
buf = append(buf, byte(xxh3.HashString(key)))
return buf
}

func decodeEntry(buf []byte) (key string, val []byte) {
func (zipmapEntry) decode(buf []byte) (key string, val []byte) {
vlen, n := binary.Uvarint(buf)
val = buf[n : n+int(vlen)]
key = b2s(buf[n+int(vlen):])
key = b2s(buf[n+int(vlen) : len(buf)-1])
return
}

func (zm *ZipMap) seek(key string) (*list.LpIterator, []byte) {
it := zm.m.Iterator().SeekLast()
b := key[len(key)-1]
func NewZipMap() *ZipMap {
return &ZipMap{list.NewListPack()}
}

func (zm *ZipMap) find(key string) (it *list.LpIterator, val []byte) {
it = zm.data.Iterator().SeekLast()
hash := byte(xxh3.HashString(key))

for !it.IsFirst() {
entry := it.Prev()
// fast equal
if entry[len(entry)-1] != b {

if entry[len(entry)-1] != hash {
continue
}

kb, vb := decodeEntry(entry)
kb, vb := zipmapEntry{}.decode(entry)
if key == kb {
return it, vb
}
Expand All @@ -53,26 +64,26 @@ func (zm *ZipMap) seek(key string) (*list.LpIterator, []byte) {
}

func (zm *ZipMap) Set(key string, val []byte) (newField bool) {
entry := b2s(encodeEntry(key, val))
it, _ := zm.seek(key)
entry := zipmapEntry{}.encode(key, val)
it, _ := zm.find(key)
if it != nil {
it.ReplaceNext(entry)
it.ReplaceNext(b2s(entry))
return false
}
zm.m.RPush(entry)
zm.data.RPush(b2s(entry))
return true
}

func (zm *ZipMap) Get(key string) ([]byte, bool) {
_, val := zm.seek(key)
_, val := zm.find(key)
if val != nil {
return val, true
}
return nil, false
}

func (zm *ZipMap) Remove(key string) bool {
it, _ := zm.seek(key)
it, _ := zm.find(key)
if it != nil {
it.RemoveNexts(1, nil)
return true
Expand All @@ -81,9 +92,9 @@ func (zm *ZipMap) Remove(key string) bool {
}

func (zm *ZipMap) Scan(fn func(string, []byte)) {
it := zm.m.Iterator().SeekLast()
it := zm.data.Iterator().SeekLast()
for !it.IsFirst() {
kb, vb := decodeEntry(it.Prev())
kb, vb := zipmapEntry{}.decode(it.Prev())
fn(kb, vb)
}
}
Expand All @@ -96,7 +107,7 @@ func (zm *ZipMap) ToMap() *Map {
return m
}

func (zm *ZipMap) Len() int { return zm.m.Size() }
func (zm *ZipMap) Len() int { return zm.data.Size() }

func b2s(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
Expand Down
64 changes: 54 additions & 10 deletions internal/hash/zipset.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,34 @@ package hash

import (
"github.com/xgzlucario/rotom/internal/list"
"github.com/zeebo/xxh3"
)

var _ SetI = (*ZipSet)(nil)

// ZipSet store datas as [key1, key2, key3...] in listpack.
type ZipSet struct {
m *list.ListPack
data *list.ListPack
}

// zipsetEntry is data format in zipset.
/*
entry format:
+-----+--------------+
| key | hash(1 Byte) |
+-----+--------------+
*/
type zipsetEntry struct{}

func (zipsetEntry) encode(key string) []byte {
buf := make([]byte, len(key)+1)[:0]
buf = append(buf, key...)
buf = append(buf, byte(xxh3.HashString(key)))
return buf
}

func (zipsetEntry) decode(buf []byte) (key string) {
return b2s(buf[:len(buf)-1])
}

func NewZipSet() *ZipSet {
Expand All @@ -19,24 +40,39 @@ func (zs *ZipSet) Add(key string) (newField bool) {
if zs.Exist(key) {
return false
}
zs.m.RPush(key)
entry := zipsetEntry{}.encode(key)
zs.data.RPush(b2s(entry))
return true
}

func (zs *ZipSet) Exist(key string) bool {
it := zs.m.Iterator().SeekLast()
it := zs.data.Iterator().SeekLast()
hash := byte(xxh3.HashString(key))

for !it.IsFirst() {
if key == b2s(it.Prev()) {
entry := it.Prev()
if entry[len(entry)-1] != hash {
continue
}
kb := zipsetEntry{}.decode(entry)
if key == kb {
return true
}
}
return false
}

func (zs *ZipSet) Remove(key string) bool {
it := zs.m.Iterator().SeekLast()
it := zs.data.Iterator().SeekLast()
hash := byte(xxh3.HashString(key))

for !it.IsFirst() {
if key == b2s(it.Prev()) {
entry := it.Prev()
if entry[len(entry)-1] != hash {
continue
}
kb := zipsetEntry{}.decode(entry)
if key == kb {
it.RemoveNexts(1, nil)
return true
}
Expand All @@ -45,15 +81,23 @@ func (zs *ZipSet) Remove(key string) bool {
}

func (zs *ZipSet) Scan(fn func(string)) {
it := zs.m.Iterator().SeekLast()
it := zs.data.Iterator().SeekLast()
for !it.IsFirst() {
fn(b2s(it.Prev()))
entry := it.Prev()
key := zipsetEntry{}.decode(entry)
fn(key)
}
}

func (zs *ZipSet) Pop() (string, bool) { return zs.m.RPop() }
func (zs *ZipSet) Pop() (string, bool) {
key, ok := zs.data.RPop()
if ok {
return key[:len(key)-1], true
}
return "", false
}

func (zs *ZipSet) Len() int { return zs.m.Size() }
func (zs *ZipSet) Len() int { return zs.data.Size() }

func (zs *ZipSet) ToSet() *Set {
s := NewSet()
Expand Down
20 changes: 5 additions & 15 deletions internal/list/list.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package list

import "math"

// +------------------------------ QuickList -----------------------------+
// | +-----------+ +-----------+ +-----------+ |
// head --- | listpack0 | <-> | listpack1 | <-> ... <-> | listpackN | --- tail
Expand Down Expand Up @@ -87,15 +85,10 @@ func (ls *QuickList) free(n *Node) {
}
}

func (ls *QuickList) Size() int {
return ls.size
}
func (ls *QuickList) Size() int { return ls.size }

func (ls *QuickList) Range(start, end int, f func(data []byte)) {
if end == -1 {
end = math.MaxInt
}
count := end - start
func (ls *QuickList) Range(start, stop int, f func(data []byte)) {
count := stop - start

lp := ls.head
for lp != nil && start > lp.Size() {
Expand All @@ -120,11 +113,8 @@ func (ls *QuickList) Range(start, end int, f func(data []byte)) {
}
}

func (ls *QuickList) RevRange(start, end int, f func(data []byte)) {
if end == -1 {
end = math.MaxInt
}
count := end - start
func (ls *QuickList) RevRange(start, stop int, f func(data []byte)) {
count := stop - start

lp := ls.tail
for lp != nil && start > lp.Size() {
Expand Down
10 changes: 5 additions & 5 deletions internal/list/list_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@ import (
"github.com/stretchr/testify/assert"
)

func genList(start, end int) *QuickList {
func genList(start, stop int) *QuickList {
lp := New()
for i := start; i < end; i++ {
for i := start; i < stop; i++ {
lp.RPush(genKey(i))
}
return lp
}

func list2slice(ls *QuickList) (res []string) {
ls.Range(0, -1, func(data []byte) {
ls.Range(0, ls.Size(), func(data []byte) {
res = append(res, string(data))
})
return
Expand Down Expand Up @@ -81,7 +81,7 @@ func TestList(t *testing.T) {
t.Run("range", func(t *testing.T) {
ls := genList(0, N)
i := 0
ls.Range(0, -1, func(data []byte) {
ls.Range(0, N, func(data []byte) {
assert.Equal(string(data), genKey(i))
i++
})
Expand All @@ -100,7 +100,7 @@ func TestList(t *testing.T) {
t.Run("revrange", func(t *testing.T) {
ls := genList(0, N)
i := 0
ls.RevRange(0, -1, func(data []byte) {
ls.RevRange(0, N, func(data []byte) {
assert.Equal(string(data), genKey(N-i-1))
i++
})
Expand Down
Loading

0 comments on commit 0b20fb1

Please sign in to comment.