Skip to content

Commit

Permalink
perf: optimize update inplaced in zipmap
Browse files Browse the repository at this point in the history
  • Loading branch information
xgzlucario committed Aug 6, 2024
1 parent e72922a commit cc1df88
Show file tree
Hide file tree
Showing 9 changed files with 47 additions and 29 deletions.
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ go 1.22

require (
github.com/chen3feng/stl4go v0.1.1
github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964
github.com/deckarep/golang-set/v2 v2.6.0
github.com/dolthub/swiss v0.2.1
github.com/influxdata/tdigest v0.0.1
github.com/redis/go-redis/v9 v9.5.2
github.com/rs/zerolog v1.33.0
Expand All @@ -19,6 +19,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dolthub/maphash v0.1.0 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo=
github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
Expand All @@ -8,8 +6,6 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q=
github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU=
github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 h1:Ew0znI2JatzKy52N1iS5muUsHkf2UJuhocH7uFW7jjs=
github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
Expand All @@ -18,6 +14,10 @@ github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80N
github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
github.com/dolthub/swiss v0.2.1 h1:gs2osYs5SJkAaH5/ggVJqXQxRXtWshF6uE0lgR/Y3Gw=
github.com/dolthub/swiss v0.2.1/go.mod h1:8AhKZZ1HK7g18j7v7k6c5cYIGEZJcPn0ARsai8cUrh0=
github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
Expand Down
4 changes: 2 additions & 2 deletions internal/dict/benchmark/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"runtime/debug"
"time"

"github.com/cockroachdb/swiss"
"github.com/dolthub/swiss"
"github.com/influxdata/tdigest"
"github.com/xgzlucario/rotom/internal/dict"
)
Expand Down Expand Up @@ -62,7 +62,7 @@ func main() {
}

case "swiss":
m := swiss.New[string, any](8)
m := swiss.NewMap[string, any](8)
for i := 0; i < entries; i++ {
k, v := genKV(i)
start := time.Now()
Expand Down
10 changes: 5 additions & 5 deletions internal/dict/dict.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"sync/atomic"
"time"

"github.com/cockroachdb/swiss"
"github.com/dolthub/swiss"
)

const (
Expand Down Expand Up @@ -40,8 +40,8 @@ type Dict struct {

func New() *Dict {
return &Dict{
data: swiss.New[string, *Object](64),
expire: swiss.New[string, int64](64),
data: swiss.NewMap[string, *Object](64),
expire: swiss.NewMap[string, int64](64),
}
}

Expand Down Expand Up @@ -124,12 +124,12 @@ func (dict *Dict) SetTTL(key string, ttl int64) int {

func (dict *Dict) EvictExpired() {
var count int
dict.expire.All(func(key string, nsec int64) bool {
dict.expire.Iter(func(key string, nsec int64) bool {
if _nsec.Load() > nsec {
dict.expire.Delete(key)
dict.data.Delete(key)
}
count++
return count <= 20
return count > 20
})
}
22 changes: 10 additions & 12 deletions internal/hash/map.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package hash

import (
"github.com/cockroachdb/swiss"
"github.com/dolthub/swiss"
)

type MapI interface {
Expand All @@ -15,36 +15,34 @@ type MapI interface {
var _ MapI = (*Map)(nil)

type Map struct {
m *swiss.Map[string, []byte]
data *swiss.Map[string, []byte]
}

func NewMap() *Map {
return &Map{m: swiss.New[string, []byte](256)}
return &Map{swiss.NewMap[string, []byte](256)}
}

func (m *Map) Get(key string) ([]byte, bool) {
return m.m.Get(key)
return m.data.Get(key)
}

func (m *Map) Set(key string, val []byte) bool {
_, ok := m.m.Get(key)
m.m.Put(key, val)
_, ok := m.data.Get(key)
m.data.Put(key, val)
return !ok
}

func (m *Map) Remove(key string) bool {
_, ok := m.m.Get(key)
m.m.Delete(key)
return ok
return m.data.Delete(key)
}

func (m *Map) Len() int {
return m.m.Len()
return m.data.Count()
}

func (m *Map) Scan(fn func(key string, val []byte)) {
m.m.All(func(key string, val []byte) (next bool) {
m.data.Iter(func(key string, val []byte) (stop bool) {
fn(key, val)
return true
return false
})
}
7 changes: 6 additions & 1 deletion internal/hash/map_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,12 @@ func testMapI(m MapI, t *testing.T) {
var nilBytes []byte
assert.Equal(val, nilBytes)

// set(update)
// set(update great size val)
assert.False(m.Set("key1", []byte("newval7")))
assert.False(m.Set("key2", []byte("newval8")))
assert.False(m.Set("key3", []byte("newval9")))

// set(update equal size val)
assert.False(m.Set("key1", []byte("newval1")))
assert.False(m.Set("key2", []byte("newval2")))
assert.False(m.Set("key3", []byte("newval3")))
Expand Down
9 changes: 8 additions & 1 deletion internal/hash/zipmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,19 @@ func (zm *ZipMap) find(key string) (it *list.LpIterator, val []byte) {
}

func (zm *ZipMap) Set(key string, val []byte) (newField bool) {
it, oldVal := zm.find(key)
// update inplace
if it != nil && len(val) == len(oldVal) {
copy(oldVal, val)
return false
}
// replace
entry := zm.encode(key, val)
it, _ := zm.find(key)
if it != nil {
it.ReplaceNext(b2s(entry))
return false
}
// insert
zm.data.RPush(b2s(entry))
return true
}
Expand Down
6 changes: 3 additions & 3 deletions internal/zset/zset.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"cmp"

"github.com/chen3feng/stl4go"
"github.com/cockroachdb/swiss"
"github.com/dolthub/swiss"
)

type node struct {
Expand All @@ -26,7 +26,7 @@ type ZSet struct {

func NewZSet() *ZSet {
return &ZSet{
m: swiss.New[string, float64](8),
m: swiss.NewMap[string, float64](8),
skl: stl4go.NewSkipListFunc[node, struct{}](nodeCompare),
}
}
Expand Down Expand Up @@ -95,5 +95,5 @@ func (z *ZSet) Range(start, stop int, fn func(key string, score float64)) {
}

func (z *ZSet) Len() int {
return z.m.Len()
return z.m.Count()
}
7 changes: 7 additions & 0 deletions rotom.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"io"
"runtime"
"runtime/debug"
"time"

"github.com/xgzlucario/rotom/internal/dict"
"github.com/xgzlucario/rotom/internal/hash"
Expand Down Expand Up @@ -238,10 +239,16 @@ func SysMonitor(loop *AeLoop, id int, extra interface{}) {
runtime.ReadMemStats(&mem)
debug.ReadGCStats(&stat)

var pause time.Duration
if stat.NumGC > 0 {
pause = stat.PauseTotal / time.Duration(stat.NumGC)
}

log.Info().
Str("gcsys", readableSize(mem.GCSys)).
Str("heapInuse", readableSize(mem.HeapInuse)).
Str("heapObjects", fmt.Sprintf("%.1fk", float64(mem.HeapObjects)/1e3)).
Str("pause", fmt.Sprintf("%v", pause)).
Int64("gc", stat.NumGC).
Msgf("[SYS]")

Expand Down

0 comments on commit cc1df88

Please sign in to comment.