feat: syncmap use gigacache (#35)

* change syncmap struct to gigacache

* remove gofakeit dependency

* fix bugs

* go mod tidy

* add cover

---------

Co-authored-by: guangzhixu <[email protected]>

Makefile

@@ -10,7 +10,7 @@ run-gc:
  GODEBUG=gctrace=1 go run main.go
 
 test-cover:
- go test -race \
+ go test -race -v \
  -coverpkg=./... \
  -coverprofile=coverage.txt -covermode=atomic
  go tool cover -html=coverage.txt -o coverage.html

db.go

@@ -7,7 +7,6 @@ import (
  "os"
  "path/filepath"
  "sync"
- "sync/atomic"
  "time"
 
  "github.com/gofrs/flock"
@@ -74,14 +73,14 @@ var cmdTable = []Cmd{
 
  switch tp {
  case TypeList:
- ls := structx.NewList[string]()
+ ls := structx.NewList()
  if err := ls.UnmarshalJSON(val); err != nil {
  return err
  }
  db.cm.Set(key, ls)
 
  case TypeSet:
- s := structx.NewSet[string]()
+ s := structx.NewSet()
  if err := s.UnmarshalJSON(val); err != nil {
  return err
  }
@@ -232,24 +231,23 @@ const (
 type (
  String = []byte
  Map = *structx.SyncMap
- Set = *structx.Set[string]
- List = *structx.List[string]
+ Set = *structx.Set
+ List = *structx.List
  ZSet = *structx.ZSet[string, float64]
  BitMap = *structx.Bitmap
 )
 
 // DB represents a rotom database.
 type DB struct {
- mu sync.Mutex
- options *Options
- fileLock *flock.Flock
- wal *wal.WAL
- loading bool // is loading finished from wal.
- closed bool
- shrinking uint32
- m *cache.GigaCache // data for bytes.
- cm cmap.ConcurrentMap[string, any] // data for built-in types.
- cron *cron.Cron // cron scheduler for auto merge task.
+ mu sync.Mutex
+ options *Options
+ fileLock *flock.Flock
+ wal *wal.WAL
+ loading bool // is loading finished from wal.
+ closed bool
+ m *cache.GigaCache // data for bytes.
+ cm cmap.ConcurrentMap[string, any] // data for built-in types.
+ cron *cron.Cron // cron scheduler for auto merge task.
 }
 
 // Open create a new db instance by options.
@@ -469,7 +467,7 @@ func (db *DB) HRemove(key string, fields ...string) (n int, err error) {
  }
  db.encode(newCodec(OpHRemove).Str(key).StrSlice(fields))
  for _, k := range fields {
- if m.Delete(k) {
+ if m.Remove(k) {
  n++
  }
  }
@@ -892,13 +890,9 @@ func (db *DB) loadFromWal() error {
 
 // Shrink rewrite db file.
 func (db *DB) Shrink() error {
- // cas
- if !atomic.CompareAndSwapUint32(&db.shrinking, 0, 1) {
+ if !db.mu.TryLock() {
  return ErrShrinkRunning
  }
- defer atomic.StoreUint32(&db.shrinking, 0)
-
- db.mu.Lock()
  defer db.mu.Unlock()
 
  // create new segment file.
@@ -974,13 +968,13 @@ func (db *DB) fetchMap(key string, setnx ...bool) (m Map, err error) {
 
 func (db *DB) fetchSet(key string, setnx ...bool) (s Set, err error) {
  return fetch(db, key, func() Set {
- return structx.NewSet[string]()
+ return structx.NewSet()
  }, setnx...)
 }
 
 func (db *DB) fetchList(key string, setnx ...bool) (m List, err error) {
  return fetch(db, key, func() List {
- return structx.NewList[string]()
+ return structx.NewList()
  }, setnx...)
 }
 

db_test.go

@@ -2,12 +2,11 @@ package rotom
 
 import (
  "fmt"
- "math/rand"
+ "math/rand/v2"
  "strconv"
  "testing"
  "time"
 
- "github.com/brianvoe/gofakeit/v6"
  "github.com/stretchr/testify/assert"
  "github.com/xgzlucario/rotom/codeman"
 )
@@ -21,15 +20,13 @@ func createDB() (*DB, error) {
  options := DefaultOptions
  options.ShardCount = 4
  options.DirPath = fmt.Sprintf("tmp-%x", time.Now().UnixNano())
- options.ShrinkCronExpr = "0/3 * 0/1 * * ?" // every 3 seconds
+ options.ShrinkCronExpr = ""
  return Open(options)
 }
 
 func TestDB(t *testing.T) {
- println("===== TestDB =====")
  assert := assert.New(t)
  const N = 5000
-
  db, err := createDB()
  assert.Nil(err)
 
@@ -113,9 +110,7 @@ func TestDB(t *testing.T) {
 }
 
 func TestHmap(t *testing.T) {
- println("===== TestHmap =====")
  assert := assert.New(t)
-
  db, err := createDB()
  assert.Nil(err)
  defer db.Close()
@@ -212,16 +207,18 @@ func TestHmap(t *testing.T) {
  }
 }
 
+func randString() string {
+ return fmt.Sprintf("%08x", rand.Uint32())
+}
+
 func TestList(t *testing.T) {
- println("===== TestList =====")
  assert := assert.New(t)
-
  db, err := createDB()
  assert.Nil(err)
 
  for i := 0; i < 10000; i++ {
  key := "list" + strconv.Itoa(i/100)
- val := gofakeit.Animal()
+ val := randString()
 
  if i%2 == 0 {
  assert.Nil(db.LRPush(key, val))
@@ -299,7 +296,7 @@ func TestList(t *testing.T) {
  assert.Equal(err, ErrIndexOutOfRange)
 
  for i := 0; i < 100; i++ {
- db.LRPush("list", gofakeit.Animal())
+ db.LRPush("list", randString())
  }
  }
 
@@ -316,9 +313,7 @@ func TestList(t *testing.T) {
 }
 
 func TestSet(t *testing.T) {
- println("===== TestSet =====")
  assert := assert.New(t)
-
  db, err := createDB()
  assert.Nil(err)
 
@@ -360,8 +355,8 @@ func TestSet(t *testing.T) {
  // Add random data
  for i := 0; i < 20; i++ {
  stri := strconv.Itoa(i)
- db.SAdd("a"+stri, gofakeit.Animal())
- db.SAdd("b"+stri, gofakeit.Animal())
+ db.SAdd("a"+stri, randString())
+ db.SAdd("b"+stri, randString())
  }
  stri := strconv.Itoa(i)
 
@@ -435,9 +430,7 @@ func TestSet(t *testing.T) {
 }
 
 func TestBitmap(t *testing.T) {
- println("===== TestBitmap =====")
  assert := assert.New(t)
-
  db, err := createDB()
  assert.Nil(err)
 
@@ -558,9 +551,7 @@ func TestBitmap(t *testing.T) {
 }
 
 func TestZSet(t *testing.T) {
- println("===== TestZSet =====")
  assert := assert.New(t)
-
  db, err := createDB()
  assert.Nil(err)
 
@@ -677,7 +668,6 @@ func TestZSet(t *testing.T) {
 }
 
 func TestInvalidCodec(t *testing.T) {
- println("===== TestInvalidCodec =====")
  assert := assert.New(t)
 
  // read args.
@@ -695,14 +685,15 @@ func TestInvalidCodec(t *testing.T) {
  assert.Panics(func() {
  reader.Byte()
  })
+ assert.Panics(func() {
+ reader.RawBytes()
+ })
 }
 
 func TestRace(t *testing.T) {
- println("===== TestRace =====")
  assert := assert.New(t)
 
- // open invalid options.
- {
+ t.Run("checkOptions", func(t *testing.T) {
  options := DefaultOptions
  options.DirPath = ""
  _, err := Open(options)
@@ -712,22 +703,37 @@ func TestRace(t *testing.T) {
  options.ShardCount = 0
  _, err = Open(options)
  assert.NotNil(err)
- }
+ })
 
- // dirpath race.
- options := DefaultOptions
- options.DirPath = "tmp-race"
- db, err := Open(options)
- assert.Nil(err)
- assert.NotNil(db)
+ t.Run("db-race", func(t *testing.T) {
+  options := DefaultOptions
+  options.DirPath = "tmp-race"
+  db, err := Open(options)
+  assert.Nil(err)
+  assert.NotNil(db)
 
- // open another db.
- _, err = Open(options)
- assert.Equal(err, ErrDatabaseIsUsing)
+ // open another db.
+ _, err = Open(options)
+ assert.Equal(err, ErrDatabaseIsUsing)
+ })
+
+ t.Run("open-wal", func(t *testing.T) {
+ options := DefaultOptions
+ options.DirPath = "README.md"
+ _, err := Open(options)
+ assert.NotNil(err)
+ })
+
+ t.Run("wait-fr-shrink", func(t *testing.T) {
+ options := DefaultOptions
+ options.ShrinkCronExpr = "0/1 * * * * ?" // every second
+ db, _ := Open(options)
+ time.Sleep(time.Second)
+ db.Close()
+ })
 }
 
 func TestUnmarshalError(t *testing.T) {
- println("===== TestUnmarshalError =====")
  assert := assert.New(t)
 
  for _, types := range []int64{TypeMap, TypeList, TypeSet, TypeZSet, TypeBitmap} {
@@ -743,7 +749,6 @@ func TestUnmarshalError(t *testing.T) {
 }
 
 func TestShrink(t *testing.T) {
- println("===== TestShrink =====")
  assert := assert.New(t)
 
  db, err := createDB()

go.mod

@@ -4,7 +4,6 @@ go 1.21
 
 require (
  github.com/RoaringBitmap/roaring v1.9.1
- github.com/brianvoe/gofakeit/v6 v6.24.0
  github.com/bytedance/sonic v1.11.3
  github.com/cockroachdb/swiss v0.0.0-20240303172742-c161743eb608
  github.com/deckarep/golang-set/v2 v2.6.0
@@ -14,7 +13,7 @@ require (
  github.com/rosedblabs/wal v1.3.6
  github.com/sakeven/RbTree v0.0.0-20240321014605-9899538dc980
  github.com/stretchr/testify v1.8.4
- github.com/xgzlucario/GigaCache v0.0.0-20240325102640-77618656847b
+ github.com/xgzlucario/GigaCache v0.0.0-20240330133804-4fb63804aa28
  github.com/zyedidia/generic v1.2.2-0.20230819223407-98022f93823f
  golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8
 )

go.sum

@@ -1,12 +1,10 @@
 github.com/RoaringBitmap/roaring v1.9.1 h1:LXcSqGGGMKm+KAzUyWn7ZeREqoOkoMX+KwLOK1thc4I=
 github.com/RoaringBitmap/roaring v1.9.1/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
-github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo=
-github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo=
+github.com/aclements/go-perfevent v0.0.0-20240318182238-eb5c9da0b102 h1:EDzu1cEaxu36ffqeGW6duvf9uZtd6kWwHH5xOfC2vus=
+github.com/aclements/go-perfevent v0.0.0-20240318182238-eb5c9da0b102/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo=
 github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
 github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
-github.com/brianvoe/gofakeit/v6 v6.24.0 h1:74yq7RRz/noddscZHRS2T84oHZisW9muwbb8sRnU52A=
-github.com/brianvoe/gofakeit/v6 v6.24.0/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
 github.com/bytedance/sonic v1.11.3 h1:jRN+yEjakWh8aK5FzrciUHG8OFXK+4/KrAX/ysEtHAA=
@@ -66,19 +64,15 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/xgzlucario/GigaCache v0.0.0-20240322084843-fb66291fa5ea h1:D+07nrMr7bjT3GM9yiSUI4hta9rFhblIWrIJJ0whhFs=
-github.com/xgzlucario/GigaCache v0.0.0-20240322084843-fb66291fa5ea/go.mod h1:lcByXeKmaSdEOOCjf1WzhB7svT/Yn0Wq/fSxebl+4tA=
-github.com/xgzlucario/GigaCache v0.0.0-20240325102640-77618656847b h1:GDobqi94vTNst6iL/Wd0zFu9WXi8EX0leUcA+SPVB+g=
-github.com/xgzlucario/GigaCache v0.0.0-20240325102640-77618656847b/go.mod h1:H2yKEBEIY+LEMEM9r3G+X7GG1W8p9jEohQRmgr7OkUg=
+github.com/xgzlucario/GigaCache v0.0.0-20240330133804-4fb63804aa28 h1:+KCjnQHuI3mgobmbrBA1kRUqg+cZuG/RielCtnCUsvY=
+github.com/xgzlucario/GigaCache v0.0.0-20240330133804-4fb63804aa28/go.mod h1:dSj1jMxZhjhMIui4c0YIKtVlueATj8rHEfbresGn5iU=
 github.com/zyedidia/generic v1.2.2-0.20230819223407-98022f93823f h1:r3wBquYQZ6jhwGnWWdc4jCydI7xxoNWZMGxzmii2jHE=
 github.com/zyedidia/generic v1.2.2-0.20230819223407-98022f93823f/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
 golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
-golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc=
-golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=