From 515856e5a19fd76e4baefbad62a9183dc535bdae Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Wed, 14 Dec 2022 20:43:40 +0800 Subject: [PATCH 01/22] add levigo engine --- go.mod | 3 + kv/server/raw_api.go | 106 +++++++++++++- .../standalone_storage/standalone_storage.go | 80 +++++++++- kv/util/engine_util/cf_iterator.go | 137 +++++++++++++----- 4 files changed, 287 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index e72ae7548..eb3d3c19e 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.3.4 github.com/google/btree v1.0.0 + github.com/jmhodges/levigo v0.0.0-00010101000000-000000000000 github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 github.com/juju/testing v0.0.0-20200510222523-6c8c298c77a0 // indirect github.com/onsi/ginkgo v1.12.1 // indirect @@ -36,3 +37,5 @@ require ( go 1.13 replace github.com/pingcap/tidb => github.com/pingcap-incubator/tinysql v0.0.0-20200518090433-a7d00f9e6aa7 + +replace github.com/jmhodges/levigo => ./levigo diff --git a/kv/server/raw_api.go b/kv/server/raw_api.go index 025cb2466..7b4574093 100644 --- a/kv/server/raw_api.go +++ b/kv/server/raw_api.go @@ -2,6 +2,9 @@ package server import ( "context" + + "github.com/pingcap-incubator/tinykv/kv/storage" + "github.com/pingcap-incubator/tinykv/kv/storage/raft_storage" "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" ) @@ -11,26 +14,121 @@ import ( // RawGet return the corresponding Get response based on RawGetRequest's CF and Key fields func (server *Server) RawGet(_ context.Context, req *kvrpcpb.RawGetRequest) (*kvrpcpb.RawGetResponse, error) { // Your Code Here (1). - return nil, nil + resp := &kvrpcpb.RawGetResponse{} + reader, err := server.storage.Reader(req.Context) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + defer reader.Close() + val, err := reader.GetCF(req.Cf, req.Key) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + resp.Value = val + if val == nil { + resp.NotFound = true + } + return resp, nil } // RawPut puts the target data into storage and returns the corresponding response func (server *Server) RawPut(_ context.Context, req *kvrpcpb.RawPutRequest) (*kvrpcpb.RawPutResponse, error) { // Your Code Here (1). // Hint: Consider using Storage.Modify to store data to be modified - return nil, nil + resp := &kvrpcpb.RawPutResponse{} + batch := []storage.Modify{ + { + Data: storage.Put{ + Cf: req.Cf, + Key: req.Key, + Value: req.Value, + }, + }} + err := server.storage.Write(req.Context, batch) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + return resp, nil } // RawDelete delete the target data from storage and returns the corresponding response func (server *Server) RawDelete(_ context.Context, req *kvrpcpb.RawDeleteRequest) (*kvrpcpb.RawDeleteResponse, error) { // Your Code Here (1). // Hint: Consider using Storage.Modify to store data to be deleted - return nil, nil + resp := &kvrpcpb.RawDeleteResponse{} + batch := []storage.Modify{ + { + Data: storage.Delete{ + Cf: req.Cf, + Key: req.Key, + }, + }} + err := server.storage.Write(req.Context, batch) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + return resp, nil } // RawScan scan the data starting from the start key up to limit. and return the corresponding result func (server *Server) RawScan(_ context.Context, req *kvrpcpb.RawScanRequest) (*kvrpcpb.RawScanResponse, error) { // Your Code Here (1). // Hint: Consider using reader.IterCF - return nil, nil + resp := &kvrpcpb.RawScanResponse{} + if req.Limit == 0 { + return resp, nil + } + reader, err := server.storage.Reader(req.Context) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + defer reader.Close() + iter := reader.IterCF(req.Cf) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + defer iter.Close() + var pairs []*kvrpcpb.KvPair + n := req.Limit + for iter.Seek(req.StartKey); iter.Valid(); iter.Next() { + item := iter.Item() + val, err := item.ValueCopy(nil) + if err != nil { + return nil, err + } + pairs = append(pairs, &kvrpcpb.KvPair{ + Key: item.KeyCopy(nil), + Value: val, + }) + n-- + if n == 0 { + break + } + } + resp.Kvs = pairs + return resp, nil } diff --git a/kv/storage/standalone_storage/standalone_storage.go b/kv/storage/standalone_storage/standalone_storage.go index 32a773364..bc5b7422e 100644 --- a/kv/storage/standalone_storage/standalone_storage.go +++ b/kv/storage/standalone_storage/standalone_storage.go @@ -1,8 +1,12 @@ package standalone_storage import ( + "fmt" + + "github.com/jmhodges/levigo" "github.com/pingcap-incubator/tinykv/kv/config" "github.com/pingcap-incubator/tinykv/kv/storage" + "github.com/pingcap-incubator/tinykv/kv/util/engine_util" "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" ) @@ -10,11 +14,32 @@ import ( // communicate with other nodes and all data is stored locally. type StandAloneStorage struct { // Your Data Here (1). + db *levigo.DB + roptions *levigo.ReadOptions + woptions *levigo.WriteOptions } func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { // Your Code Here (1). - return nil + dbName := "./dbtest1" + opt := levigo.NewOptions() + ropt := levigo.NewReadOptions() + wopt := levigo.NewWriteOptions() + opt.SetCreateIfMissing(true) + opt.SetWriteBufferSize(67108864) + policy := levigo.NewBloomFilter(10) + opt.SetFilterPolicy(policy) + db, err := levigo.Open(dbName, opt) + if err != nil { + fmt.Printf("open leveldb error!\n") + return nil + } + s := &StandAloneStorage{ + db, + ropt, + wopt, + } + return s } func (s *StandAloneStorage) Start() error { @@ -24,15 +49,66 @@ func (s *StandAloneStorage) Start() error { func (s *StandAloneStorage) Stop() error { // Your Code Here (1). + s.db.Close() return nil } func (s *StandAloneStorage) Reader(ctx *kvrpcpb.Context) (storage.StorageReader, error) { // Your Code Here (1). - return nil, nil + return &StandAloneStorageReader{ + s.db, + s.roptions, + }, nil } func (s *StandAloneStorage) Write(ctx *kvrpcpb.Context, batch []storage.Modify) error { // Your Code Here (1). + for _, m := range batch { + switch m.Data.(type) { + case storage.Put: + put := m.Data.(storage.Put) + if err := s.db.Put(s.woptions, engine_util.KeyWithCF(put.Cf, put.Key), put.Value); err != nil { + return err + } + case storage.Delete: + del := m.Data.(storage.Delete) + if err := s.db.Delete(s.woptions, engine_util.KeyWithCF(del.Cf, del.Key)); err != nil { + return err + } + } + } return nil } + +type StandAloneStorageReader struct { + db *levigo.DB + roptions *levigo.ReadOptions +} + +func (sReader *StandAloneStorageReader) Close() { + return +} + +func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { + val, err := sReader.db.Get(sReader.roptions, engine_util.KeyWithCF(cf, key)) + + return val, err +} + +func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { + return engine_util.NewLDBIterator(cf, sReader.db, sReader.roptions) +} + +// func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { +// value, err := sReader.db.Get(sReader.) + +// v, e := engine_util.GetCF(sReader.db, cf, key) +// if e == badger.ErrKeyNotFound { +// return nil, nil +// } +// return v, e +// } +// func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { +// txn := sReader.db.NewTransaction(false) +// return engine_util.NewCFIterator(cf, txn) +// } diff --git a/kv/util/engine_util/cf_iterator.go b/kv/util/engine_util/cf_iterator.go index 6a9e4e06f..591dc9256 100644 --- a/kv/util/engine_util/cf_iterator.go +++ b/kv/util/engine_util/cf_iterator.go @@ -2,8 +2,112 @@ package engine_util import ( "github.com/Connor1996/badger" + "github.com/Connor1996/badger/y" + "github.com/jmhodges/levigo" ) +type LdbItem struct { + key []byte + value []byte + prefixLen int +} + +func (i *LdbItem) Key() []byte { + return i.key[i.prefixLen:] +} + +func (i *LdbItem) KeyCopy(dst []byte) []byte { + return y.SafeCopy(dst, i.key[i.prefixLen:]) +} + +func (i *LdbItem) Value() ([]byte, error) { + return i.value, nil +} + +func (i *LdbItem) ValueSize() int { + return len(i.value) +} + +func (i *LdbItem) ValueCopy(dst []byte) ([]byte, error) { + return y.SafeCopy(dst, i.value), nil +} + +type LdbIterator struct { + iter *levigo.Iterator + prefix string +} + +func NewLDBIterator(cf string, db *levigo.DB, roptions *levigo.ReadOptions) *LdbIterator { + return &LdbIterator{ + iter: db.NewIterator(roptions), + prefix: cf + "_", + } +} + +func (it *LdbIterator) Item() DBItem { + return &LdbItem{ + key: it.iter.Key(), + value: it.iter.Value(), + prefixLen: len(it.prefix), + } +} + +func (it *LdbIterator) Valid() bool { return it.iter.Valid() } + +// func (it *BadgerIterator) ValidForPrefix(prefix []byte) bool { +// return it.iter.ValidForPrefix(append([]byte(it.prefix), prefix...)) +// } + +func (it *LdbIterator) Close() { + it.iter.Close() +} + +func (it *LdbIterator) Next() { + it.iter.Next() +} + +func (it *LdbIterator) Seek(key []byte) { + it.iter.Seek(append([]byte(it.prefix), key...)) + // it.iter.Seek(append([]byte(it.prefix), key...)) +} + +// func (it *LdbIterator) Rewind() { +// it.iter.Rewind() +// } + +type DBIterator interface { + // Item returns pointer to the current key-value pair. + Item() DBItem + // Valid returns false when iteration is done. + Valid() bool + // Next would advance the iterator by one. Always check it.Valid() after a Next() + // to ensure you have access to a valid it.Item(). + Next() + // Seek would seek to the provided key if present. If absent, it would seek to the next smallest key + // greater than provided. + Seek([]byte) + + // Close the iterator + Close() +} + +type DBItem interface { + // Key returns the key. + Key() []byte + // KeyCopy returns a copy of the key of the item, writing it to dst slice. + // If nil is passed, or capacity of dst isn't sufficient, a new slice would be allocated and + // returned. + KeyCopy(dst []byte) []byte + // Value retrieves the value of the item. + Value() ([]byte, error) + // ValueSize returns the size of the value. + ValueSize() int + // ValueCopy returns a copy of the value of the item from the value log, writing it to dst slice. + // If nil is passed, or capacity of dst isn't sufficient, a new slice would be allocated and + // returned. + ValueCopy(dst []byte) ([]byte, error) +} + type CFItem struct { item *badger.Item prefixLen int @@ -94,36 +198,3 @@ func (it *BadgerIterator) Seek(key []byte) { func (it *BadgerIterator) Rewind() { it.iter.Rewind() } - -type DBIterator interface { - // Item returns pointer to the current key-value pair. - Item() DBItem - // Valid returns false when iteration is done. - Valid() bool - // Next would advance the iterator by one. Always check it.Valid() after a Next() - // to ensure you have access to a valid it.Item(). - Next() - // Seek would seek to the provided key if present. If absent, it would seek to the next smallest key - // greater than provided. - Seek([]byte) - - // Close the iterator - Close() -} - -type DBItem interface { - // Key returns the key. - Key() []byte - // KeyCopy returns a copy of the key of the item, writing it to dst slice. - // If nil is passed, or capacity of dst isn't sufficient, a new slice would be allocated and - // returned. - KeyCopy(dst []byte) []byte - // Value retrieves the value of the item. - Value() ([]byte, error) - // ValueSize returns the size of the value. - ValueSize() int - // ValueCopy returns a copy of the value of the item from the value log, writing it to dst slice. - // If nil is passed, or capacity of dst isn't sufficient, a new slice would be allocated and - // returned. - ValueCopy(dst []byte) ([]byte, error) -} From a5577787318fb67aefd7b0f53871234c9bcd12e9 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Wed, 14 Dec 2022 21:43:47 +0800 Subject: [PATCH 02/22] add levigo --- levigo/.github/renovate.json | 6 + levigo/.github/versions/go | 1 + levigo/.github/workflows/go.yml | 30 ++ levigo/.gitignore | 6 + levigo/.travis/.gitignore | 3 + levigo/.travis/Makefile | 49 ++++ levigo/LICENSE | 7 + levigo/README.md | 62 ++++ levigo/batch.go | 66 +++++ levigo/cache.go | 32 +++ levigo/comparator.go | 13 + levigo/conv.go | 19 ++ levigo/db.go | 390 ++++++++++++++++++++++++++ levigo/doc.go | 65 +++++ levigo/env.go | 29 ++ levigo/examples/comparator_example.go | 46 +++ levigo/filterpolicy.go | 33 +++ levigo/go.mod | 3 + levigo/go.sum | 0 levigo/iterator.go | 150 ++++++++++ levigo/leveldb_test.go | 359 ++++++++++++++++++++++++ levigo/levigo.sh | 51 ++++ levigo/options.go | 228 +++++++++++++++ levigo/version.go | 19 ++ 24 files changed, 1667 insertions(+) create mode 100644 levigo/.github/renovate.json create mode 100644 levigo/.github/versions/go create mode 100644 levigo/.github/workflows/go.yml create mode 100644 levigo/.gitignore create mode 100644 levigo/.travis/.gitignore create mode 100644 levigo/.travis/Makefile create mode 100644 levigo/LICENSE create mode 100644 levigo/README.md create mode 100644 levigo/batch.go create mode 100644 levigo/cache.go create mode 100644 levigo/comparator.go create mode 100644 levigo/conv.go create mode 100644 levigo/db.go create mode 100644 levigo/doc.go create mode 100644 levigo/env.go create mode 100644 levigo/examples/comparator_example.go create mode 100644 levigo/filterpolicy.go create mode 100644 levigo/go.mod create mode 100644 levigo/go.sum create mode 100644 levigo/iterator.go create mode 100644 levigo/leveldb_test.go create mode 100755 levigo/levigo.sh create mode 100644 levigo/options.go create mode 100644 levigo/version.go diff --git a/levigo/.github/renovate.json b/levigo/.github/renovate.json new file mode 100644 index 000000000..2be084fbb --- /dev/null +++ b/levigo/.github/renovate.json @@ -0,0 +1,6 @@ +{ + "extends": [ + "config:base" + ], + "postUpdateOptions": ["gomodTidy"] +} diff --git a/levigo/.github/versions/go b/levigo/.github/versions/go new file mode 100644 index 000000000..01b756823 --- /dev/null +++ b/levigo/.github/versions/go @@ -0,0 +1 @@ +1.13.3 diff --git a/levigo/.github/workflows/go.yml b/levigo/.github/workflows/go.yml new file mode 100644 index 000000000..e986b5353 --- /dev/null +++ b/levigo/.github/workflows/go.yml @@ -0,0 +1,30 @@ +name: Go +on: + pull_request: + branches: + - master + push: + branches: + - master +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Read Go versions + run: echo "##[set-output name=go_version;]$(cat .github/versions/go)" + id: go_versions + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: ${{ steps.go_versions.outputs.go_version }} + id: go + + - name: Build and test + run: cd .travis && make diff --git a/levigo/.gitignore b/levigo/.gitignore new file mode 100644 index 000000000..2af17d56f --- /dev/null +++ b/levigo/.gitignore @@ -0,0 +1,6 @@ +*.o +*.a +*.6 +*.out +_testmain.go +_obj diff --git a/levigo/.travis/.gitignore b/levigo/.travis/.gitignore new file mode 100644 index 000000000..b709f1812 --- /dev/null +++ b/levigo/.travis/.gitignore @@ -0,0 +1,3 @@ +archives +root + diff --git a/levigo/.travis/Makefile b/levigo/.travis/Makefile new file mode 100644 index 000000000..c42d2edd7 --- /dev/null +++ b/levigo/.travis/Makefile @@ -0,0 +1,49 @@ +all: test + +LEVELDB_VERSION ?= v1.18 +SNAPPY_VERSION ?= 1.1.7 + +export CFLAGS ?= -I$(PWD)/root/snappy-$(SNAPPY_VERSION)/include +export CXXFLAGS ?= -I$(PWD)/root/snappy-$(SNAPPY_VERSION)/build +export LDFLAGS ?= -L$(PWD)/root/snappy-$(SNAPPY_VERSION)/build +export CGO_CFLAGS ?= -I$(PWD)/root/snappy-$(SNAPPY_VERSION)/build -I$(PWD)/root/leveldb/include +export CGO_LDFLAGS ?= -L$(PWD)/root/snappy-$(SNAPPY_VERSION)/build -L$(PWD)/root/leveldb -lsnappy +export GOPATH ?= $(PWD)/root/go +export LD_LIBRARY_PATH := $(PWD)/root/snappy-$(SNAPPY_VERSION)/build:$(PWD)/root/leveldb:$(LD_LIBRARY_PATH) + +archives/snappy-$(SNAPPY_VERSION).tar.gz: archives + curl -L https://github.com/google/snappy/archive/$(SNAPPY_VERSION).tar.gz > $@ + +archives: + mkdir -v archives + +levigo: root/snappy-$(SNAPPY_VERSION)/STAMP root/leveldb/STAMP + cd ../ && go get -d . + cd ../ && go build . + cd ../ && go test -test.v=true . + +root: + mkdir -v root + +root/leveldb: root + cd root && git clone https://github.com/google/leveldb.git + +root/leveldb/STAMP: root/leveldb root/snappy-$(SNAPPY_VERSION)/STAMP + cd root/leveldb && git checkout $(LEVELDB_VERSION) + $(MAKE) -C root/leveldb + touch $@ + +root/snappy-$(SNAPPY_VERSION): archives/snappy-$(SNAPPY_VERSION).tar.gz root + tar xzvf archives/snappy-$(SNAPPY_VERSION).tar.gz -C root + +root/snappy-$(SNAPPY_VERSION)/STAMP: root/snappy-$(SNAPPY_VERSION) + mkdir -p root/snappy-$(SNAPPY_VERSION)/build && cd root/snappy-$(SNAPPY_VERSION)/build && cmake -DCMAKE_INSTALL_PREFIX:PATH=$(pwd) ../ && make + touch $@ + +test: levigo + +clean: + -rm -rf archives + -rm -rf root + +.PHONY: levigo test diff --git a/levigo/LICENSE b/levigo/LICENSE new file mode 100644 index 000000000..c7c73befc --- /dev/null +++ b/levigo/LICENSE @@ -0,0 +1,7 @@ +Copyright (c) 2012 Jeffrey M Hodges + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/levigo/README.md b/levigo/README.md new file mode 100644 index 000000000..3d92d0f61 --- /dev/null +++ b/levigo/README.md @@ -0,0 +1,62 @@ +[![GoDoc](https://godoc.org/github.com/jmhodges/levigo?status.svg)](https://godoc.org/github.com/jmhodges/levigo) + +# levigo + +levigo is a Go wrapper for LevelDB. + +The API has been godoc'ed and [is available on the +web](http://godoc.org/github.com/jmhodges/levigo). + +Questions answered at `golang-nuts@googlegroups.com`. + +## Building + +You'll need the shared library build of +[LevelDB](http://code.google.com/p/leveldb/) installed on your machine. The +current LevelDB will build it by default. + +The minimum version of LevelDB required is currently 1.7. If you require the +use of an older version of LevelDB, see the [fork of levigo for LevelDB +1.4](https://github.com/jmhodges/levigo_leveldb_1.4). Prefer putting in the +work to be up to date as LevelDB moves very quickly. + +Now, if you build LevelDB and put the shared library and headers in one of the +standard places for your OS, you'll be able to simply run: + + go get github.com/jmhodges/levigo + +But, suppose you put the shared LevelDB library somewhere weird like +/path/to/lib and the headers were installed in /path/to/include. To install +levigo remotely, you'll run: + + CGO_CFLAGS="-I/path/to/leveldb/include" CGO_LDFLAGS="-L/path/to/leveldb/lib" go get github.com/jmhodges/levigo + +and there you go. + +In order to build with snappy, you'll have to explicitly add "-lsnappy" to the +`CGO_LDFLAGS`. Supposing that both snappy and leveldb are in weird places, +you'll run something like: + + CGO_CFLAGS="-I/path/to/leveldb/include -I/path/to/snappy/include" + CGO_LDFLAGS="-L/path/to/leveldb/lib -L/path/to/snappy/lib -lsnappy" go get github.com/jmhodges/levigo + +(and make sure the -lsnappy is after the snappy library path!). + +Of course, these same rules apply when doing `go build`, as well. + +## Caveats + +Comparators and WriteBatch iterators must be written in C in your own +library. This seems like a pain in the ass, but remember that you'll have the +LevelDB C API available to your in your client package when you import levigo. + +An example of writing your own Comparator can be found in +. + +## Status + +Build: [![Build Status](https://travis-ci.org/jmhodges/levigo.svg)](https://travis-ci.org/jmhodges/levigo) + +Documentation: [![GoDoc](https://godoc.org/github.com/jmhodges/levigo?status.svg)](https://godoc.org/github.com/jmhodges/levigo) + +Lint: [Go Lint](http://go-lint.appspot.com/github.com/jmhodges/levigo) diff --git a/levigo/batch.go b/levigo/batch.go new file mode 100644 index 000000000..0e774ed08 --- /dev/null +++ b/levigo/batch.go @@ -0,0 +1,66 @@ +package levigo + +// #cgo LDFLAGS: -lleveldb -lstdc++ +// #include "leveldb/c.h" +import "C" + +import ( + "unsafe" +) + +// WriteBatch is a batching of Puts, and Deletes to be written atomically to a +// database. A WriteBatch is written when passed to DB.Write. +// +// To prevent memory leaks, call Close when the program no longer needs the +// WriteBatch object. +type WriteBatch struct { + wbatch *C.leveldb_writebatch_t +} + +// NewWriteBatch creates a fully allocated WriteBatch. +func NewWriteBatch() *WriteBatch { + wb := C.leveldb_writebatch_create() + return &WriteBatch{wb} +} + +// Close releases the underlying memory of a WriteBatch. +func (w *WriteBatch) Close() { + C.leveldb_writebatch_destroy(w.wbatch) +} + +// Put places a key-value pair into the WriteBatch for writing later. +// +// Both the key and value byte slices may be reused as WriteBatch takes a copy +// of them before returning. +// +func (w *WriteBatch) Put(key, value []byte) { + // leveldb_writebatch_put, and _delete call memcpy() (by way of + // Memtable::Add) when called, so we do not need to worry about these + // []byte being reclaimed by GC. + var k, v *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + if len(value) != 0 { + v = (*C.char)(unsafe.Pointer(&value[0])) + } + + lenk := len(key) + lenv := len(value) + + C.leveldb_writebatch_put(w.wbatch, k, C.size_t(lenk), v, C.size_t(lenv)) +} + +// Delete queues a deletion of the data at key to be deleted later. +// +// The key byte slice may be reused safely. Delete takes a copy of +// them before returning. +func (w *WriteBatch) Delete(key []byte) { + C.leveldb_writebatch_delete(w.wbatch, + (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) +} + +// Clear removes all the enqueued Put and Deletes in the WriteBatch. +func (w *WriteBatch) Clear() { + C.leveldb_writebatch_clear(w.wbatch) +} diff --git a/levigo/cache.go b/levigo/cache.go new file mode 100644 index 000000000..cf4da44c6 --- /dev/null +++ b/levigo/cache.go @@ -0,0 +1,32 @@ +package levigo + +// #cgo LDFLAGS: -lleveldb +// #include +// #include "leveldb/c.h" +import "C" + +// Cache is a cache used to store data read from data in memory. +// +// Typically, NewLRUCache is all you will need, but advanced users may +// implement their own *C.leveldb_cache_t and create a Cache. +// +// To prevent memory leaks, a Cache must have Close called on it when it is +// no longer needed by the program. Note: if the process is shutting down, +// this may not be necessary and could be avoided to shorten shutdown time. +type Cache struct { + Cache *C.leveldb_cache_t +} + +// NewLRUCache creates a new Cache object with the capacity given. +// +// To prevent memory leaks, Close should be called on the Cache when the +// program no longer needs it. Note: if the process is shutting down, this may +// not be necessary and could be avoided to shorten shutdown time. +func NewLRUCache(capacity int) *Cache { + return &Cache{C.leveldb_cache_create_lru(C.size_t(capacity))} +} + +// Close deallocates the underlying memory of the Cache object. +func (c *Cache) Close() { + C.leveldb_cache_destroy(c.Cache) +} diff --git a/levigo/comparator.go b/levigo/comparator.go new file mode 100644 index 000000000..be5e6acaa --- /dev/null +++ b/levigo/comparator.go @@ -0,0 +1,13 @@ +package levigo + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +import "C" + +// DestroyComparator deallocates a *C.leveldb_comparator_t. +// +// This is provided as a convienience to advanced users that have implemented +// their own comparators in C in their own code. +func DestroyComparator(cmp *C.leveldb_comparator_t) { + C.leveldb_comparator_destroy(cmp) +} diff --git a/levigo/conv.go b/levigo/conv.go new file mode 100644 index 000000000..79ca81030 --- /dev/null +++ b/levigo/conv.go @@ -0,0 +1,19 @@ +package levigo + +// #include "leveldb/c.h" +import "C" + +func boolToUchar(b bool) C.uchar { + uc := C.uchar(0) + if b { + uc = C.uchar(1) + } + return uc +} + +func ucharToBool(uc C.uchar) bool { + if uc == C.uchar(0) { + return false + } + return true +} diff --git a/levigo/db.go b/levigo/db.go new file mode 100644 index 000000000..d0e2ccb9a --- /dev/null +++ b/levigo/db.go @@ -0,0 +1,390 @@ +package levigo + +/* +#cgo LDFLAGS: -lleveldb +#include +#include "leveldb/c.h" + +// This function exists only to clean up lack-of-const warnings when +// leveldb_approximate_sizes is called from Go-land. +void levigo_leveldb_approximate_sizes( + leveldb_t* db, + int num_ranges, + char** range_start_key, const size_t* range_start_key_len, + char** range_limit_key, const size_t* range_limit_key_len, + uint64_t* sizes) { + leveldb_approximate_sizes(db, + num_ranges, + (const char* const*)range_start_key, + range_start_key_len, + (const char* const*)range_limit_key, + range_limit_key_len, + sizes); +} +*/ +import "C" + +import ( + "errors" + "unsafe" +) + +// DatabaseError wraps general internal LevelDB errors for user consumption. +type DatabaseError string + +func (e DatabaseError) Error() string { + return "levigo: " + string(e) +} + +// ErrDBClosed is returned by DB.Close when its been called previously. +var ErrDBClosed = errors.New("database is closed") + +// DB is a reusable handle to a LevelDB database on disk, created by Open. +// +// To avoid memory and file descriptor leaks, call Close when the process no +// longer needs the handle. Calls to any DB method made after Close will +// panic. +// +// The DB instance may be shared between goroutines. The usual data race +// conditions will occur if the same key is written to from more than one, of +// course. +type DB struct { + Ldb *C.leveldb_t + + // TLDR: Closed is not racey, it's a best attempt. If `-race` says it's racey, + // then it's your code that is racey, not levigo. + // + // This indicates if the DB is closed or not. LevelDB provides it's own closed + // detection that appears in the form of a sigtrap panic, which can be quite + // confusing. So rather than users hit that, we attempt to give them a better + // panic by checking this flag in functions that LevelDB would have done + // it's own panic. This is not protected by a mutex because it's a best case + // attempt to catch invalid usage. If access in racey and gets to LevelDB, + // so be it, the user will see the sigtrap panic. + // Because LevelDB has it's own mutex to detect the usage, we didn't want to + // put another one up here and drive performance down even more. + // So if you use `-race` and it says closed is racey, it's your code that is + // racey, not levigo. + closed bool +} + +// Range is a range of keys in the database. GetApproximateSizes calls with it +// begin at the key Start and end right before the key Limit. +type Range struct { + Start []byte + Limit []byte +} + +// Snapshot provides a consistent view of read operations in a DB. +// +// Snapshot is used in read operations by setting it on a +// ReadOptions. Snapshots are created by calling DB.NewSnapshot. +// +// To prevent memory leaks and resource strain in the database, the snapshot +// returned must be released with DB.ReleaseSnapshot method on the DB that +// created it. +type Snapshot struct { + snap *C.leveldb_snapshot_t +} + +// Open opens a database. +// +// Creating a new database is done by calling SetCreateIfMissing(true) on the +// Options passed to Open. +// +// It is usually wise to set a Cache object on the Options with SetCache to +// keep recently used data from that database in memory. +func Open(dbname string, o *Options) (*DB, error) { + var errStr *C.char + ldbname := C.CString(dbname) + defer C.free(unsafe.Pointer(ldbname)) + + leveldb := C.leveldb_open(o.Opt, ldbname, &errStr) + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return nil, DatabaseError(gs) + } + return &DB{leveldb, false}, nil +} + +// DestroyDatabase removes a database entirely, removing everything from the +// filesystem. +func DestroyDatabase(dbname string, o *Options) error { + var errStr *C.char + ldbname := C.CString(dbname) + defer C.free(unsafe.Pointer(ldbname)) + + C.leveldb_destroy_db(o.Opt, ldbname, &errStr) + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return DatabaseError(gs) + } + return nil +} + +// RepairDatabase attempts to repair a database. +// +// If the database is unrepairable, an error is returned. +func RepairDatabase(dbname string, o *Options) error { + var errStr *C.char + ldbname := C.CString(dbname) + defer C.free(unsafe.Pointer(ldbname)) + + C.leveldb_repair_db(o.Opt, ldbname, &errStr) + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return DatabaseError(gs) + } + return nil +} + +// Put writes data associated with a key to the database. +// +// If a nil []byte is passed in as value, it will be returned by Get +// as an zero-length slice. The WriteOptions passed in can be reused +// by multiple calls to this and if the WriteOptions is left unchanged. +// +// The key and value byte slices may be reused safely. Put takes a copy of +// them before returning. +func (db *DB) Put(wo *WriteOptions, key, value []byte) error { + if db.closed { + panic(ErrDBClosed) + } + + var errStr *C.char + // leveldb_put, _get, and _delete call memcpy() (by way of Memtable::Add) + // when called, so we do not need to worry about these []byte being + // reclaimed by GC. + var k, v *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + if len(value) != 0 { + v = (*C.char)(unsafe.Pointer(&value[0])) + } + + lenk := len(key) + lenv := len(value) + C.leveldb_put( + db.Ldb, wo.Opt, k, C.size_t(lenk), v, C.size_t(lenv), &errStr) + + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return DatabaseError(gs) + } + return nil +} + +// Get returns the data associated with the key from the database. +// +// If the key does not exist in the database, a nil []byte is returned. If the +// key does exist, but the data is zero-length in the database, a zero-length +// []byte will be returned. +// +// The key byte slice may be reused safely. Get takes a copy of +// them before returning. +func (db *DB) Get(ro *ReadOptions, key []byte) ([]byte, error) { + if db.closed { + panic(ErrDBClosed) + } + + var errStr *C.char + var vallen C.size_t + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + value := C.leveldb_get( + db.Ldb, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) + + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return nil, DatabaseError(gs) + } + + if value == nil { + return nil, nil + } + + defer C.leveldb_free(unsafe.Pointer(value)) + return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil +} + +// Delete removes the data associated with the key from the database. +// +// The key byte slice may be reused safely. Delete takes a copy of +// them before returning. The WriteOptions passed in can be reused by +// multiple calls to this and if the WriteOptions is left unchanged. +func (db *DB) Delete(wo *WriteOptions, key []byte) error { + if db.closed { + panic(ErrDBClosed) + } + + var errStr *C.char + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + C.leveldb_delete( + db.Ldb, wo.Opt, k, C.size_t(len(key)), &errStr) + + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return DatabaseError(gs) + } + return nil +} + +// Write atomically writes a WriteBatch to disk. The WriteOptions +// passed in can be reused by multiple calls to this and other methods. +func (db *DB) Write(wo *WriteOptions, w *WriteBatch) error { + if db.closed { + panic(ErrDBClosed) + } + + var errStr *C.char + C.leveldb_write(db.Ldb, wo.Opt, w.wbatch, &errStr) + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return DatabaseError(gs) + } + return nil +} + +// NewIterator returns an Iterator over the the database that uses the +// ReadOptions given. +// +// Often, this is used for large, offline bulk reads while serving live +// traffic. In that case, it may be wise to disable caching so that the data +// processed by the returned Iterator does not displace the already cached +// data. This can be done by calling SetFillCache(false) on the ReadOptions +// before passing it here. +// +// Similarly, ReadOptions.SetSnapshot is also useful. +// +// The ReadOptions passed in can be reused by multiple calls to this +// and other methods if the ReadOptions is left unchanged. +func (db *DB) NewIterator(ro *ReadOptions) *Iterator { + if db.closed { + panic(ErrDBClosed) + } + + it := C.leveldb_create_iterator(db.Ldb, ro.Opt) + return &Iterator{Iter: it} +} + +// GetApproximateSizes returns the approximate number of bytes of file system +// space used by one or more key ranges. +// +// The keys counted will begin at Range.Start and end on the key before +// Range.Limit. +func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { + starts := make([]*C.char, len(ranges)) + limits := make([]*C.char, len(ranges)) + startLens := make([]C.size_t, len(ranges)) + limitLens := make([]C.size_t, len(ranges)) + for i, r := range ranges { + starts[i] = C.CString(string(r.Start)) + startLens[i] = C.size_t(len(r.Start)) + limits[i] = C.CString(string(r.Limit)) + limitLens[i] = C.size_t(len(r.Limit)) + } + sizes := make([]uint64, len(ranges)) + numranges := C.int(len(ranges)) + startsPtr := &starts[0] + limitsPtr := &limits[0] + startLensPtr := &startLens[0] + limitLensPtr := &limitLens[0] + sizesPtr := (*C.uint64_t)(&sizes[0]) + C.levigo_leveldb_approximate_sizes( + db.Ldb, numranges, startsPtr, startLensPtr, + limitsPtr, limitLensPtr, sizesPtr) + for i := range ranges { + C.free(unsafe.Pointer(starts[i])) + C.free(unsafe.Pointer(limits[i])) + } + return sizes +} + +// PropertyValue returns the value of a database property. +// +// Examples of properties include "leveldb.stats", "leveldb.sstables", +// and "leveldb.num-files-at-level0". +func (db *DB) PropertyValue(propName string) string { + if db.closed { + panic(ErrDBClosed) + } + + cname := C.CString(propName) + value := C.GoString(C.leveldb_property_value(db.Ldb, cname)) + C.free(unsafe.Pointer(cname)) + return value +} + +// NewSnapshot creates a new snapshot of the database. +// +// The Snapshot, when used in a ReadOptions, provides a consistent +// view of state of the database at the the snapshot was created. +// +// To prevent memory leaks and resource strain in the database, the snapshot +// returned must be released with DB.ReleaseSnapshot method on the DB that +// created it. +// +// See the LevelDB documentation for details. +func (db *DB) NewSnapshot() *Snapshot { + if db.closed { + panic(ErrDBClosed) + } + + return &Snapshot{C.leveldb_create_snapshot(db.Ldb)} +} + +// ReleaseSnapshot removes the snapshot from the database's list of snapshots, +// and deallocates it. +func (db *DB) ReleaseSnapshot(snap *Snapshot) { + if db.closed { + panic(ErrDBClosed) + } + + C.leveldb_release_snapshot(db.Ldb, snap.snap) +} + +// CompactRange runs a manual compaction on the Range of keys given. This is +// not likely to be needed for typical usage. +func (db *DB) CompactRange(r Range) { + if db.closed { + panic(ErrDBClosed) + } + + var start, limit *C.char + if len(r.Start) != 0 { + start = (*C.char)(unsafe.Pointer(&r.Start[0])) + } + if len(r.Limit) != 0 { + limit = (*C.char)(unsafe.Pointer(&r.Limit[0])) + } + C.leveldb_compact_range( + db.Ldb, start, C.size_t(len(r.Start)), limit, C.size_t(len(r.Limit))) +} + +// Close closes the database, rendering it unusable for I/O, by deallocating +// the underlying handle. +// +// Any attempts to use the DB after Close is called will panic. +func (db *DB) Close() { + if db.closed { + return + } + + db.closed = true + C.leveldb_close(db.Ldb) +} diff --git a/levigo/doc.go b/levigo/doc.go new file mode 100644 index 000000000..ac6b293dd --- /dev/null +++ b/levigo/doc.go @@ -0,0 +1,65 @@ +/* + +Package levigo provides the ability to create and access LevelDB databases. + +levigo.Open opens and creates databases. + + opts := levigo.NewOptions() + opts.SetCache(levigo.NewLRUCache(3<<30)) + opts.SetCreateIfMissing(true) + db, err := levigo.Open("/path/to/db", opts) + +The DB struct returned by Open provides DB.Get, DB.Put and DB.Delete to modify +and query the database. + + ro := levigo.NewReadOptions() + wo := levigo.NewWriteOptions() + // if ro and wo are not used again, be sure to Close them. + data, err := db.Get(ro, []byte("key")) + ... + err = db.Put(wo, []byte("anotherkey"), data) + ... + err = db.Delete(wo, []byte("key")) + +For bulk reads, use an Iterator. If you want to avoid disturbing your live +traffic while doing the bulk read, be sure to call SetFillCache(false) on the +ReadOptions you use when creating the Iterator. + + ro := levigo.NewReadOptions() + ro.SetFillCache(false) + it := db.NewIterator(ro) + defer it.Close() + for it.Seek(mykey); it.Valid(); it.Next() { + munge(it.Key(), it.Value()) + } + if err := it.GetError(); err != nil { + ... + } + +Batched, atomic writes can be performed with a WriteBatch and +DB.Write. + + wb := levigo.NewWriteBatch() + // defer wb.Close or use wb.Clear and reuse. + wb.Delete([]byte("removed")) + wb.Put([]byte("added"), []byte("data")) + wb.Put([]byte("anotheradded"), []byte("more")) + err := db.Write(wo, wb) + +If your working dataset does not fit in memory, you'll want to add a bloom +filter to your database. NewBloomFilter and Options.SetFilterPolicy is what +you want. NewBloomFilter is amount of bits in the filter to use per key in +your database. + + filter := levigo.NewBloomFilter(10) + opts.SetFilterPolicy(filter) + db, err := levigo.Open("/path/to/db", opts) + +If you're using a custom comparator in your code, be aware you may have to +make your own filter policy object. + +This documentation is not a complete discussion of LevelDB. Please read the +LevelDB documentation for information on +its operation. You'll find lots of goodies there. +*/ +package levigo diff --git a/levigo/env.go b/levigo/env.go new file mode 100644 index 000000000..af9f808ef --- /dev/null +++ b/levigo/env.go @@ -0,0 +1,29 @@ +package levigo + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +import "C" + +// Env is a system call environment used by a database. +// +// Typically, NewDefaultEnv is all you need. Advanced users may create their +// own Env with a *C.leveldb_env_t of their own creation. +// +// To prevent memory leaks, an Env must have Close called on it when it is +// no longer needed by the program. +type Env struct { + Env *C.leveldb_env_t +} + +// NewDefaultEnv creates a default environment for use in an Options. +// +// To prevent memory leaks, the Env returned should be deallocated with +// Close. +func NewDefaultEnv() *Env { + return &Env{C.leveldb_create_default_env()} +} + +// Close deallocates the Env, freeing the underlying struct. +func (env *Env) Close() { + C.leveldb_env_destroy(env.Env) +} diff --git a/levigo/examples/comparator_example.go b/levigo/examples/comparator_example.go new file mode 100644 index 000000000..1c1b0f2f1 --- /dev/null +++ b/levigo/examples/comparator_example.go @@ -0,0 +1,46 @@ +package main + +/* +#cgo LDFLAGS: -lleveldb +#include +#include + +static void CmpDestroy(void* arg) { } + +static int CmpCompare(void* arg, const char* a, size_t alen, + const char* b, size_t blen) { + int n = (alen < blen) ? alen : blen; + int r = memcmp(a, b, n); + if (r == 0) { + if (alen < blen) r = -1; + else if (alen > blen) r = +1; + } + return r; +} + +static const char* CmpName(void* arg) { + return "foo"; +} + +static leveldb_comparator_t* CmpFooNew() { + return leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); +} + +*/ +import "C" + +type Comparator struct { + Comparator *C.leveldb_comparator_t +} + +func NewFooComparator() *Comparator { + return &Comparator{C.CmpFooNew()} +} + +func (cmp *Comparator) Close() { + C.leveldb_comparator_destroy(cmp.Comparator) +} + +func main() { + NewFooComparator().Close() +} diff --git a/levigo/filterpolicy.go b/levigo/filterpolicy.go new file mode 100644 index 000000000..a6cf48c80 --- /dev/null +++ b/levigo/filterpolicy.go @@ -0,0 +1,33 @@ +package levigo + +// #cgo LDFLAGS: -lleveldb +// #include +// #include "leveldb/c.h" +import "C" + +// FilterPolicy is a factory type that allows the LevelDB database to create a +// filter, such as a bloom filter, that is stored in the sstables and used by +// DB.Get to reduce reads. +// +// An instance of this struct may be supplied to Options when opening a +// DB. Typical usage is to call NewBloomFilter to get an instance. +// +// To prevent memory leaks, a FilterPolicy must have Close called on it when +// it is no longer needed by the program. +type FilterPolicy struct { + Policy *C.leveldb_filterpolicy_t +} + +// NewBloomFilter creates a filter policy that will create a bloom filter when +// necessary with the given number of bits per key. +// +// See the FilterPolicy documentation for more. +func NewBloomFilter(bitsPerKey int) *FilterPolicy { + policy := C.leveldb_filterpolicy_create_bloom(C.int(bitsPerKey)) + return &FilterPolicy{policy} +} + +// Close reaps the resources associated with this FilterPolicy. +func (fp *FilterPolicy) Close() { + C.leveldb_filterpolicy_destroy(fp.Policy) +} diff --git a/levigo/go.mod b/levigo/go.mod new file mode 100644 index 000000000..95d9fc3f7 --- /dev/null +++ b/levigo/go.mod @@ -0,0 +1,3 @@ +module github.com/jmhodges/levigo + +go 1.13 diff --git a/levigo/go.sum b/levigo/go.sum new file mode 100644 index 000000000..e69de29bb diff --git a/levigo/iterator.go b/levigo/iterator.go new file mode 100644 index 000000000..ce8091dd8 --- /dev/null +++ b/levigo/iterator.go @@ -0,0 +1,150 @@ +package levigo + +// #cgo LDFLAGS: -lleveldb +// #include +// #include "leveldb/c.h" +import "C" + +import ( + "unsafe" +) + +// IteratorError wraps general internal LevelDB iterator errors for user consumption. +type IteratorError string + +func (e IteratorError) Error() string { + return "levigo: " + string(e) +} + +// Iterator is a read-only iterator through a LevelDB database. It provides a +// way to seek to specific keys and iterate through the keyspace from that +// point, as well as access the values of those keys. +// +// Care must be taken when using an Iterator. If the method Valid returns +// false, calls to Key, Value, Next, and Prev will result in panics. However, +// Seek, SeekToFirst, SeekToLast, GetError, Valid, and Close will still be +// safe to call. +// +// GetError will only return an error in the event of a LevelDB error. It will +// return a nil on iterators that are simply invalid. Given that behavior, +// GetError is not a replacement for a Valid. +// +// A typical use looks like: +// +// db := levigo.Open(...) +// +// it := db.NewIterator(readOpts) +// defer it.Close() +// for it.Seek(mykey); it.Valid(); it.Next() { +// useKeyAndValue(it.Key(), it.Value()) +// } +// if err := it.GetError() { +// ... +// } +// +// To prevent memory leaks, an Iterator must have Close called on it when it +// is no longer needed by the program. +type Iterator struct { + Iter *C.leveldb_iterator_t +} + +// Valid returns false only when an Iterator has iterated past either the +// first or the last key in the database. +func (it *Iterator) Valid() bool { + return ucharToBool(C.leveldb_iter_valid(it.Iter)) +} + +// Key returns a copy the key in the database the iterator currently holds. +// +// If Valid returns false, this method will panic. +func (it *Iterator) Key() []byte { + var klen C.size_t + kdata := C.leveldb_iter_key(it.Iter, &klen) + if kdata == nil { + return nil + } + // Unlike DB.Get, the key, kdata, returned is not meant to be freed by the + // client. It's a direct reference to data managed by the iterator_t + // instead of a copy. So, we must not free it here but simply copy it + // with GoBytes. + return C.GoBytes(unsafe.Pointer(kdata), C.int(klen)) +} + +// Value returns a copy of the value in the database the iterator currently +// holds. +// +// If Valid returns false, this method will panic. +func (it *Iterator) Value() []byte { + var vlen C.size_t + vdata := C.leveldb_iter_value(it.Iter, &vlen) + if vdata == nil { + return nil + } + // Unlike DB.Get, the value, vdata, returned is not meant to be freed by + // the client. It's a direct reference to data managed by the iterator_t + // instead of a copy. So, we must not free it here but simply copy it with + // GoBytes. + return C.GoBytes(unsafe.Pointer(vdata), C.int(vlen)) +} + +// Next moves the iterator to the next sequential key in the database, as +// defined by the Comparator in the ReadOptions used to create this Iterator. +// +// If Valid returns false, this method will panic. +func (it *Iterator) Next() { + C.leveldb_iter_next(it.Iter) +} + +// Prev moves the iterator to the previous sequential key in the database, as +// defined by the Comparator in the ReadOptions used to create this Iterator. +// +// If Valid returns false, this method will panic. +func (it *Iterator) Prev() { + C.leveldb_iter_prev(it.Iter) +} + +// SeekToFirst moves the iterator to the first key in the database, as defined +// by the Comparator in the ReadOptions used to create this Iterator. +// +// This method is safe to call when Valid returns false. +func (it *Iterator) SeekToFirst() { + C.leveldb_iter_seek_to_first(it.Iter) +} + +// SeekToLast moves the iterator to the last key in the database, as defined +// by the Comparator in the ReadOptions used to create this Iterator. +// +// This method is safe to call when Valid returns false. +func (it *Iterator) SeekToLast() { + C.leveldb_iter_seek_to_last(it.Iter) +} + +// Seek moves the iterator the position of the key given or, if the key +// doesn't exist, the next key that does exist in the database. If the key +// doesn't exist, and there is no next key, the Iterator becomes invalid. +// +// This method is safe to call when Valid returns false. +func (it *Iterator) Seek(key []byte) { + C.leveldb_iter_seek(it.Iter, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) +} + +// GetError returns an IteratorError from LevelDB if it had one during +// iteration. +// +// This method is safe to call when Valid returns false. +func (it *Iterator) GetError() error { + var errStr *C.char + C.leveldb_iter_get_error(it.Iter, &errStr) + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return IteratorError(gs) + } + return nil +} + +// Close deallocates the given Iterator, freeing the underlying C struct. +func (it *Iterator) Close() { + C.leveldb_iter_destroy(it.Iter) + it.Iter = nil +} diff --git a/levigo/leveldb_test.go b/levigo/leveldb_test.go new file mode 100644 index 000000000..f4537fdc5 --- /dev/null +++ b/levigo/leveldb_test.go @@ -0,0 +1,359 @@ +package levigo + +import ( + "bytes" + "fmt" + "math/rand" + "os" + "path/filepath" + "testing" + "time" +) + +func init() { + rand.Seed(int64(time.Now().Nanosecond())) +} + +// This testcase is a port of leveldb's c_test.c. +func TestC(t *testing.T) { + if GetLevelDBMajorVersion() <= 0 { + t.Errorf("Major version cannot be less than zero") + } + + dbname := tempDir(t) + defer deleteDBDirectory(t, dbname) + env := NewDefaultEnv() + cache := NewLRUCache(1 << 20) + + options := NewOptions() + // options.SetComparator(cmp) + options.SetErrorIfExists(true) + options.SetCache(cache) + options.SetEnv(env) + options.SetInfoLog(nil) + options.SetWriteBufferSize(1 << 20) + options.SetParanoidChecks(true) + options.SetMaxOpenFiles(10) + options.SetBlockSize(1024) + options.SetBlockRestartInterval(8) + options.SetCompression(NoCompression) + + roptions := NewReadOptions() + roptions.SetVerifyChecksums(true) + roptions.SetFillCache(false) + + woptions := NewWriteOptions() + woptions.SetSync(true) + + _ = DestroyDatabase(dbname, options) + + db, err := Open(dbname, options) + if err == nil { + t.Errorf("Open on missing db should have failed") + } + + options.SetCreateIfMissing(true) + db, err = Open(dbname, options) + if err != nil { + t.Fatalf("Open failed: %v", err) + } + + putKey := []byte("foo") + putValue := []byte("hello") + err = db.Put(woptions, putKey, putValue) + if err != nil { + t.Errorf("Put failed: %v", err) + } + + CheckGet(t, "after Put", db, roptions, putKey, putValue) + + wb := NewWriteBatch() + wb.Put([]byte("foo"), []byte("a")) + wb.Clear() + wb.Put([]byte("bar"), []byte("b")) + wb.Put([]byte("box"), []byte("c")) + wb.Delete([]byte("bar")) + err = db.Write(woptions, wb) + if err != nil { + t.Errorf("Write batch failed: %v", err) + } + CheckGet(t, "after WriteBatch", db, roptions, []byte("foo"), []byte("hello")) + CheckGet(t, "after WriteBatch", db, roptions, []byte("bar"), nil) + CheckGet(t, "after WriteBatch", db, roptions, []byte("box"), []byte("c")) + // TODO: WriteBatch iteration isn't easy. Suffers same problems as + // Comparator. + // wbiter := &TestWBIter{t: t} + // wb.Iterate(wbiter) + // if wbiter.pos != 3 { + // t.Errorf("After Iterate, on the wrong pos: %d", wbiter.pos) + // } + wb.Close() + + iter := db.NewIterator(roptions) + if iter.Valid() { + t.Errorf("Read iterator should not be valid, yet") + } + iter.SeekToFirst() + if !iter.Valid() { + t.Errorf("Read iterator should be valid after seeking to first record") + } + CheckIter(t, iter, []byte("box"), []byte("c")) + iter.Next() + CheckIter(t, iter, []byte("foo"), []byte("hello")) + iter.Prev() + CheckIter(t, iter, []byte("box"), []byte("c")) + iter.Prev() + if iter.Valid() { + t.Errorf("Read iterator should not be valid after go back past the first record") + } + iter.SeekToLast() + CheckIter(t, iter, []byte("foo"), []byte("hello")) + iter.Seek([]byte("b")) + CheckIter(t, iter, []byte("box"), []byte("c")) + if iter.GetError() != nil { + t.Errorf("Read iterator has an error we didn't expect: %v", iter.GetError()) + } + iter.Close() + + // approximate sizes + n := 20000 + for i := 0; i < n; i++ { + keybuf := []byte(fmt.Sprintf("k%020d", i)) + valbuf := []byte(fmt.Sprintf("v%020d", i)) + err := db.Put(woptions, keybuf, valbuf) + if err != nil { + t.Errorf("Put error in approximate size test: %v", err) + } + } + + ranges := []Range{ + {[]byte("a"), []byte("k00000000000000010000")}, + {[]byte("k00000000000000010000"), []byte("z")}, + } + sizes := db.GetApproximateSizes(ranges) + if len(sizes) == 2 { + if sizes[0] <= 0 { + t.Errorf("First size range was %d", sizes[0]) + } + if sizes[1] <= 0 { + t.Errorf("Second size range was %d", sizes[1]) + } + } else { + t.Errorf("Expected 2 approx. sizes back, got %d", len(sizes)) + } + + // property + prop := db.PropertyValue("nosuchprop") + if prop != "" { + t.Errorf("property nosuchprop should not have a value") + } + prop = db.PropertyValue("leveldb.stats") + if prop == "" { + t.Errorf("property leveldb.stats should have a value") + } + + // snapshot + snap := db.NewSnapshot() + err = db.Delete(woptions, []byte("foo")) + if err != nil { + t.Errorf("Delete during snapshot test errored: %v", err) + } + roptions.SetSnapshot(snap) + CheckGet(t, "from snapshot", db, roptions, []byte("foo"), []byte("hello")) + roptions.SetSnapshot(nil) + CheckGet(t, "from snapshot", db, roptions, []byte("foo"), nil) + db.ReleaseSnapshot(snap) + + // repair + db.Close() + options.SetCreateIfMissing(false) + options.SetErrorIfExists(false) + err = RepairDatabase(dbname, options) + if err != nil { + t.Errorf("Repairing db failed: %v", err) + } + db, err = Open(dbname, options) + if err != nil { + t.Errorf("Unable to open repaired db: %v", err) + } + CheckGet(t, "repair", db, roptions, []byte("foo"), nil) + CheckGet(t, "repair", db, roptions, []byte("bar"), nil) + CheckGet(t, "repair", db, roptions, []byte("box"), []byte("c")) + options.SetCreateIfMissing(true) + options.SetErrorIfExists(true) + + // filter + policy := NewBloomFilter(10) + db.Close() + DestroyDatabase(dbname, options) + options.SetFilterPolicy(policy) + db, err = Open(dbname, options) + if err != nil { + t.Fatalf("Unable to recreate db for filter tests: %v", err) + } + err = db.Put(woptions, []byte("foo"), []byte("foovalue")) + if err != nil { + t.Errorf("Unable to put 'foo' with filter: %v", err) + } + err = db.Put(woptions, []byte("bar"), []byte("barvalue")) + if err != nil { + t.Errorf("Unable to put 'bar' with filter: %v", err) + } + db.CompactRange(Range{nil, nil}) + CheckGet(t, "filter", db, roptions, []byte("foo"), []byte("foovalue")) + CheckGet(t, "filter", db, roptions, []byte("bar"), []byte("barvalue")) + options.SetFilterPolicy(nil) + policy.Close() + + // cleanup + db.Close() + options.Close() + roptions.Close() + woptions.Close() + cache.Close() + // DestroyComparator(cmp) + env.Close() +} + +func TestNilSlicesInDb(t *testing.T) { + dbname := tempDir(t) + defer deleteDBDirectory(t, dbname) + options := NewOptions() + options.SetErrorIfExists(true) + options.SetCreateIfMissing(true) + ro := NewReadOptions() + _ = DestroyDatabase(dbname, options) + db, err := Open(dbname, options) + if err != nil { + t.Fatalf("Database could not be opened: %v", err) + } + defer db.Close() + val, err := db.Get(ro, []byte("missing")) + if err != nil { + t.Errorf("Get failed: %v", err) + } + if val != nil { + t.Errorf("A key not in the db should return nil, not %v", val) + } + wo := NewWriteOptions() + db.Put(wo, nil, []byte("love")) + val, err = db.Get(ro, nil) + if !bytes.Equal([]byte("love"), val) { + t.Errorf("Get should see the nil key: %v", val) + } + val, err = db.Get(ro, []byte{}) + if !bytes.Equal([]byte("love"), val) { + t.Errorf("Get shouldn't distinguish between nil key and empty slice key: %v", val) + } + + err = db.Put(wo, []byte("nilvalue"), nil) + if err != nil { + t.Errorf("nil value Put errored: %v", err) + } + // Compare with the []byte("missing") case. We expect Get to return a + // []byte{} here, but expect a nil returned there. + CheckGet(t, "nil value Put", db, ro, []byte("nilvalue"), []byte{}) + + err = db.Put(wo, []byte("emptyvalue"), []byte{}) + if err != nil { + t.Errorf("empty value Put errored: %v", err) + } + CheckGet(t, "empty value Put", db, ro, []byte("emptyvalue"), []byte{}) + + err = db.Delete(wo, nil) + if err != nil { + t.Errorf("nil key Delete errored: %v", err) + } + err = db.Delete(wo, []byte{}) + if err != nil { + t.Errorf("empty slice key Delete errored: %v", err) + } + +} + +func TestIterationValidityLimits(t *testing.T) { + dbname := tempDir(t) + defer deleteDBDirectory(t, dbname) + options := NewOptions() + options.SetErrorIfExists(true) + options.SetCreateIfMissing(true) + ro := NewReadOptions() + wo := NewWriteOptions() + _ = DestroyDatabase(dbname, options) + db, err := Open(dbname, options) + if err != nil { + t.Fatalf("Database could not be opened: %v", err) + } + defer db.Close() + db.Put(wo, []byte("bat"), []byte("somedata")) + db.Put(wo, []byte("done"), []byte("somedata")) + it := db.NewIterator(ro) + defer it.Close() + if it.Valid() { + t.Errorf("new Iterator was valid") + } + it.Seek([]byte("bat")) + if !it.Valid() { + t.Errorf("Seek to %#v failed.", []byte("bat")) + } + if !bytes.Equal([]byte("bat"), it.Key()) { + t.Errorf("did not seek to []byte(\"bat\")") + } + key := it.Key() + it.Next() + if bytes.Equal(key, it.Key()) { + t.Errorf("key should be a copy of last key") + } + it.Next() + if it.Valid() { + t.Errorf("iterating off the db should result in an invalid iterator") + } + err = it.GetError() + if err != nil { + t.Errorf("should not have seen an error on an invalid iterator") + } + it.Seek([]byte("bat")) + if !it.Valid() { + t.Errorf("Iterator should be valid again") + } +} + +func CheckGet(t *testing.T, where string, db *DB, roptions *ReadOptions, key, expected []byte) { + getValue, err := db.Get(roptions, key) + + if err != nil { + t.Errorf("%s, Get failed: %v", where, err) + } + if !bytes.Equal(getValue, expected) { + t.Errorf("%s, expected Get value %v, got %v", where, expected, getValue) + } +} + +func WBIterCheckEqual(t *testing.T, where string, which string, pos int, expected, given []byte) { + if !bytes.Equal(expected, given) { + t.Errorf("%s at pos %d, %s expected: %v, got: %v", where, pos, which, expected, given) + } +} + +func CheckIter(t *testing.T, it *Iterator, key, value []byte) { + if !bytes.Equal(key, it.Key()) { + t.Errorf("Iterator: expected key %v, got %v", key, it.Key()) + } + if !bytes.Equal(value, it.Value()) { + t.Errorf("Iterator: expected value %v, got %v", value, it.Value()) + } +} + +func deleteDBDirectory(t *testing.T, dirPath string) { + err := os.RemoveAll(dirPath) + if err != nil { + t.Errorf("Unable to remove database directory: %s", dirPath) + } +} + +func tempDir(t *testing.T) string { + bottom := fmt.Sprintf("levigo-test-%d", rand.Int()) + path := filepath.Join(os.TempDir(), bottom) + deleteDBDirectory(t, path) + return path +} diff --git a/levigo/levigo.sh b/levigo/levigo.sh new file mode 100755 index 000000000..9008d5fcd --- /dev/null +++ b/levigo/levigo.sh @@ -0,0 +1,51 @@ +#!/bin/bash +#refer https://github.com/norton/lets/blob/master/c_src/build_deps.sh + +#你必须在这里设置实际的snappy以及leveldb源代码地址 +SNAPPY_SRC=/home/kvgroup/QingyangZ/kv/snappy +LEVELDB_SRC=/home/kvgroup/QingyangZ/kv/leveldb + +SNAPPY_DIR=/home/kvgroup/QingyangZ/kv/snappy/build +LEVELDB_DIR=/home/kvgroup/QingyangZ/kv/leveldb/build + +if [ ! -f $SNAPPY_DIR/libsnappy.a ]; then + (cd $SNAPPY_SRC && \ + ./configure --prefix=$SNAPPY_DIR && \ + make && \ + make install) +else + echo "skip install snappy" +fi + +if [ ! -f $LEVELDB_DIR/libleveldb.so ]; then + (cd $LEVELDB_SRC && \ + echo "echo \"PLATFORM_CFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform && + echo "echo \"PLATFORM_CXXFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform && + echo "echo \"PLATFORM_LDFLAGS+=-L $SNAPPY_DIR/lib -lsnappy\" >> build_config.mk" >> build_detect_platform && + make SNAPPY=1 && \ + make && \ + mkdir -p $LEVELDB_DIR/include/leveldb && \ + install include/leveldb/*.h $LEVELDB_DIR/include/leveldb && \ + mkdir -p $LEVELDB_DIR/lib && \ + cp -af libleveldb.* $LEVELDB_DIR/lib) +else + echo "skip install leveldb" +fi + +function add_path() +{ + # $1 path variable + # $2 path to add + if [ -d "$2" ] && [[ ":$1:" != *":$2:"* ]]; then + echo "$1:$2" + else + echo "$1" + fi +} + +export CGO_CFLAGS="-I$LEVELDB_DIR/include -I$SNAPPY_DIR/include" +export CGO_LDFLAGS="-L$LEVELDB_DIR -L$SNAPPY_DIR -lsnappy" +export LD_LIBRARY_PATH=$(add_path $LD_LIBRARY_PATH $SNAPPY_DIR/lib) +export LD_LIBRARY_PATH=$(add_path $LD_LIBRARY_PATH $LEVELDB_DIR/lib) + +go get github.com/jmhodges/levigo diff --git a/levigo/options.go b/levigo/options.go new file mode 100644 index 000000000..dae187b21 --- /dev/null +++ b/levigo/options.go @@ -0,0 +1,228 @@ +package levigo + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +import "C" + +// CompressionOpt is a value for Options.SetCompression. +type CompressionOpt int + +// Known compression arguments for Options.SetCompression. +const ( + NoCompression = CompressionOpt(0) + SnappyCompression = CompressionOpt(1) +) + +// Options represent all of the available options when opening a database with +// Open. Options should be created with NewOptions. +// +// It is usually with to call SetCache with a cache object. Otherwise, all +// data will be read off disk. +// +// To prevent memory leaks, Close must be called on an Options when the +// program no longer needs it. +type Options struct { + Opt *C.leveldb_options_t +} + +// ReadOptions represent all of the available options when reading from a +// database. +// +// To prevent memory leaks, Close must called on a ReadOptions when the +// program no longer needs it. +type ReadOptions struct { + Opt *C.leveldb_readoptions_t +} + +// WriteOptions represent all of the available options when writing from a +// database. +// +// To prevent memory leaks, Close must called on a WriteOptions when the +// program no longer needs it. +type WriteOptions struct { + Opt *C.leveldb_writeoptions_t +} + +// NewOptions allocates a new Options object. +func NewOptions() *Options { + opt := C.leveldb_options_create() + return &Options{opt} +} + +// NewReadOptions allocates a new ReadOptions object. +func NewReadOptions() *ReadOptions { + opt := C.leveldb_readoptions_create() + return &ReadOptions{opt} +} + +// NewWriteOptions allocates a new WriteOptions object. +func NewWriteOptions() *WriteOptions { + opt := C.leveldb_writeoptions_create() + return &WriteOptions{opt} +} + +// Close deallocates the Options, freeing its underlying C struct. +func (o *Options) Close() { + C.leveldb_options_destroy(o.Opt) +} + +// SetComparator sets the comparator to be used for all read and write +// operations. +// +// The comparator that created a database must be the same one (technically, +// one with the same name string) that is used to perform read and write +// operations. +// +// The default comparator is usually sufficient. +func (o *Options) SetComparator(cmp *C.leveldb_comparator_t) { + C.leveldb_options_set_comparator(o.Opt, cmp) +} + +// SetErrorIfExists causes the opening of a database that already exists to +// throw an error if true. +func (o *Options) SetErrorIfExists(errorIfExists bool) { + eie := boolToUchar(errorIfExists) + C.leveldb_options_set_error_if_exists(o.Opt, eie) +} + +// SetCache places a cache object in the database when a database is opened. +// +// This is usually wise to use. See also ReadOptions.SetFillCache. +func (o *Options) SetCache(cache *Cache) { + C.leveldb_options_set_cache(o.Opt, cache.Cache) +} + +// SetEnv sets the Env object for the new database handle. +func (o *Options) SetEnv(env *Env) { + C.leveldb_options_set_env(o.Opt, env.Env) +} + +// SetInfoLog sets a *C.leveldb_logger_t object as the informational logger +// for the database. +func (o *Options) SetInfoLog(log *C.leveldb_logger_t) { + C.leveldb_options_set_info_log(o.Opt, log) +} + +// SetWriteBufferSize sets the number of bytes the database will build up in +// memory (backed by an unsorted log on disk) before converting to a sorted +// on-disk file. +func (o *Options) SetWriteBufferSize(s int) { + C.leveldb_options_set_write_buffer_size(o.Opt, C.size_t(s)) +} + +// SetParanoidChecks causes the database to do aggressive checking of the data +// it is processing and will stop early if it detects errors if true. +// +// See the LevelDB documentation docs for details. +func (o *Options) SetParanoidChecks(pc bool) { + C.leveldb_options_set_paranoid_checks(o.Opt, boolToUchar(pc)) +} + +// SetMaxOpenFiles sets the number of files than can be used at once by the +// database. +// +// See the LevelDB documentation for details. +func (o *Options) SetMaxOpenFiles(n int) { + C.leveldb_options_set_max_open_files(o.Opt, C.int(n)) +} + +// SetBlockSize sets the approximate size of user data packed per block. +// +// The default is roughly 4096 uncompressed bytes. A better setting depends on +// your use case. See the LevelDB documentation for details. +func (o *Options) SetBlockSize(s int) { + C.leveldb_options_set_block_size(o.Opt, C.size_t(s)) +} + +// SetBlockRestartInterval is the number of keys between restarts points for +// delta encoding keys. +// +// Most clients should leave this parameter alone. See the LevelDB +// documentation for details. +func (o *Options) SetBlockRestartInterval(n int) { + C.leveldb_options_set_block_restart_interval(o.Opt, C.int(n)) +} + +// SetCompression sets whether to compress blocks using the specified +// compresssion algorithm. +// +// The default value is SnappyCompression and it is fast enough that it is +// unlikely you want to turn it off. The other option is NoCompression. +// +// If the LevelDB library was built without Snappy compression enabled, the +// SnappyCompression setting will be ignored. +func (o *Options) SetCompression(t CompressionOpt) { + C.leveldb_options_set_compression(o.Opt, C.int(t)) +} + +// SetCreateIfMissing causes Open to create a new database on disk if it does +// not already exist. +func (o *Options) SetCreateIfMissing(b bool) { + C.leveldb_options_set_create_if_missing(o.Opt, boolToUchar(b)) +} + +// SetFilterPolicy causes Open to create a new database that will uses filter +// created from the filter policy passed in. +func (o *Options) SetFilterPolicy(fp *FilterPolicy) { + var policy *C.leveldb_filterpolicy_t + if fp != nil { + policy = fp.Policy + } + C.leveldb_options_set_filter_policy(o.Opt, policy) +} + +// Close deallocates the ReadOptions, freeing its underlying C struct. +func (ro *ReadOptions) Close() { + C.leveldb_readoptions_destroy(ro.Opt) +} + +// SetVerifyChecksums controls whether all data read with this ReadOptions +// will be verified against corresponding checksums. +// +// It defaults to false. See the LevelDB documentation for details. +func (ro *ReadOptions) SetVerifyChecksums(b bool) { + C.leveldb_readoptions_set_verify_checksums(ro.Opt, boolToUchar(b)) +} + +// SetFillCache controls whether reads performed with this ReadOptions will +// fill the Cache of the server. It defaults to true. +// +// It is useful to turn this off on ReadOptions for DB.Iterator (and DB.Get) +// calls used in offline threads to prevent bulk scans from flushing out live +// user data in the cache. +// +// See also Options.SetCache +func (ro *ReadOptions) SetFillCache(b bool) { + C.leveldb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b)) +} + +// SetSnapshot causes reads to provided as they were when the passed in +// Snapshot was created by DB.NewSnapshot. This is useful for getting +// consistent reads during a bulk operation. +// +// See the LevelDB documentation for details. +func (ro *ReadOptions) SetSnapshot(snap *Snapshot) { + var s *C.leveldb_snapshot_t + if snap != nil { + s = snap.snap + } + C.leveldb_readoptions_set_snapshot(ro.Opt, s) +} + +// Close deallocates the WriteOptions, freeing its underlying C struct. +func (wo *WriteOptions) Close() { + C.leveldb_writeoptions_destroy(wo.Opt) +} + +// SetSync controls whether each write performed with this WriteOptions will +// be flushed from the operating system buffer cache before the write is +// considered complete. +// +// If called with true, this will significantly slow down writes. If called +// with false, and the host machine crashes, some recent writes may be +// lost. The default is false. +// +// See the LevelDB documentation for details. +func (wo *WriteOptions) SetSync(b bool) { + C.leveldb_writeoptions_set_sync(wo.Opt, boolToUchar(b)) +} diff --git a/levigo/version.go b/levigo/version.go new file mode 100644 index 000000000..58e25d1b7 --- /dev/null +++ b/levigo/version.go @@ -0,0 +1,19 @@ +package levigo + +/* +#cgo LDFLAGS: -lleveldb +#include "leveldb/c.h" +*/ +import "C" + +// GetLevelDBMajorVersion returns the underlying LevelDB implementation's major +// version. +func GetLevelDBMajorVersion() int { + return int(C.leveldb_major_version()) +} + +// GetLevelDBMinorVersion returns the underlying LevelDB implementation's minor +// version. +func GetLevelDBMinorVersion() int { + return int(C.leveldb_minor_version()) +} From c5c1142837e059e5848f261f21ee7dfbd02fe355 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Wed, 14 Dec 2022 22:19:47 +0800 Subject: [PATCH 03/22] debug --- kv/storage/standalone_storage/standalone_storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kv/storage/standalone_storage/standalone_storage.go b/kv/storage/standalone_storage/standalone_storage.go index bc5b7422e..6a939cb39 100644 --- a/kv/storage/standalone_storage/standalone_storage.go +++ b/kv/storage/standalone_storage/standalone_storage.go @@ -21,7 +21,7 @@ type StandAloneStorage struct { func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { // Your Code Here (1). - dbName := "./dbtest1" + dbName := conf.DBPath opt := levigo.NewOptions() ropt := levigo.NewReadOptions() wopt := levigo.NewWriteOptions() From 3740d06eedf32a523bf5fa031b007e4867e9a6c2 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Mon, 19 Dec 2022 17:19:34 +0800 Subject: [PATCH 04/22] add gorocksdb --- go.mod | 5 +- go.sum | 16 +- gorocksdb | 1 + kv/server/server_test.go | 2 +- .../standalone_storage/standalone_storage.go | 2 +- .../standalone_storage.go | 160 ++++++++++++++++++ kv/util/engine_util/cf_iterator.go | 40 +++++ 7 files changed, 222 insertions(+), 4 deletions(-) create mode 160000 gorocksdb create mode 100644 kv/storage/standalone_storage_rdb/standalone_storage.go diff --git a/go.mod b/go.mod index eb3d3c19e..84a18d133 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,8 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible github.com/sirupsen/logrus v1.2.0 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.8.1 + github.com/tecbot/gorocksdb v0.0.0-00010101000000-000000000000 github.com/yusufpapurcu/wmi v1.2.2 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.uber.org/zap v1.14.0 @@ -39,3 +40,5 @@ go 1.13 replace github.com/pingcap/tidb => github.com/pingcap-incubator/tinysql v0.0.0-20200518090433-a7d00f9e6aa7 replace github.com/jmhodges/levigo => ./levigo + +replace github.com/tecbot/gorocksdb => ./gorocksdb diff --git a/go.sum b/go.sum index 4c7736a8e..9c8ac7596 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,12 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -254,10 +260,15 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -411,6 +422,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= diff --git a/gorocksdb b/gorocksdb new file mode 160000 index 000000000..f0fad39f3 --- /dev/null +++ b/gorocksdb @@ -0,0 +1 @@ +Subproject commit f0fad39f321c61bf05fcb4a01fe259606235e712 diff --git a/kv/server/server_test.go b/kv/server/server_test.go index 0c2cf10d0..5ccb36c4f 100644 --- a/kv/server/server_test.go +++ b/kv/server/server_test.go @@ -6,7 +6,7 @@ import ( "github.com/pingcap-incubator/tinykv/kv/config" "github.com/pingcap-incubator/tinykv/kv/storage" - "github.com/pingcap-incubator/tinykv/kv/storage/standalone_storage" + standalone_storage "github.com/pingcap-incubator/tinykv/kv/storage/standalone_storage" "github.com/pingcap-incubator/tinykv/kv/util/engine_util" "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" "github.com/stretchr/testify/assert" diff --git a/kv/storage/standalone_storage/standalone_storage.go b/kv/storage/standalone_storage/standalone_storage.go index 6a939cb39..5936894b8 100644 --- a/kv/storage/standalone_storage/standalone_storage.go +++ b/kv/storage/standalone_storage/standalone_storage.go @@ -1,4 +1,4 @@ -package standalone_storage +package standalone_storage_ldb import ( "fmt" diff --git a/kv/storage/standalone_storage_rdb/standalone_storage.go b/kv/storage/standalone_storage_rdb/standalone_storage.go new file mode 100644 index 000000000..8ab8a42d2 --- /dev/null +++ b/kv/storage/standalone_storage_rdb/standalone_storage.go @@ -0,0 +1,160 @@ +package standalone_storage + +import ( + "errors" + "log" + + "github.com/tecbot/gorocksdb" + + "github.com/pingcap-incubator/tinykv/kv/config" + "github.com/pingcap-incubator/tinykv/kv/storage" + "github.com/pingcap-incubator/tinykv/kv/util/engine_util" + "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" +) + +// StandAloneStorage is an implementation of `Storage` for a single-node TinyKV instance. It does not +// communicate with other nodes and all data is stored locally. +type StandAloneStorage struct { + // Your Data Here (1). + db *gorocksdb.DB + roptions *gorocksdb.ReadOptions + woptions *gorocksdb.WriteOptions +} + +// opendb +func OpenDB(conf *config.Config) (*gorocksdb.DB, error) { + options := gorocksdb.NewDefaultOptions() + options.SetCreateIfMissing(true) + + bloomFilter := gorocksdb.NewBloomFilter(10) + + readOptions := gorocksdb.NewDefaultReadOptions() + readOptions.SetFillCache(false) + + rateLimiter := gorocksdb.NewRateLimiter(10000000, 10000, 10) + options.SetRateLimiter(rateLimiter) + options.SetCreateIfMissing(true) + options.EnableStatistics() + options.SetWriteBufferSize(8 * 1024) + options.SetMaxWriteBufferNumber(3) + options.SetMaxBackgroundCompactions(10) + // options.SetCompression(gorocksdb.SnappyCompression) + // options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle) + + options.SetHashSkipListRep(2000000, 4, 4) + + blockBasedTableOptions := gorocksdb.NewDefaultBlockBasedTableOptions() + blockBasedTableOptions.SetBlockCache(gorocksdb.NewLRUCache(64 * 1024)) + blockBasedTableOptions.SetFilterPolicy(bloomFilter) + blockBasedTableOptions.SetBlockSizeDeviation(5) + blockBasedTableOptions.SetBlockRestartInterval(10) + blockBasedTableOptions.SetBlockCacheCompressed(gorocksdb.NewLRUCache(64 * 1024)) + blockBasedTableOptions.SetCacheIndexAndFilterBlocks(true) + blockBasedTableOptions.SetIndexType(gorocksdb.KHashSearchIndexType) + + options.SetBlockBasedTableFactory(blockBasedTableOptions) + //log.Println(bloomFilter, readOptions) + options.SetPrefixExtractor(gorocksdb.NewFixedPrefixTransform(3)) + + options.SetAllowConcurrentMemtableWrites(false) + + db, err := gorocksdb.OpenDb(options, conf.DBPath) + + if err != nil { + log.Fatalln("OPEN DB error", db, err) + db.Close() + return nil, errors.New("fail to open db") + } else { + log.Println("OPEN DB success", db) + } + return db, nil +} + +func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { + // Your Code Here (1). + db, err := OpenDB(conf) + + if err != nil { + log.Println("fail to open db,", nil, db) + } + + ropt := gorocksdb.NewDefaultReadOptions() + wopt := gorocksdb.NewDefaultWriteOptions() + s := &StandAloneStorage{ + db, + ropt, + wopt, + } + return s +} + +func (s *StandAloneStorage) Start() error { + // Your Code Here (1). + return nil +} + +func (s *StandAloneStorage) Stop() error { + // Your Code Here (1). + s.db.Close() + return nil +} + +func (s *StandAloneStorage) Reader(ctx *kvrpcpb.Context) (storage.StorageReader, error) { + // Your Code Here (1). + return &StandAloneStorageReader{ + s.db, + s.roptions, + }, nil +} + +func (s *StandAloneStorage) Write(ctx *kvrpcpb.Context, batch []storage.Modify) error { + // Your Code Here (1). + for _, m := range batch { + switch m.Data.(type) { + case storage.Put: + put := m.Data.(storage.Put) + if err := s.db.Put(s.woptions, engine_util.KeyWithCF(put.Cf, put.Key), put.Value); err != nil { + return err + } + case storage.Delete: + del := m.Data.(storage.Delete) + if err := s.db.Delete(s.woptions, engine_util.KeyWithCF(del.Cf, del.Key)); err != nil { + return err + } + } + } + return nil +} + +type StandAloneStorageReader struct { + db *gorocksdb.DB + roptions *gorocksdb.ReadOptions +} + +func (sReader *StandAloneStorageReader) Close() { + return +} + +func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { + val, err := sReader.db.Get(sReader.roptions, engine_util.KeyWithCF(cf, key)) + + return val.Data(), err +} + +func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { + return engine_util.NewRDBIterator(cf, sReader.db, sReader.roptions) +} + +// func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { +// value, err := sReader.db.Get(sReader.) + +// v, e := engine_util.GetCF(sReader.db, cf, key) +// if e == badger.ErrKeyNotFound { +// return nil, nil +// } +// return v, e +// } +// func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { +// txn := sReader.db.NewTransaction(false) +// return engine_util.NewCFIterator(cf, txn) +// } diff --git a/kv/util/engine_util/cf_iterator.go b/kv/util/engine_util/cf_iterator.go index 591dc9256..937816fb0 100644 --- a/kv/util/engine_util/cf_iterator.go +++ b/kv/util/engine_util/cf_iterator.go @@ -4,6 +4,7 @@ import ( "github.com/Connor1996/badger" "github.com/Connor1996/badger/y" "github.com/jmhodges/levigo" + "github.com/tecbot/gorocksdb" ) type LdbItem struct { @@ -37,6 +38,11 @@ type LdbIterator struct { prefix string } +type RdbIterator struct { + iter *gorocksdb.Iterator + prefix string +} + func NewLDBIterator(cf string, db *levigo.DB, roptions *levigo.ReadOptions) *LdbIterator { return &LdbIterator{ iter: db.NewIterator(roptions), @@ -75,6 +81,40 @@ func (it *LdbIterator) Seek(key []byte) { // it.iter.Rewind() // } +func NewRDBIterator(cf string, db *gorocksdb.DB, roptions *gorocksdb.ReadOptions) *RdbIterator { + return &RdbIterator{ + iter: db.NewIterator(roptions), + prefix: cf + "_", + } +} + +func (it *RdbIterator) Item() DBItem { + return &LdbItem{ + key: it.iter.Key().Data(), + value: it.iter.Value().Data(), + prefixLen: len(it.prefix), + } +} + +func (it *RdbIterator) Valid() bool { return it.iter.Valid() } + +// func (it *BadgerIterator) ValidForPrefix(prefix []byte) bool { +// return it.iter.ValidForPrefix(append([]byte(it.prefix), prefix...)) +// } + +func (it *RdbIterator) Close() { + it.iter.Close() +} + +func (it *RdbIterator) Next() { + it.iter.Next() +} + +func (it *RdbIterator) Seek(key []byte) { + it.iter.Seek(append([]byte(it.prefix), key...)) + // it.iter.Seek(append([]byte(it.prefix), key...)) +} + type DBIterator interface { // Item returns pointer to the current key-value pair. Item() DBItem From db719ee18336a24fe4faa636e68e934832463286 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Mon, 19 Dec 2022 17:25:15 +0800 Subject: [PATCH 05/22] add gorocksdb file --- gorocksdb | 1 - gorocksdb/.gitignore | 1 + gorocksdb/.travis.yml | 35 + gorocksdb/LICENSE | 19 + gorocksdb/README.md | 18 + gorocksdb/array.go | 54 ++ gorocksdb/backup.go | 169 ++++ gorocksdb/cache.go | 35 + gorocksdb/cf_handle.go | 36 + gorocksdb/cf_test.go | 229 +++++ gorocksdb/checkpoint.go | 56 ++ gorocksdb/checkpoint_test.go | 57 ++ gorocksdb/compaction_filter.go | 73 ++ gorocksdb/compaction_filter_test.go | 61 ++ gorocksdb/comparator.go | 53 ++ gorocksdb/comparator_test.go | 47 + gorocksdb/cow.go | 42 + gorocksdb/cow_test.go | 48 + gorocksdb/db.go | 943 ++++++++++++++++++ gorocksdb/db_external_file_test.go | 57 ++ gorocksdb/db_test.go | 245 +++++ gorocksdb/dbpath.go | 48 + gorocksdb/doc.go | 74 ++ gorocksdb/dynflag.go | 6 + gorocksdb/env.go | 45 + gorocksdb/filter_policy.go | 94 ++ gorocksdb/filter_policy_test.go | 76 ++ gorocksdb/go.mod | 16 + gorocksdb/go.sum | 23 + gorocksdb/gorocksdb.c | 70 ++ gorocksdb/gorocksdb.h | 30 + gorocksdb/iterator.go | 126 +++ gorocksdb/iterator_test.go | 31 + gorocksdb/memory_usage.go | 58 ++ gorocksdb/memory_usage_test.go | 56 ++ gorocksdb/merge_operator.go | 168 ++++ gorocksdb/merge_operator_test.go | 185 ++++ gorocksdb/options.go | 1213 ++++++++++++++++++++++++ gorocksdb/options_block_based_table.go | 237 +++++ gorocksdb/options_compaction.go | 130 +++ gorocksdb/options_compression.go | 24 + gorocksdb/options_env.go | 25 + gorocksdb/options_flush.go | 32 + gorocksdb/options_ingest.go | 65 ++ gorocksdb/options_read.go | 136 +++ gorocksdb/options_transaction.go | 66 ++ gorocksdb/options_transactiondb.go | 72 ++ gorocksdb/options_write.go | 42 + gorocksdb/ratelimiter.go | 31 + gorocksdb/slice.go | 84 ++ gorocksdb/slice_transform.go | 82 ++ gorocksdb/slice_transform_test.go | 52 + gorocksdb/snapshot.go | 14 + gorocksdb/sst_file_writer.go | 67 ++ gorocksdb/staticflag_linux.go | 6 + gorocksdb/transaction.go | 125 +++ gorocksdb/transactiondb.go | 143 +++ gorocksdb/transactiondb_test.go | 139 +++ gorocksdb/util.go | 74 ++ gorocksdb/wal_iterator.go | 49 + gorocksdb/write_batch.go | 283 ++++++ gorocksdb/write_batch_test.go | 87 ++ 62 files changed, 6662 insertions(+), 1 deletion(-) delete mode 160000 gorocksdb create mode 100644 gorocksdb/.gitignore create mode 100644 gorocksdb/.travis.yml create mode 100644 gorocksdb/LICENSE create mode 100644 gorocksdb/README.md create mode 100644 gorocksdb/array.go create mode 100644 gorocksdb/backup.go create mode 100644 gorocksdb/cache.go create mode 100644 gorocksdb/cf_handle.go create mode 100644 gorocksdb/cf_test.go create mode 100644 gorocksdb/checkpoint.go create mode 100644 gorocksdb/checkpoint_test.go create mode 100644 gorocksdb/compaction_filter.go create mode 100644 gorocksdb/compaction_filter_test.go create mode 100644 gorocksdb/comparator.go create mode 100644 gorocksdb/comparator_test.go create mode 100644 gorocksdb/cow.go create mode 100644 gorocksdb/cow_test.go create mode 100755 gorocksdb/db.go create mode 100644 gorocksdb/db_external_file_test.go create mode 100755 gorocksdb/db_test.go create mode 100644 gorocksdb/dbpath.go create mode 100644 gorocksdb/doc.go create mode 100644 gorocksdb/dynflag.go create mode 100644 gorocksdb/env.go create mode 100644 gorocksdb/filter_policy.go create mode 100644 gorocksdb/filter_policy_test.go create mode 100644 gorocksdb/go.mod create mode 100644 gorocksdb/go.sum create mode 100644 gorocksdb/gorocksdb.c create mode 100644 gorocksdb/gorocksdb.h create mode 100644 gorocksdb/iterator.go create mode 100644 gorocksdb/iterator_test.go create mode 100644 gorocksdb/memory_usage.go create mode 100644 gorocksdb/memory_usage_test.go create mode 100644 gorocksdb/merge_operator.go create mode 100644 gorocksdb/merge_operator_test.go create mode 100644 gorocksdb/options.go create mode 100644 gorocksdb/options_block_based_table.go create mode 100644 gorocksdb/options_compaction.go create mode 100644 gorocksdb/options_compression.go create mode 100644 gorocksdb/options_env.go create mode 100644 gorocksdb/options_flush.go create mode 100644 gorocksdb/options_ingest.go create mode 100644 gorocksdb/options_read.go create mode 100644 gorocksdb/options_transaction.go create mode 100644 gorocksdb/options_transactiondb.go create mode 100644 gorocksdb/options_write.go create mode 100644 gorocksdb/ratelimiter.go create mode 100644 gorocksdb/slice.go create mode 100644 gorocksdb/slice_transform.go create mode 100644 gorocksdb/slice_transform_test.go create mode 100644 gorocksdb/snapshot.go create mode 100644 gorocksdb/sst_file_writer.go create mode 100644 gorocksdb/staticflag_linux.go create mode 100644 gorocksdb/transaction.go create mode 100644 gorocksdb/transactiondb.go create mode 100644 gorocksdb/transactiondb_test.go create mode 100644 gorocksdb/util.go create mode 100755 gorocksdb/wal_iterator.go create mode 100644 gorocksdb/write_batch.go create mode 100644 gorocksdb/write_batch_test.go diff --git a/gorocksdb b/gorocksdb deleted file mode 160000 index f0fad39f3..000000000 --- a/gorocksdb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f0fad39f321c61bf05fcb4a01fe259606235e712 diff --git a/gorocksdb/.gitignore b/gorocksdb/.gitignore new file mode 100644 index 000000000..485dee64b --- /dev/null +++ b/gorocksdb/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/gorocksdb/.travis.yml b/gorocksdb/.travis.yml new file mode 100644 index 000000000..9b331467c --- /dev/null +++ b/gorocksdb/.travis.yml @@ -0,0 +1,35 @@ +dist: xenial +language: go +go: + - 1.12.x + - 1.13.x + - tip + +before_install: + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo apt-get update -qq + - sudo apt-get install gcc-6 g++-6 libsnappy-dev zlib1g-dev libbz2-dev -qq + - export CXX="g++-6" CC="gcc-6" + + - wget https://launchpad.net/ubuntu/+archive/primary/+files/libgflags2_2.0-1.1ubuntu1_amd64.deb + - sudo dpkg -i libgflags2_2.0-1.1ubuntu1_amd64.deb + - wget https://launchpad.net/ubuntu/+archive/primary/+files/libgflags-dev_2.0-1.1ubuntu1_amd64.deb + - sudo dpkg -i libgflags-dev_2.0-1.1ubuntu1_amd64.deb + +install: + - git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb + - pushd /tmp/rocksdb + - make clean + - make shared_lib -j`nproc` + - sudo cp --preserve=links ./librocksdb.* /usr/lib/ + - sudo cp -r ./include/rocksdb/ /usr/include/ + - popd + - go get -t ./... + +script: + - go test -v ./ + +notifications: + email: + on_success: change + on_failure: always diff --git a/gorocksdb/LICENSE b/gorocksdb/LICENSE new file mode 100644 index 000000000..dcd9e57fe --- /dev/null +++ b/gorocksdb/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2016 Thomas Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/gorocksdb/README.md b/gorocksdb/README.md new file mode 100644 index 000000000..b39aa2ba0 --- /dev/null +++ b/gorocksdb/README.md @@ -0,0 +1,18 @@ +# gorocksdb, a Go wrapper for RocksDB + +[![Build Status](https://travis-ci.org/tecbot/gorocksdb.svg)](https://travis-ci.org/tecbot/gorocksdb) [![GoDoc](https://godoc.org/github.com/tecbot/gorocksdb?status.svg)](http://godoc.org/github.com/tecbot/gorocksdb) + +## Install + +You'll need to build [RocksDB](https://github.com/facebook/rocksdb) v5.16+ on your machine. + +After that, you can install gorocksdb using the following command: + + CGO_CFLAGS="-I/path/to/rocksdb/include" \ + CGO_LDFLAGS="-L/path/to/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" \ + go get github.com/tecbot/gorocksdb + +Please note that this package might upgrade the required RocksDB version at any moment. +Vendoring is thus highly recommended if you require high stability. + +*The [embedded CockroachDB RocksDB](https://github.com/cockroachdb/c-rocksdb) is no longer supported in gorocksdb.* diff --git a/gorocksdb/array.go b/gorocksdb/array.go new file mode 100644 index 000000000..c5a12289d --- /dev/null +++ b/gorocksdb/array.go @@ -0,0 +1,54 @@ +package gorocksdb + +// #include "stdlib.h" +// #include "rocksdb/c.h" +import "C" +import ( + "reflect" + "unsafe" +) + +type charsSlice []*C.char +type sizeTSlice []C.size_t +type columnFamilySlice []*C.rocksdb_column_family_handle_t + +func (s charsSlice) c() **C.char { + sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) + return (**C.char)(unsafe.Pointer(sH.Data)) +} + +func (s sizeTSlice) c() *C.size_t { + sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) + return (*C.size_t)(unsafe.Pointer(sH.Data)) +} + +func (s columnFamilySlice) c() **C.rocksdb_column_family_handle_t { + sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) + return (**C.rocksdb_column_family_handle_t)(unsafe.Pointer(sH.Data)) +} + +// bytesSliceToCSlices converts a slice of byte slices to two slices with C +// datatypes. One containing pointers to copies of the byte slices and one +// containing their sizes. +// IMPORTANT: All the contents of the charsSlice array are malloced and +// should be freed using the Destroy method of charsSlice. +func byteSlicesToCSlices(vals [][]byte) (charsSlice, sizeTSlice) { + if len(vals) == 0 { + return nil, nil + } + + chars := make(charsSlice, len(vals)) + sizes := make(sizeTSlice, len(vals)) + for i, val := range vals { + chars[i] = (*C.char)(C.CBytes(val)) + sizes[i] = C.size_t(len(val)) + } + + return chars, sizes +} + +func (s charsSlice) Destroy() { + for _, chars := range s { + C.free(unsafe.Pointer(chars)) + } +} diff --git a/gorocksdb/backup.go b/gorocksdb/backup.go new file mode 100644 index 000000000..87621dd96 --- /dev/null +++ b/gorocksdb/backup.go @@ -0,0 +1,169 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import ( + "errors" + "unsafe" +) + +// BackupEngineInfo represents the information about the backups +// in a backup engine instance. Use this to get the state of the +// backup like number of backups and their ids and timestamps etc. +type BackupEngineInfo struct { + c *C.rocksdb_backup_engine_info_t +} + +// GetCount gets the number backsup available. +func (b *BackupEngineInfo) GetCount() int { + return int(C.rocksdb_backup_engine_info_count(b.c)) +} + +// GetTimestamp gets the timestamp at which the backup index was taken. +func (b *BackupEngineInfo) GetTimestamp(index int) int64 { + return int64(C.rocksdb_backup_engine_info_timestamp(b.c, C.int(index))) +} + +// GetBackupId gets an id that uniquely identifies a backup +// regardless of its position. +func (b *BackupEngineInfo) GetBackupId(index int) int64 { + return int64(C.rocksdb_backup_engine_info_backup_id(b.c, C.int(index))) +} + +// GetSize get the size of the backup in bytes. +func (b *BackupEngineInfo) GetSize(index int) int64 { + return int64(C.rocksdb_backup_engine_info_size(b.c, C.int(index))) +} + +// GetNumFiles gets the number of files in the backup index. +func (b *BackupEngineInfo) GetNumFiles(index int) int32 { + return int32(C.rocksdb_backup_engine_info_number_files(b.c, C.int(index))) +} + +// Destroy destroys the backup engine info instance. +func (b *BackupEngineInfo) Destroy() { + C.rocksdb_backup_engine_info_destroy(b.c) + b.c = nil +} + +// RestoreOptions captures the options to be used during +// restoration of a backup. +type RestoreOptions struct { + c *C.rocksdb_restore_options_t +} + +// NewRestoreOptions creates a RestoreOptions instance. +func NewRestoreOptions() *RestoreOptions { + return &RestoreOptions{ + c: C.rocksdb_restore_options_create(), + } +} + +// SetKeepLogFiles is used to set or unset the keep_log_files option +// If true, restore won't overwrite the existing log files in wal_dir. It will +// also move all log files from archive directory to wal_dir. +// By default, this is false. +func (ro *RestoreOptions) SetKeepLogFiles(v int) { + C.rocksdb_restore_options_set_keep_log_files(ro.c, C.int(v)) +} + +// Destroy destroys this RestoreOptions instance. +func (ro *RestoreOptions) Destroy() { + C.rocksdb_restore_options_destroy(ro.c) +} + +// BackupEngine is a reusable handle to a RocksDB Backup, created by +// OpenBackupEngine. +type BackupEngine struct { + c *C.rocksdb_backup_engine_t + path string + opts *Options +} + +// OpenBackupEngine opens a backup engine with specified options. +func OpenBackupEngine(opts *Options, path string) (*BackupEngine, error) { + var cErr *C.char + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + + be := C.rocksdb_backup_engine_open(opts.c, cpath, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return &BackupEngine{ + c: be, + path: path, + opts: opts, + }, nil +} + +// UnsafeGetBackupEngine returns the underlying c backup engine. +func (b *BackupEngine) UnsafeGetBackupEngine() unsafe.Pointer { + return unsafe.Pointer(b.c) +} + +// CreateNewBackupFlush takes a new backup from db. If flush is set to true, +// it flushes the WAL before taking the backup. +func (b *BackupEngine) CreateNewBackupFlush(db *DB, flush bool) error { + var cErr *C.char + + C.rocksdb_backup_engine_create_new_backup_flush(b.c, db.c, boolToChar(flush), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + + return nil +} + +// CreateNewBackup takes a new backup from db. +func (b *BackupEngine) CreateNewBackup(db *DB) error { + return b.CreateNewBackupFlush(db, false) +} + +// GetInfo gets an object that gives information about +// the backups that have already been taken +func (b *BackupEngine) GetInfo() *BackupEngineInfo { + return &BackupEngineInfo{ + c: C.rocksdb_backup_engine_get_backup_info(b.c), + } +} + +// RestoreDBFromLatestBackup restores the latest backup to dbDir. walDir +// is where the write ahead logs are restored to and usually the same as dbDir. +func (b *BackupEngine) RestoreDBFromLatestBackup(dbDir, walDir string, ro *RestoreOptions) error { + var cErr *C.char + cDbDir := C.CString(dbDir) + cWalDir := C.CString(walDir) + defer func() { + C.free(unsafe.Pointer(cDbDir)) + C.free(unsafe.Pointer(cWalDir)) + }() + + C.rocksdb_backup_engine_restore_db_from_latest_backup(b.c, cDbDir, cWalDir, ro.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// PurgeOldBackups deletes all backups older than the latest 'n' backups +func (b *BackupEngine) PurgeOldBackups(n uint32) error { + var cErr *C.char + C.rocksdb_backup_engine_purge_old_backups(b.c, C.uint32_t(n), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Close close the backup engine and cleans up state +// The backups already taken remain on storage. +func (b *BackupEngine) Close() { + C.rocksdb_backup_engine_close(b.c) + b.c = nil +} diff --git a/gorocksdb/cache.go b/gorocksdb/cache.go new file mode 100644 index 000000000..866326dc1 --- /dev/null +++ b/gorocksdb/cache.go @@ -0,0 +1,35 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// Cache is a cache used to store data read from data in memory. +type Cache struct { + c *C.rocksdb_cache_t +} + +// NewLRUCache creates a new LRU Cache object with the capacity given. +func NewLRUCache(capacity uint64) *Cache { + return NewNativeCache(C.rocksdb_cache_create_lru(C.size_t(capacity))) +} + +// NewNativeCache creates a Cache object. +func NewNativeCache(c *C.rocksdb_cache_t) *Cache { + return &Cache{c} +} + +// GetUsage returns the Cache memory usage. +func (c *Cache) GetUsage() uint64 { + return uint64(C.rocksdb_cache_get_usage(c.c)) +} + +// GetPinnedUsage returns the Cache pinned memory usage. +func (c *Cache) GetPinnedUsage() uint64 { + return uint64(C.rocksdb_cache_get_pinned_usage(c.c)) +} + +// Destroy deallocates the Cache object. +func (c *Cache) Destroy() { + C.rocksdb_cache_destroy(c.c) + c.c = nil +} diff --git a/gorocksdb/cf_handle.go b/gorocksdb/cf_handle.go new file mode 100644 index 000000000..6ded4c590 --- /dev/null +++ b/gorocksdb/cf_handle.go @@ -0,0 +1,36 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import "unsafe" + +// ColumnFamilyHandle represents a handle to a ColumnFamily. +type ColumnFamilyHandle struct { + c *C.rocksdb_column_family_handle_t +} + +// NewNativeColumnFamilyHandle creates a ColumnFamilyHandle object. +func NewNativeColumnFamilyHandle(c *C.rocksdb_column_family_handle_t) *ColumnFamilyHandle { + return &ColumnFamilyHandle{c} +} + +// UnsafeGetCFHandler returns the underlying c column family handle. +func (h *ColumnFamilyHandle) UnsafeGetCFHandler() unsafe.Pointer { + return unsafe.Pointer(h.c) +} + +// Destroy calls the destructor of the underlying column family handle. +func (h *ColumnFamilyHandle) Destroy() { + C.rocksdb_column_family_handle_destroy(h.c) +} + +type ColumnFamilyHandles []*ColumnFamilyHandle + +func (cfs ColumnFamilyHandles) toCSlice() columnFamilySlice { + cCFs := make(columnFamilySlice, len(cfs)) + for i, cf := range cfs { + cCFs[i] = cf.c + } + return cCFs +} diff --git a/gorocksdb/cf_test.go b/gorocksdb/cf_test.go new file mode 100644 index 000000000..f6db7c1fe --- /dev/null +++ b/gorocksdb/cf_test.go @@ -0,0 +1,229 @@ +package gorocksdb + +import ( + "io/ioutil" + "testing" + + "github.com/facebookgo/ensure" +) + +func TestColumnFamilyOpen(t *testing.T) { + dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyOpen") + ensure.Nil(t, err) + + givenNames := []string{"default", "guide"} + opts := NewDefaultOptions() + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetCreateIfMissing(true) + db, cfh, err := OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) + ensure.Nil(t, err) + defer db.Close() + ensure.DeepEqual(t, len(cfh), 2) + cfh[0].Destroy() + cfh[1].Destroy() + + actualNames, err := ListColumnFamilies(opts, dir) + ensure.Nil(t, err) + ensure.SameElements(t, actualNames, givenNames) +} + +func TestColumnFamilyCreateDrop(t *testing.T) { + dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyCreate") + ensure.Nil(t, err) + + opts := NewDefaultOptions() + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetCreateIfMissing(true) + db, err := OpenDb(opts, dir) + ensure.Nil(t, err) + defer db.Close() + cf, err := db.CreateColumnFamily(opts, "guide") + ensure.Nil(t, err) + defer cf.Destroy() + + actualNames, err := ListColumnFamilies(opts, dir) + ensure.Nil(t, err) + ensure.SameElements(t, actualNames, []string{"default", "guide"}) + + ensure.Nil(t, db.DropColumnFamily(cf)) + + actualNames, err = ListColumnFamilies(opts, dir) + ensure.Nil(t, err) + ensure.SameElements(t, actualNames, []string{"default"}) +} + +func TestColumnFamilyBatchPutGet(t *testing.T) { + dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyPutGet") + ensure.Nil(t, err) + + givenNames := []string{"default", "guide"} + opts := NewDefaultOptions() + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetCreateIfMissing(true) + db, cfh, err := OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) + ensure.Nil(t, err) + defer db.Close() + ensure.DeepEqual(t, len(cfh), 2) + defer cfh[0].Destroy() + defer cfh[1].Destroy() + + wo := NewDefaultWriteOptions() + defer wo.Destroy() + ro := NewDefaultReadOptions() + defer ro.Destroy() + + givenKey0 := []byte("hello0") + givenVal0 := []byte("world0") + givenKey1 := []byte("hello1") + givenVal1 := []byte("world1") + + b0 := NewWriteBatch() + defer b0.Destroy() + b0.PutCF(cfh[0], givenKey0, givenVal0) + ensure.Nil(t, db.Write(wo, b0)) + actualVal0, err := db.GetCF(ro, cfh[0], givenKey0) + defer actualVal0.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal0.Data(), givenVal0) + + b1 := NewWriteBatch() + defer b1.Destroy() + b1.PutCF(cfh[1], givenKey1, givenVal1) + ensure.Nil(t, db.Write(wo, b1)) + actualVal1, err := db.GetCF(ro, cfh[1], givenKey1) + defer actualVal1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal1.Data(), givenVal1) + + actualVal, err := db.GetCF(ro, cfh[0], givenKey1) + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal.Size(), 0) + actualVal, err = db.GetCF(ro, cfh[1], givenKey0) + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal.Size(), 0) +} + +func TestColumnFamilyPutGetDelete(t *testing.T) { + dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyPutGet") + ensure.Nil(t, err) + + givenNames := []string{"default", "guide"} + opts := NewDefaultOptions() + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetCreateIfMissing(true) + db, cfh, err := OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) + ensure.Nil(t, err) + defer db.Close() + ensure.DeepEqual(t, len(cfh), 2) + defer cfh[0].Destroy() + defer cfh[1].Destroy() + + wo := NewDefaultWriteOptions() + defer wo.Destroy() + ro := NewDefaultReadOptions() + defer ro.Destroy() + + givenKey0 := []byte("hello0") + givenVal0 := []byte("world0") + givenKey1 := []byte("hello1") + givenVal1 := []byte("world1") + + ensure.Nil(t, db.PutCF(wo, cfh[0], givenKey0, givenVal0)) + actualVal0, err := db.GetCF(ro, cfh[0], givenKey0) + defer actualVal0.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal0.Data(), givenVal0) + + ensure.Nil(t, db.PutCF(wo, cfh[1], givenKey1, givenVal1)) + actualVal1, err := db.GetCF(ro, cfh[1], givenKey1) + defer actualVal1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal1.Data(), givenVal1) + + actualVal, err := db.GetCF(ro, cfh[0], givenKey1) + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal.Size(), 0) + actualVal, err = db.GetCF(ro, cfh[1], givenKey0) + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal.Size(), 0) + + ensure.Nil(t, db.DeleteCF(wo, cfh[0], givenKey0)) + actualVal, err = db.GetCF(ro, cfh[0], givenKey0) + ensure.Nil(t, err) + ensure.DeepEqual(t, actualVal.Size(), 0) +} + +func newTestDBCF(t *testing.T, name string) (db *DB, cfh []*ColumnFamilyHandle, cleanup func()) { + dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyPutGet") + ensure.Nil(t, err) + + givenNames := []string{"default", "guide"} + opts := NewDefaultOptions() + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetCreateIfMissing(true) + db, cfh, err = OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) + ensure.Nil(t, err) + cleanup = func() { + for _, cf := range cfh { + cf.Destroy() + } + db.Close() + } + return db, cfh, cleanup +} + +func TestColumnFamilyMultiGet(t *testing.T) { + db, cfh, cleanup := newTestDBCF(t, "TestDBMultiGet") + defer cleanup() + + var ( + givenKey1 = []byte("hello1") + givenKey2 = []byte("hello2") + givenKey3 = []byte("hello3") + givenVal1 = []byte("world1") + givenVal2 = []byte("world2") + givenVal3 = []byte("world3") + wo = NewDefaultWriteOptions() + ro = NewDefaultReadOptions() + ) + + // create + ensure.Nil(t, db.PutCF(wo, cfh[0], givenKey1, givenVal1)) + ensure.Nil(t, db.PutCF(wo, cfh[1], givenKey2, givenVal2)) + ensure.Nil(t, db.PutCF(wo, cfh[1], givenKey3, givenVal3)) + + // column family 0 only has givenKey1 + values, err := db.MultiGetCF(ro, cfh[0], []byte("noexist"), givenKey1, givenKey2, givenKey3) + defer values.Destroy() + ensure.Nil(t, err) + ensure.DeepEqual(t, len(values), 4) + + ensure.DeepEqual(t, values[0].Data(), []byte(nil)) + ensure.DeepEqual(t, values[1].Data(), givenVal1) + ensure.DeepEqual(t, values[2].Data(), []byte(nil)) + ensure.DeepEqual(t, values[3].Data(), []byte(nil)) + + // column family 1 only has givenKey2 and givenKey3 + values, err = db.MultiGetCF(ro, cfh[1], []byte("noexist"), givenKey1, givenKey2, givenKey3) + defer values.Destroy() + ensure.Nil(t, err) + ensure.DeepEqual(t, len(values), 4) + + ensure.DeepEqual(t, values[0].Data(), []byte(nil)) + ensure.DeepEqual(t, values[1].Data(), []byte(nil)) + ensure.DeepEqual(t, values[2].Data(), givenVal2) + ensure.DeepEqual(t, values[3].Data(), givenVal3) + + // getting them all from the right CF should return them all + values, err = db.MultiGetCFMultiCF(ro, + ColumnFamilyHandles{cfh[0], cfh[1], cfh[1]}, + [][]byte{givenKey1, givenKey2, givenKey3}, + ) + defer values.Destroy() + ensure.Nil(t, err) + ensure.DeepEqual(t, len(values), 3) + + ensure.DeepEqual(t, values[0].Data(), givenVal1) + ensure.DeepEqual(t, values[1].Data(), givenVal2) + ensure.DeepEqual(t, values[2].Data(), givenVal3) +} diff --git a/gorocksdb/checkpoint.go b/gorocksdb/checkpoint.go new file mode 100644 index 000000000..4a6436d2b --- /dev/null +++ b/gorocksdb/checkpoint.go @@ -0,0 +1,56 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" + +import ( + "errors" + "unsafe" +) + +// Checkpoint provides Checkpoint functionality. +// Checkpoints provide persistent snapshots of RocksDB databases. +type Checkpoint struct { + c *C.rocksdb_checkpoint_t +} + +// NewNativeCheckpoint creates a new checkpoint. +func NewNativeCheckpoint(c *C.rocksdb_checkpoint_t) *Checkpoint { + return &Checkpoint{c} +} + +// CreateCheckpoint builds an openable snapshot of RocksDB on the same disk, which +// accepts an output directory on the same disk, and under the directory +// (1) hard-linked SST files pointing to existing live SST files +// SST files will be copied if output directory is on a different filesystem +// (2) a copied manifest files and other files +// The directory should not already exist and will be created by this API. +// The directory will be an absolute path +// log_size_for_flush: if the total log file size is equal or larger than +// this value, then a flush is triggered for all the column families. The +// default value is 0, which means flush is always triggered. If you move +// away from the default, the checkpoint may not contain up-to-date data +// if WAL writing is not always enabled. +// Flush will always trigger if it is 2PC. +func (checkpoint *Checkpoint) CreateCheckpoint(checkpoint_dir string, log_size_for_flush uint64) error { + var ( + cErr *C.char + ) + + cDir := C.CString(checkpoint_dir) + defer C.free(unsafe.Pointer(cDir)) + + C.rocksdb_checkpoint_create(checkpoint.c, cDir, C.uint64_t(log_size_for_flush), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Destroy deallocates the Checkpoint object. +func (checkpoint *Checkpoint) Destroy() { + C.rocksdb_checkpoint_object_destroy(checkpoint.c) + checkpoint.c = nil +} diff --git a/gorocksdb/checkpoint_test.go b/gorocksdb/checkpoint_test.go new file mode 100644 index 000000000..1ea10fdbe --- /dev/null +++ b/gorocksdb/checkpoint_test.go @@ -0,0 +1,57 @@ +package gorocksdb + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/facebookgo/ensure" +) + +func TestCheckpoint(t *testing.T) { + + suffix := "checkpoint" + dir, err := ioutil.TempDir("", "gorocksdb-"+suffix) + ensure.Nil(t, err) + err = os.RemoveAll(dir) + ensure.Nil(t, err) + + db := newTestDB(t, "TestCheckpoint", nil) + defer db.Close() + + // insert keys + givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} + givenVal := []byte("val") + wo := NewDefaultWriteOptions() + for _, k := range givenKeys { + ensure.Nil(t, db.Put(wo, k, givenVal)) + } + + var dbCheck *DB + var checkpoint *Checkpoint + + checkpoint, err = db.NewCheckpoint() + defer checkpoint.Destroy() + ensure.NotNil(t, checkpoint) + ensure.Nil(t, err) + + err = checkpoint.CreateCheckpoint(dir, 0) + ensure.Nil(t, err) + + opts := NewDefaultOptions() + opts.SetCreateIfMissing(true) + dbCheck, err = OpenDb(opts, dir) + defer dbCheck.Close() + ensure.Nil(t, err) + + // test keys + var value *Slice + ro := NewDefaultReadOptions() + for _, k := range givenKeys { + value, err = dbCheck.Get(ro, k) + defer value.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, value.Data(), givenVal) + } + +} diff --git a/gorocksdb/compaction_filter.go b/gorocksdb/compaction_filter.go new file mode 100644 index 000000000..bda27e207 --- /dev/null +++ b/gorocksdb/compaction_filter.go @@ -0,0 +1,73 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// A CompactionFilter can be used to filter keys during compaction time. +type CompactionFilter interface { + // If the Filter function returns false, it indicates + // that the kv should be preserved, while a return value of true + // indicates that this key-value should be removed from the + // output of the compaction. The application can inspect + // the existing value of the key and make decision based on it. + // + // When the value is to be preserved, the application has the option + // to modify the existing value and pass it back through a new value. + // To retain the previous value, simply return nil + // + // If multithreaded compaction is being used *and* a single CompactionFilter + // instance was supplied via SetCompactionFilter, this the Filter function may be + // called from different threads concurrently. The application must ensure + // that the call is thread-safe. + Filter(level int, key, val []byte) (remove bool, newVal []byte) + + // The name of the compaction filter, for logging + Name() string +} + +// NewNativeCompactionFilter creates a CompactionFilter object. +func NewNativeCompactionFilter(c *C.rocksdb_compactionfilter_t) CompactionFilter { + return nativeCompactionFilter{c} +} + +type nativeCompactionFilter struct { + c *C.rocksdb_compactionfilter_t +} + +func (c nativeCompactionFilter) Filter(level int, key, val []byte) (remove bool, newVal []byte) { + return false, nil +} +func (c nativeCompactionFilter) Name() string { return "" } + +// Hold references to compaction filters. +var compactionFilters = NewCOWList() + +type compactionFilterWrapper struct { + name *C.char + filter CompactionFilter +} + +func registerCompactionFilter(filter CompactionFilter) int { + return compactionFilters.Append(compactionFilterWrapper{C.CString(filter.Name()), filter}) +} + +//export gorocksdb_compactionfilter_filter +func gorocksdb_compactionfilter_filter(idx int, cLevel C.int, cKey *C.char, cKeyLen C.size_t, cVal *C.char, cValLen C.size_t, cNewVal **C.char, cNewValLen *C.size_t, cValChanged *C.uchar) C.int { + key := charToByte(cKey, cKeyLen) + val := charToByte(cVal, cValLen) + + remove, newVal := compactionFilters.Get(idx).(compactionFilterWrapper).filter.Filter(int(cLevel), key, val) + if remove { + return C.int(1) + } else if newVal != nil { + *cNewVal = byteToChar(newVal) + *cNewValLen = C.size_t(len(newVal)) + *cValChanged = C.uchar(1) + } + return C.int(0) +} + +//export gorocksdb_compactionfilter_name +func gorocksdb_compactionfilter_name(idx int) *C.char { + return compactionFilters.Get(idx).(compactionFilterWrapper).name +} diff --git a/gorocksdb/compaction_filter_test.go b/gorocksdb/compaction_filter_test.go new file mode 100644 index 000000000..1dfcd63e0 --- /dev/null +++ b/gorocksdb/compaction_filter_test.go @@ -0,0 +1,61 @@ +package gorocksdb + +import ( + "bytes" + "testing" + + "github.com/facebookgo/ensure" +) + +func TestCompactionFilter(t *testing.T) { + var ( + changeKey = []byte("change") + changeValOld = []byte("old") + changeValNew = []byte("new") + deleteKey = []byte("delete") + ) + db := newTestDB(t, "TestCompactionFilter", func(opts *Options) { + opts.SetCompactionFilter(&mockCompactionFilter{ + filter: func(level int, key, val []byte) (remove bool, newVal []byte) { + if bytes.Equal(key, changeKey) { + return false, changeValNew + } + if bytes.Equal(key, deleteKey) { + return true, val + } + t.Errorf("key %q not expected during compaction", key) + return false, nil + }, + }) + }) + defer db.Close() + + // insert the test keys + wo := NewDefaultWriteOptions() + ensure.Nil(t, db.Put(wo, changeKey, changeValOld)) + ensure.Nil(t, db.Put(wo, deleteKey, changeValNew)) + + // trigger a compaction + db.CompactRange(Range{nil, nil}) + + // ensure that the value is changed after compaction + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, changeKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), changeValNew) + + // ensure that the key is deleted after compaction + v2, err := db.Get(ro, deleteKey) + ensure.Nil(t, err) + ensure.True(t, v2.Data() == nil) +} + +type mockCompactionFilter struct { + filter func(level int, key, val []byte) (remove bool, newVal []byte) +} + +func (m *mockCompactionFilter) Name() string { return "gorocksdb.test" } +func (m *mockCompactionFilter) Filter(level int, key, val []byte) (bool, []byte) { + return m.filter(level, key, val) +} diff --git a/gorocksdb/comparator.go b/gorocksdb/comparator.go new file mode 100644 index 000000000..242771e31 --- /dev/null +++ b/gorocksdb/comparator.go @@ -0,0 +1,53 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// A Comparator object provides a total order across slices that are +// used as keys in an sstable or a database. +type Comparator interface { + // Three-way comparison. Returns value: + // < 0 iff "a" < "b", + // == 0 iff "a" == "b", + // > 0 iff "a" > "b" + Compare(a, b []byte) int + + // The name of the comparator. + Name() string +} + +// NewNativeComparator creates a Comparator object. +func NewNativeComparator(c *C.rocksdb_comparator_t) Comparator { + return nativeComparator{c} +} + +type nativeComparator struct { + c *C.rocksdb_comparator_t +} + +func (c nativeComparator) Compare(a, b []byte) int { return 0 } +func (c nativeComparator) Name() string { return "" } + +// Hold references to comperators. +var comperators = NewCOWList() + +type comperatorWrapper struct { + name *C.char + comparator Comparator +} + +func registerComperator(cmp Comparator) int { + return comperators.Append(comperatorWrapper{C.CString(cmp.Name()), cmp}) +} + +//export gorocksdb_comparator_compare +func gorocksdb_comparator_compare(idx int, cKeyA *C.char, cKeyALen C.size_t, cKeyB *C.char, cKeyBLen C.size_t) C.int { + keyA := charToByte(cKeyA, cKeyALen) + keyB := charToByte(cKeyB, cKeyBLen) + return C.int(comperators.Get(idx).(comperatorWrapper).comparator.Compare(keyA, keyB)) +} + +//export gorocksdb_comparator_name +func gorocksdb_comparator_name(idx int) *C.char { + return comperators.Get(idx).(comperatorWrapper).name +} diff --git a/gorocksdb/comparator_test.go b/gorocksdb/comparator_test.go new file mode 100644 index 000000000..76f3e5655 --- /dev/null +++ b/gorocksdb/comparator_test.go @@ -0,0 +1,47 @@ +package gorocksdb + +import ( + "bytes" + "testing" + + "github.com/facebookgo/ensure" +) + +func TestComparator(t *testing.T) { + db := newTestDB(t, "TestComparator", func(opts *Options) { + opts.SetComparator(&bytesReverseComparator{}) + }) + defer db.Close() + + // insert keys + givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} + wo := NewDefaultWriteOptions() + for _, k := range givenKeys { + ensure.Nil(t, db.Put(wo, k, []byte("val"))) + } + + // create a iterator to collect the keys + ro := NewDefaultReadOptions() + iter := db.NewIterator(ro) + defer iter.Close() + + // we seek to the last key and iterate in reverse order + // to match given keys + var actualKeys [][]byte + for iter.SeekToLast(); iter.Valid(); iter.Prev() { + key := make([]byte, 4) + copy(key, iter.Key().Data()) + actualKeys = append(actualKeys, key) + } + ensure.Nil(t, iter.Err()) + + // ensure that the order is correct + ensure.DeepEqual(t, actualKeys, givenKeys) +} + +type bytesReverseComparator struct{} + +func (cmp *bytesReverseComparator) Name() string { return "gorocksdb.bytes-reverse" } +func (cmp *bytesReverseComparator) Compare(a, b []byte) int { + return bytes.Compare(a, b) * -1 +} diff --git a/gorocksdb/cow.go b/gorocksdb/cow.go new file mode 100644 index 000000000..dfcee687e --- /dev/null +++ b/gorocksdb/cow.go @@ -0,0 +1,42 @@ +package gorocksdb + +import ( + "sync" + "sync/atomic" +) + +// COWList implements a copy-on-write list. It is intended to be used by go +// callback registry for CGO, which is read-heavy with occasional writes. +// Reads do not block; Writes do not block reads (or vice versa), but only +// one write can occur at once; +type COWList struct { + v *atomic.Value + mu *sync.Mutex +} + +// NewCOWList creates a new COWList. +func NewCOWList() *COWList { + var list []interface{} + v := &atomic.Value{} + v.Store(list) + return &COWList{v: v, mu: new(sync.Mutex)} +} + +// Append appends an item to the COWList and returns the index for that item. +func (c *COWList) Append(i interface{}) int { + c.mu.Lock() + defer c.mu.Unlock() + list := c.v.Load().([]interface{}) + newLen := len(list) + 1 + newList := make([]interface{}, newLen) + copy(newList, list) + newList[newLen-1] = i + c.v.Store(newList) + return newLen - 1 +} + +// Get gets the item at index. +func (c *COWList) Get(index int) interface{} { + list := c.v.Load().([]interface{}) + return list[index] +} diff --git a/gorocksdb/cow_test.go b/gorocksdb/cow_test.go new file mode 100644 index 000000000..2af9aa566 --- /dev/null +++ b/gorocksdb/cow_test.go @@ -0,0 +1,48 @@ +package gorocksdb + +import ( + "fmt" + "sync" + "testing" + + "github.com/facebookgo/ensure" +) + +func TestCOWList(t *testing.T) { + cl := NewCOWList() + cl.Append("hello") + cl.Append("world") + cl.Append("!") + ensure.DeepEqual(t, cl.Get(0), "hello") + ensure.DeepEqual(t, cl.Get(1), "world") + ensure.DeepEqual(t, cl.Get(2), "!") +} + +func TestCOWListMT(t *testing.T) { + cl := NewCOWList() + expectedRes := make([]int, 3) + var wg sync.WaitGroup + for i := 0; i < 3; i++ { + wg.Add(1) + go func(v int) { + defer wg.Done() + index := cl.Append(v) + expectedRes[index] = v + }(i) + } + wg.Wait() + for i, v := range expectedRes { + ensure.DeepEqual(t, cl.Get(i), v) + } +} + +func BenchmarkCOWList_Get(b *testing.B) { + cl := NewCOWList() + for i := 0; i < 10; i++ { + cl.Append(fmt.Sprintf("helloworld%d", i)) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = cl.Get(i % 10).(string) + } +} diff --git a/gorocksdb/db.go b/gorocksdb/db.go new file mode 100755 index 000000000..79c949686 --- /dev/null +++ b/gorocksdb/db.go @@ -0,0 +1,943 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import ( + "errors" + "fmt" + "unsafe" +) + +// Range is a range of keys in the database. GetApproximateSizes calls with it +// begin at the key Start and end right before the key Limit. +type Range struct { + Start []byte + Limit []byte +} + +// DB is a reusable handle to a RocksDB database on disk, created by Open. +type DB struct { + c *C.rocksdb_t + name string + opts *Options +} + +// OpenDb opens a database with the specified options. +func OpenDb(opts *Options, name string) (*DB, error) { + var ( + cErr *C.char + cName = C.CString(name) + ) + defer C.free(unsafe.Pointer(cName)) + db := C.rocksdb_open(opts.c, cName, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return &DB{ + name: name, + c: db, + opts: opts, + }, nil +} + +// OpenDbWithTTL opens a database with TTL support with the specified options. +func OpenDbWithTTL(opts *Options, name string, ttl int) (*DB, error) { + var ( + cErr *C.char + cName = C.CString(name) + ) + defer C.free(unsafe.Pointer(cName)) + db := C.rocksdb_open_with_ttl(opts.c, cName, C.int(ttl), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return &DB{ + name: name, + c: db, + opts: opts, + }, nil +} + +// OpenDbForReadOnly opens a database with the specified options for readonly usage. +func OpenDbForReadOnly(opts *Options, name string, errorIfLogFileExist bool) (*DB, error) { + var ( + cErr *C.char + cName = C.CString(name) + ) + defer C.free(unsafe.Pointer(cName)) + db := C.rocksdb_open_for_read_only(opts.c, cName, boolToChar(errorIfLogFileExist), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return &DB{ + name: name, + c: db, + opts: opts, + }, nil +} + +// OpenDbColumnFamilies opens a database with the specified column families. +func OpenDbColumnFamilies( + opts *Options, + name string, + cfNames []string, + cfOpts []*Options, +) (*DB, []*ColumnFamilyHandle, error) { + numColumnFamilies := len(cfNames) + if numColumnFamilies != len(cfOpts) { + return nil, nil, errors.New("must provide the same number of column family names and options") + } + + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + cNames := make([]*C.char, numColumnFamilies) + for i, s := range cfNames { + cNames[i] = C.CString(s) + } + defer func() { + for _, s := range cNames { + C.free(unsafe.Pointer(s)) + } + }() + + cOpts := make([]*C.rocksdb_options_t, numColumnFamilies) + for i, o := range cfOpts { + cOpts[i] = o.c + } + + cHandles := make([]*C.rocksdb_column_family_handle_t, numColumnFamilies) + + var cErr *C.char + db := C.rocksdb_open_column_families( + opts.c, + cName, + C.int(numColumnFamilies), + &cNames[0], + &cOpts[0], + &cHandles[0], + &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, nil, errors.New(C.GoString(cErr)) + } + + cfHandles := make([]*ColumnFamilyHandle, numColumnFamilies) + for i, c := range cHandles { + cfHandles[i] = NewNativeColumnFamilyHandle(c) + } + + return &DB{ + name: name, + c: db, + opts: opts, + }, cfHandles, nil +} + +// OpenDbForReadOnlyColumnFamilies opens a database with the specified column +// families in read only mode. +func OpenDbForReadOnlyColumnFamilies( + opts *Options, + name string, + cfNames []string, + cfOpts []*Options, + errorIfLogFileExist bool, +) (*DB, []*ColumnFamilyHandle, error) { + numColumnFamilies := len(cfNames) + if numColumnFamilies != len(cfOpts) { + return nil, nil, errors.New("must provide the same number of column family names and options") + } + + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + cNames := make([]*C.char, numColumnFamilies) + for i, s := range cfNames { + cNames[i] = C.CString(s) + } + defer func() { + for _, s := range cNames { + C.free(unsafe.Pointer(s)) + } + }() + + cOpts := make([]*C.rocksdb_options_t, numColumnFamilies) + for i, o := range cfOpts { + cOpts[i] = o.c + } + + cHandles := make([]*C.rocksdb_column_family_handle_t, numColumnFamilies) + + var cErr *C.char + db := C.rocksdb_open_for_read_only_column_families( + opts.c, + cName, + C.int(numColumnFamilies), + &cNames[0], + &cOpts[0], + &cHandles[0], + boolToChar(errorIfLogFileExist), + &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, nil, errors.New(C.GoString(cErr)) + } + + cfHandles := make([]*ColumnFamilyHandle, numColumnFamilies) + for i, c := range cHandles { + cfHandles[i] = NewNativeColumnFamilyHandle(c) + } + + return &DB{ + name: name, + c: db, + opts: opts, + }, cfHandles, nil +} + +// ListColumnFamilies lists the names of the column families in the DB. +func ListColumnFamilies(opts *Options, name string) ([]string, error) { + var ( + cErr *C.char + cLen C.size_t + cName = C.CString(name) + ) + defer C.free(unsafe.Pointer(cName)) + cNames := C.rocksdb_list_column_families(opts.c, cName, &cLen, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + namesLen := int(cLen) + names := make([]string, namesLen) + // The maximum capacity of the following two slices is limited to (2^29)-1 to remain compatible + // with 32-bit platforms. The size of a `*C.char` (a pointer) is 4 Byte on a 32-bit system + // and (2^29)*4 == math.MaxInt32 + 1. -- See issue golang/go#13656 + cNamesArr := (*[(1 << 29) - 1]*C.char)(unsafe.Pointer(cNames))[:namesLen:namesLen] + for i, n := range cNamesArr { + names[i] = C.GoString(n) + } + C.rocksdb_list_column_families_destroy(cNames, cLen) + return names, nil +} + +// UnsafeGetDB returns the underlying c rocksdb instance. +func (db *DB) UnsafeGetDB() unsafe.Pointer { + return unsafe.Pointer(db.c) +} + +// Name returns the name of the database. +func (db *DB) Name() string { + return db.name +} + +// Get returns the data associated with the key from the database. +func (db *DB) Get(opts *ReadOptions, key []byte) (*Slice, error) { + var ( + cErr *C.char + cValLen C.size_t + cKey = byteToChar(key) + ) + cValue := C.rocksdb_get(db.c, opts.c, cKey, C.size_t(len(key)), &cValLen, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return NewSlice(cValue, cValLen), nil +} + +// GetBytes is like Get but returns a copy of the data. +func (db *DB) GetBytes(opts *ReadOptions, key []byte) ([]byte, error) { + var ( + cErr *C.char + cValLen C.size_t + cKey = byteToChar(key) + ) + cValue := C.rocksdb_get(db.c, opts.c, cKey, C.size_t(len(key)), &cValLen, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + if cValue == nil { + return nil, nil + } + defer C.rocksdb_free(unsafe.Pointer(cValue)) + return C.GoBytes(unsafe.Pointer(cValue), C.int(cValLen)), nil +} + +// GetCF returns the data associated with the key from the database and column family. +func (db *DB) GetCF(opts *ReadOptions, cf *ColumnFamilyHandle, key []byte) (*Slice, error) { + var ( + cErr *C.char + cValLen C.size_t + cKey = byteToChar(key) + ) + cValue := C.rocksdb_get_cf(db.c, opts.c, cf.c, cKey, C.size_t(len(key)), &cValLen, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return NewSlice(cValue, cValLen), nil +} + +// GetPinned returns the data associated with the key from the database. +func (db *DB) GetPinned(opts *ReadOptions, key []byte) (*PinnableSliceHandle, error) { + var ( + cErr *C.char + cKey = byteToChar(key) + ) + cHandle := C.rocksdb_get_pinned(db.c, opts.c, cKey, C.size_t(len(key)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return NewNativePinnableSliceHandle(cHandle), nil +} + +// MultiGet returns the data associated with the passed keys from the database +func (db *DB) MultiGet(opts *ReadOptions, keys ...[]byte) (Slices, error) { + cKeys, cKeySizes := byteSlicesToCSlices(keys) + defer cKeys.Destroy() + vals := make(charsSlice, len(keys)) + valSizes := make(sizeTSlice, len(keys)) + rocksErrs := make(charsSlice, len(keys)) + + C.rocksdb_multi_get( + db.c, + opts.c, + C.size_t(len(keys)), + cKeys.c(), + cKeySizes.c(), + vals.c(), + valSizes.c(), + rocksErrs.c(), + ) + + var errs []error + + for i, rocksErr := range rocksErrs { + if rocksErr != nil { + defer C.rocksdb_free(unsafe.Pointer(rocksErr)) + err := fmt.Errorf("getting %q failed: %v", string(keys[i]), C.GoString(rocksErr)) + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("failed to get %d keys, first error: %v", len(errs), errs[0]) + } + + slices := make(Slices, len(keys)) + for i, val := range vals { + slices[i] = NewSlice(val, valSizes[i]) + } + + return slices, nil +} + +// MultiGetCF returns the data associated with the passed keys from the column family +func (db *DB) MultiGetCF(opts *ReadOptions, cf *ColumnFamilyHandle, keys ...[]byte) (Slices, error) { + cfs := make(ColumnFamilyHandles, len(keys)) + for i := 0; i < len(keys); i++ { + cfs[i] = cf + } + return db.MultiGetCFMultiCF(opts, cfs, keys) +} + +// MultiGetCFMultiCF returns the data associated with the passed keys and +// column families. +func (db *DB) MultiGetCFMultiCF(opts *ReadOptions, cfs ColumnFamilyHandles, keys [][]byte) (Slices, error) { + cKeys, cKeySizes := byteSlicesToCSlices(keys) + defer cKeys.Destroy() + vals := make(charsSlice, len(keys)) + valSizes := make(sizeTSlice, len(keys)) + rocksErrs := make(charsSlice, len(keys)) + + C.rocksdb_multi_get_cf( + db.c, + opts.c, + cfs.toCSlice().c(), + C.size_t(len(keys)), + cKeys.c(), + cKeySizes.c(), + vals.c(), + valSizes.c(), + rocksErrs.c(), + ) + + var errs []error + + for i, rocksErr := range rocksErrs { + if rocksErr != nil { + defer C.rocksdb_free(unsafe.Pointer(rocksErr)) + err := fmt.Errorf("getting %q failed: %v", string(keys[i]), C.GoString(rocksErr)) + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("failed to get %d keys, first error: %v", len(errs), errs[0]) + } + + slices := make(Slices, len(keys)) + for i, val := range vals { + slices[i] = NewSlice(val, valSizes[i]) + } + + return slices, nil +} + +// Put writes data associated with a key to the database. +func (db *DB) Put(opts *WriteOptions, key, value []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + cValue = byteToChar(value) + ) + C.rocksdb_put(db.c, opts.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// PutCF writes data associated with a key to the database and column family. +func (db *DB) PutCF(opts *WriteOptions, cf *ColumnFamilyHandle, key, value []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + cValue = byteToChar(value) + ) + C.rocksdb_put_cf(db.c, opts.c, cf.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Delete removes the data associated with the key from the database. +func (db *DB) Delete(opts *WriteOptions, key []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + ) + C.rocksdb_delete(db.c, opts.c, cKey, C.size_t(len(key)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// DeleteCF removes the data associated with the key from the database and column family. +func (db *DB) DeleteCF(opts *WriteOptions, cf *ColumnFamilyHandle, key []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + ) + C.rocksdb_delete_cf(db.c, opts.c, cf.c, cKey, C.size_t(len(key)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Merge merges the data associated with the key with the actual data in the database. +func (db *DB) Merge(opts *WriteOptions, key []byte, value []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + cValue = byteToChar(value) + ) + C.rocksdb_merge(db.c, opts.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// MergeCF merges the data associated with the key with the actual data in the +// database and column family. +func (db *DB) MergeCF(opts *WriteOptions, cf *ColumnFamilyHandle, key []byte, value []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + cValue = byteToChar(value) + ) + C.rocksdb_merge_cf(db.c, opts.c, cf.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Write writes a WriteBatch to the database +func (db *DB) Write(opts *WriteOptions, batch *WriteBatch) error { + var cErr *C.char + C.rocksdb_write(db.c, opts.c, batch.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// NewIterator returns an Iterator over the the database that uses the +// ReadOptions given. +func (db *DB) NewIterator(opts *ReadOptions) *Iterator { + cIter := C.rocksdb_create_iterator(db.c, opts.c) + return NewNativeIterator(unsafe.Pointer(cIter)) +} + +// NewIteratorCF returns an Iterator over the the database and column family +// that uses the ReadOptions given. +func (db *DB) NewIteratorCF(opts *ReadOptions, cf *ColumnFamilyHandle) *Iterator { + cIter := C.rocksdb_create_iterator_cf(db.c, opts.c, cf.c) + return NewNativeIterator(unsafe.Pointer(cIter)) +} + +func (db *DB) GetUpdatesSince(seqNumber uint64) (*WalIterator, error) { + var cErr *C.char + cIter := C.rocksdb_get_updates_since(db.c, C.uint64_t(seqNumber), nil, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return NewNativeWalIterator(unsafe.Pointer(cIter)), nil +} + +func (db *DB) GetLatestSequenceNumber() uint64 { + return uint64(C.rocksdb_get_latest_sequence_number(db.c)) +} + +// NewSnapshot creates a new snapshot of the database. +func (db *DB) NewSnapshot() *Snapshot { + cSnap := C.rocksdb_create_snapshot(db.c) + return NewNativeSnapshot(cSnap) +} + +// ReleaseSnapshot releases the snapshot and its resources. +func (db *DB) ReleaseSnapshot(snapshot *Snapshot) { + C.rocksdb_release_snapshot(db.c, snapshot.c) + snapshot.c = nil +} + +// GetProperty returns the value of a database property. +func (db *DB) GetProperty(propName string) string { + cprop := C.CString(propName) + defer C.free(unsafe.Pointer(cprop)) + cValue := C.rocksdb_property_value(db.c, cprop) + defer C.rocksdb_free(unsafe.Pointer(cValue)) + return C.GoString(cValue) +} + +// GetPropertyCF returns the value of a database property. +func (db *DB) GetPropertyCF(propName string, cf *ColumnFamilyHandle) string { + cProp := C.CString(propName) + defer C.free(unsafe.Pointer(cProp)) + cValue := C.rocksdb_property_value_cf(db.c, cf.c, cProp) + defer C.rocksdb_free(unsafe.Pointer(cValue)) + return C.GoString(cValue) +} + +// CreateColumnFamily create a new column family. +func (db *DB) CreateColumnFamily(opts *Options, name string) (*ColumnFamilyHandle, error) { + var ( + cErr *C.char + cName = C.CString(name) + ) + defer C.free(unsafe.Pointer(cName)) + cHandle := C.rocksdb_create_column_family(db.c, opts.c, cName, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return NewNativeColumnFamilyHandle(cHandle), nil +} + +// DropColumnFamily drops a column family. +func (db *DB) DropColumnFamily(c *ColumnFamilyHandle) error { + var cErr *C.char + C.rocksdb_drop_column_family(db.c, c.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// GetApproximateSizes returns the approximate number of bytes of file system +// space used by one or more key ranges. +// +// The keys counted will begin at Range.Start and end on the key before +// Range.Limit. +func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { + sizes := make([]uint64, len(ranges)) + var cErr *C.char + if len(ranges) == 0 { + return sizes + } + + cStarts := make([]*C.char, len(ranges)) + cLimits := make([]*C.char, len(ranges)) + cStartLens := make([]C.size_t, len(ranges)) + cLimitLens := make([]C.size_t, len(ranges)) + for i, r := range ranges { + cStarts[i] = (*C.char)(C.CBytes(r.Start)) + cStartLens[i] = C.size_t(len(r.Start)) + cLimits[i] = (*C.char)(C.CBytes(r.Limit)) + cLimitLens[i] = C.size_t(len(r.Limit)) + } + + defer func() { + for i := range ranges { + C.free(unsafe.Pointer(cStarts[i])) + C.free(unsafe.Pointer(cLimits[i])) + } + }() + + C.rocksdb_approximate_sizes( + db.c, + C.int(len(ranges)), + &cStarts[0], + &cStartLens[0], + &cLimits[0], + &cLimitLens[0], + (*C.uint64_t)(&sizes[0]), + &cErr) + + return sizes +} + +// GetApproximateSizesCF returns the approximate number of bytes of file system +// space used by one or more key ranges in the column family. +// +// The keys counted will begin at Range.Start and end on the key before +// Range.Limit. +func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []uint64 { + sizes := make([]uint64, len(ranges)) + var cErr *C.char + if len(ranges) == 0 { + return sizes + } + + cStarts := make([]*C.char, len(ranges)) + cLimits := make([]*C.char, len(ranges)) + cStartLens := make([]C.size_t, len(ranges)) + cLimitLens := make([]C.size_t, len(ranges)) + for i, r := range ranges { + cStarts[i] = (*C.char)(C.CBytes(r.Start)) + cStartLens[i] = C.size_t(len(r.Start)) + cLimits[i] = (*C.char)(C.CBytes(r.Limit)) + cLimitLens[i] = C.size_t(len(r.Limit)) + } + + defer func() { + for i := range ranges { + C.free(unsafe.Pointer(cStarts[i])) + C.free(unsafe.Pointer(cLimits[i])) + } + }() + + C.rocksdb_approximate_sizes_cf( + db.c, + cf.c, + C.int(len(ranges)), + &cStarts[0], + &cStartLens[0], + &cLimits[0], + &cLimitLens[0], + (*C.uint64_t)(&sizes[0]), + &cErr) + + return sizes +} + +// SetOptions dynamically changes options through the SetOptions API. +func (db *DB) SetOptions(keys, values []string) error { + num_keys := len(keys) + + if num_keys == 0 { + return nil + } + + cKeys := make([]*C.char, num_keys) + cValues := make([]*C.char, num_keys) + for i := range keys { + cKeys[i] = C.CString(keys[i]) + cValues[i] = C.CString(values[i]) + } + + var cErr *C.char + + C.rocksdb_set_options( + db.c, + C.int(num_keys), + &cKeys[0], + &cValues[0], + &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// LiveFileMetadata is a metadata which is associated with each SST file. +type LiveFileMetadata struct { + Name string + Level int + Size int64 + SmallestKey []byte + LargestKey []byte +} + +// GetLiveFilesMetaData returns a list of all table files with their +// level, start key and end key. +func (db *DB) GetLiveFilesMetaData() []LiveFileMetadata { + lf := C.rocksdb_livefiles(db.c) + defer C.rocksdb_livefiles_destroy(lf) + + count := C.rocksdb_livefiles_count(lf) + liveFiles := make([]LiveFileMetadata, int(count)) + for i := C.int(0); i < count; i++ { + var liveFile LiveFileMetadata + liveFile.Name = C.GoString(C.rocksdb_livefiles_name(lf, i)) + liveFile.Level = int(C.rocksdb_livefiles_level(lf, i)) + liveFile.Size = int64(C.rocksdb_livefiles_size(lf, i)) + + var cSize C.size_t + key := C.rocksdb_livefiles_smallestkey(lf, i, &cSize) + liveFile.SmallestKey = C.GoBytes(unsafe.Pointer(key), C.int(cSize)) + + key = C.rocksdb_livefiles_largestkey(lf, i, &cSize) + liveFile.LargestKey = C.GoBytes(unsafe.Pointer(key), C.int(cSize)) + liveFiles[int(i)] = liveFile + } + return liveFiles +} + +// CompactRange runs a manual compaction on the Range of keys given. This is +// not likely to be needed for typical usage. +func (db *DB) CompactRange(r Range) { + cStart := byteToChar(r.Start) + cLimit := byteToChar(r.Limit) + C.rocksdb_compact_range(db.c, cStart, C.size_t(len(r.Start)), cLimit, C.size_t(len(r.Limit))) +} + +// CompactRangeCF runs a manual compaction on the Range of keys given on the +// given column family. This is not likely to be needed for typical usage. +func (db *DB) CompactRangeCF(cf *ColumnFamilyHandle, r Range) { + cStart := byteToChar(r.Start) + cLimit := byteToChar(r.Limit) + C.rocksdb_compact_range_cf(db.c, cf.c, cStart, C.size_t(len(r.Start)), cLimit, C.size_t(len(r.Limit))) +} + +// Flush triggers a manuel flush for the database. +func (db *DB) Flush(opts *FlushOptions) error { + var cErr *C.char + C.rocksdb_flush(db.c, opts.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// DisableFileDeletions disables file deletions and should be used when backup the database. +func (db *DB) DisableFileDeletions() error { + var cErr *C.char + C.rocksdb_disable_file_deletions(db.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// EnableFileDeletions enables file deletions for the database. +func (db *DB) EnableFileDeletions(force bool) error { + var cErr *C.char + C.rocksdb_enable_file_deletions(db.c, boolToChar(force), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// DeleteFile deletes the file name from the db directory and update the internal state to +// reflect that. Supports deletion of sst and log files only. 'name' must be +// path relative to the db directory. eg. 000001.sst, /archive/000003.log. +func (db *DB) DeleteFile(name string) { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + C.rocksdb_delete_file(db.c, cName) +} + +// DeleteFileInRange deletes SST files that contain keys between the Range, [r.Start, r.Limit] +func (db *DB) DeleteFileInRange(r Range) error { + cStartKey := byteToChar(r.Start) + cLimitKey := byteToChar(r.Limit) + + var cErr *C.char + + C.rocksdb_delete_file_in_range( + db.c, + cStartKey, C.size_t(len(r.Start)), + cLimitKey, C.size_t(len(r.Limit)), + &cErr, + ) + + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// DeleteFileInRangeCF deletes SST files that contain keys between the Range, [r.Start, r.Limit], and +// belong to a given column family +func (db *DB) DeleteFileInRangeCF(cf *ColumnFamilyHandle, r Range) error { + cStartKey := byteToChar(r.Start) + cLimitKey := byteToChar(r.Limit) + + var cErr *C.char + + C.rocksdb_delete_file_in_range_cf( + db.c, + cf.c, + cStartKey, C.size_t(len(r.Start)), + cLimitKey, C.size_t(len(r.Limit)), + &cErr, + ) + + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// IngestExternalFile loads a list of external SST files. +func (db *DB) IngestExternalFile(filePaths []string, opts *IngestExternalFileOptions) error { + cFilePaths := make([]*C.char, len(filePaths)) + for i, s := range filePaths { + cFilePaths[i] = C.CString(s) + } + defer func() { + for _, s := range cFilePaths { + C.free(unsafe.Pointer(s)) + } + }() + + var cErr *C.char + + C.rocksdb_ingest_external_file( + db.c, + &cFilePaths[0], + C.size_t(len(filePaths)), + opts.c, + &cErr, + ) + + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// IngestExternalFileCF loads a list of external SST files for a column family. +func (db *DB) IngestExternalFileCF(handle *ColumnFamilyHandle, filePaths []string, opts *IngestExternalFileOptions) error { + cFilePaths := make([]*C.char, len(filePaths)) + for i, s := range filePaths { + cFilePaths[i] = C.CString(s) + } + defer func() { + for _, s := range cFilePaths { + C.free(unsafe.Pointer(s)) + } + }() + + var cErr *C.char + + C.rocksdb_ingest_external_file_cf( + db.c, + handle.c, + &cFilePaths[0], + C.size_t(len(filePaths)), + opts.c, + &cErr, + ) + + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// NewCheckpoint creates a new Checkpoint for this db. +func (db *DB) NewCheckpoint() (*Checkpoint, error) { + var ( + cErr *C.char + ) + cCheckpoint := C.rocksdb_checkpoint_object_create( + db.c, &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + + return NewNativeCheckpoint(cCheckpoint), nil +} + +// Close closes the database. +func (db *DB) Close() { + C.rocksdb_close(db.c) +} + +// DestroyDb removes a database entirely, removing everything from the +// filesystem. +func DestroyDb(name string, opts *Options) error { + var ( + cErr *C.char + cName = C.CString(name) + ) + defer C.free(unsafe.Pointer(cName)) + C.rocksdb_destroy_db(opts.c, cName, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// RepairDb repairs a database. +func RepairDb(name string, opts *Options) error { + var ( + cErr *C.char + cName = C.CString(name) + ) + defer C.free(unsafe.Pointer(cName)) + C.rocksdb_repair_db(opts.c, cName, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} diff --git a/gorocksdb/db_external_file_test.go b/gorocksdb/db_external_file_test.go new file mode 100644 index 000000000..9eae5aab5 --- /dev/null +++ b/gorocksdb/db_external_file_test.go @@ -0,0 +1,57 @@ +package gorocksdb + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/facebookgo/ensure" +) + +func TestExternalFile(t *testing.T) { + db := newTestDB(t, "TestDBExternalFile", nil) + defer db.Close() + + envOpts := NewDefaultEnvOptions() + opts := NewDefaultOptions() + w := NewSSTFileWriter(envOpts, opts) + defer w.Destroy() + + filePath, err := ioutil.TempFile("", "sst-file-test") + ensure.Nil(t, err) + defer os.Remove(filePath.Name()) + + err = w.Open(filePath.Name()) + ensure.Nil(t, err) + + err = w.Add([]byte("aaa"), []byte("aaaValue")) + ensure.Nil(t, err) + err = w.Add([]byte("bbb"), []byte("bbbValue")) + ensure.Nil(t, err) + err = w.Add([]byte("ccc"), []byte("cccValue")) + ensure.Nil(t, err) + err = w.Add([]byte("ddd"), []byte("dddValue")) + ensure.Nil(t, err) + + err = w.Finish() + ensure.Nil(t, err) + + ingestOpts := NewDefaultIngestExternalFileOptions() + err = db.IngestExternalFile([]string{filePath.Name()}, ingestOpts) + ensure.Nil(t, err) + + readOpts := NewDefaultReadOptions() + + v1, err := db.Get(readOpts, []byte("aaa")) + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), []byte("aaaValue")) + v2, err := db.Get(readOpts, []byte("bbb")) + ensure.Nil(t, err) + ensure.DeepEqual(t, v2.Data(), []byte("bbbValue")) + v3, err := db.Get(readOpts, []byte("ccc")) + ensure.Nil(t, err) + ensure.DeepEqual(t, v3.Data(), []byte("cccValue")) + v4, err := db.Get(readOpts, []byte("ddd")) + ensure.Nil(t, err) + ensure.DeepEqual(t, v4.Data(), []byte("dddValue")) +} diff --git a/gorocksdb/db_test.go b/gorocksdb/db_test.go new file mode 100755 index 000000000..4ccc7aa8c --- /dev/null +++ b/gorocksdb/db_test.go @@ -0,0 +1,245 @@ +package gorocksdb + +import ( + "io/ioutil" + "strconv" + "testing" + + "github.com/facebookgo/ensure" +) + +func TestOpenDb(t *testing.T) { + db := newTestDB(t, "TestOpenDb", nil) + defer db.Close() +} + +func TestDBCRUD(t *testing.T) { + db := newTestDB(t, "TestDBGet", nil) + defer db.Close() + + var ( + givenKey = []byte("hello") + givenVal1 = []byte("") + givenVal2 = []byte("world1") + wo = NewDefaultWriteOptions() + ro = NewDefaultReadOptions() + ) + + // create + ensure.Nil(t, db.Put(wo, givenKey, givenVal1)) + + // retrieve + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), givenVal1) + + // update + ensure.Nil(t, db.Put(wo, givenKey, givenVal2)) + v2, err := db.Get(ro, givenKey) + defer v2.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v2.Data(), givenVal2) + + // retrieve pinned + v3, err := db.GetPinned(ro, givenKey) + defer v3.Destroy() + ensure.Nil(t, err) + ensure.DeepEqual(t, v3.Data(), givenVal2) + + // delete + ensure.Nil(t, db.Delete(wo, givenKey)) + v4, err := db.Get(ro, givenKey) + ensure.Nil(t, err) + ensure.True(t, v4.Data() == nil) + + // retrieve missing pinned + v5, err := db.GetPinned(ro, givenKey) + defer v5.Destroy() + ensure.Nil(t, err) + ensure.True(t, v5.Data() == nil) +} + +func TestDBCRUDDBPaths(t *testing.T) { + names := make([]string, 4) + target_sizes := make([]uint64, len(names)) + + for i := range names { + names[i] = "TestDBGet_" + strconv.FormatInt(int64(i), 10) + target_sizes[i] = uint64(1024 * 1024 * (i + 1)) + } + + db := newTestDBPathNames(t, "TestDBGet", names, target_sizes, nil) + defer db.Close() + + var ( + givenKey = []byte("hello") + givenVal1 = []byte("") + givenVal2 = []byte("world1") + givenVal3 = []byte("world2") + wo = NewDefaultWriteOptions() + ro = NewDefaultReadOptions() + ) + + // retrieve before create + noexist, err := db.Get(ro, givenKey) + defer noexist.Free() + ensure.Nil(t, err) + ensure.False(t, noexist.Exists()) + ensure.DeepEqual(t, noexist.Data(), []byte(nil)) + + // create + ensure.Nil(t, db.Put(wo, givenKey, givenVal1)) + + // retrieve + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.True(t, v1.Exists()) + ensure.DeepEqual(t, v1.Data(), givenVal1) + + // update + ensure.Nil(t, db.Put(wo, givenKey, givenVal2)) + v2, err := db.Get(ro, givenKey) + defer v2.Free() + ensure.Nil(t, err) + ensure.True(t, v2.Exists()) + ensure.DeepEqual(t, v2.Data(), givenVal2) + + // update + ensure.Nil(t, db.Put(wo, givenKey, givenVal3)) + v3, err := db.Get(ro, givenKey) + defer v3.Free() + ensure.Nil(t, err) + ensure.True(t, v3.Exists()) + ensure.DeepEqual(t, v3.Data(), givenVal3) + + // delete + ensure.Nil(t, db.Delete(wo, givenKey)) + v4, err := db.Get(ro, givenKey) + defer v4.Free() + ensure.Nil(t, err) + ensure.False(t, v4.Exists()) + ensure.DeepEqual(t, v4.Data(), []byte(nil)) +} + +func newTestDB(t *testing.T, name string, applyOpts func(opts *Options)) *DB { + dir, err := ioutil.TempDir("", "gorocksdb-"+name) + ensure.Nil(t, err) + + opts := NewDefaultOptions() + // test the ratelimiter + rateLimiter := NewRateLimiter(1024, 100*1000, 10) + opts.SetRateLimiter(rateLimiter) + opts.SetCreateIfMissing(true) + if applyOpts != nil { + applyOpts(opts) + } + db, err := OpenDb(opts, dir) + ensure.Nil(t, err) + + return db +} + +func newTestDBPathNames(t *testing.T, name string, names []string, target_sizes []uint64, applyOpts func(opts *Options)) *DB { + ensure.DeepEqual(t, len(target_sizes), len(names)) + ensure.NotDeepEqual(t, len(names), 0) + + dir, err := ioutil.TempDir("", "gorocksdb-"+name) + ensure.Nil(t, err) + + paths := make([]string, len(names)) + for i, name := range names { + dir, err := ioutil.TempDir("", "gorocksdb-"+name) + ensure.Nil(t, err) + paths[i] = dir + } + + dbpaths := NewDBPathsFromData(paths, target_sizes) + defer DestroyDBPaths(dbpaths) + + opts := NewDefaultOptions() + opts.SetDBPaths(dbpaths) + // test the ratelimiter + rateLimiter := NewRateLimiter(1024, 100*1000, 10) + opts.SetRateLimiter(rateLimiter) + opts.SetCreateIfMissing(true) + if applyOpts != nil { + applyOpts(opts) + } + db, err := OpenDb(opts, dir) + ensure.Nil(t, err) + + return db +} + +func TestDBMultiGet(t *testing.T) { + db := newTestDB(t, "TestDBMultiGet", nil) + defer db.Close() + + var ( + givenKey1 = []byte("hello1") + givenKey2 = []byte("hello2") + givenKey3 = []byte("hello3") + givenVal1 = []byte("world1") + givenVal2 = []byte("world2") + givenVal3 = []byte("world3") + wo = NewDefaultWriteOptions() + ro = NewDefaultReadOptions() + ) + + // create + ensure.Nil(t, db.Put(wo, givenKey1, givenVal1)) + ensure.Nil(t, db.Put(wo, givenKey2, givenVal2)) + ensure.Nil(t, db.Put(wo, givenKey3, givenVal3)) + + // retrieve + values, err := db.MultiGet(ro, []byte("noexist"), givenKey1, givenKey2, givenKey3) + defer values.Destroy() + ensure.Nil(t, err) + ensure.DeepEqual(t, len(values), 4) + + ensure.DeepEqual(t, values[0].Data(), []byte(nil)) + ensure.DeepEqual(t, values[1].Data(), givenVal1) + ensure.DeepEqual(t, values[2].Data(), givenVal2) + ensure.DeepEqual(t, values[3].Data(), givenVal3) +} + +func TestDBGetApproximateSizes(t *testing.T) { + db := newTestDB(t, "TestDBGetApproximateSizes", nil) + defer db.Close() + + // no ranges + sizes := db.GetApproximateSizes(nil) + ensure.DeepEqual(t, len(sizes), 0) + + // range will nil start and limit + sizes = db.GetApproximateSizes([]Range{{Start: nil, Limit: nil}}) + ensure.DeepEqual(t, sizes, []uint64{0}) + + // valid range + sizes = db.GetApproximateSizes([]Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}}) + ensure.DeepEqual(t, sizes, []uint64{0}) +} + +func TestDBGetApproximateSizesCF(t *testing.T) { + db := newTestDB(t, "TestDBGetApproximateSizesCF", nil) + defer db.Close() + + o := NewDefaultOptions() + + cf, err := db.CreateColumnFamily(o, "other") + ensure.Nil(t, err) + + // no ranges + sizes := db.GetApproximateSizesCF(cf, nil) + ensure.DeepEqual(t, len(sizes), 0) + + // range will nil start and limit + sizes = db.GetApproximateSizesCF(cf, []Range{{Start: nil, Limit: nil}}) + ensure.DeepEqual(t, sizes, []uint64{0}) + + // valid range + sizes = db.GetApproximateSizesCF(cf, []Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}}) + ensure.DeepEqual(t, sizes, []uint64{0}) +} diff --git a/gorocksdb/dbpath.go b/gorocksdb/dbpath.go new file mode 100644 index 000000000..bec984add --- /dev/null +++ b/gorocksdb/dbpath.go @@ -0,0 +1,48 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import "unsafe" + +// DBPath represents options for a dbpath. +type DBPath struct { + c *C.rocksdb_dbpath_t +} + +// NewDBPath creates a DBPath object +// with the given path and target_size. +func NewDBPath(path string, target_size uint64) *DBPath { + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + return NewNativeDBPath(C.rocksdb_dbpath_create(cpath, C.uint64_t(target_size))) +} + +// NewNativeDBPath creates a DBPath object. +func NewNativeDBPath(c *C.rocksdb_dbpath_t) *DBPath { + return &DBPath{c} +} + +// Destroy deallocates the DBPath object. +func (dbpath *DBPath) Destroy() { + C.rocksdb_dbpath_destroy(dbpath.c) +} + +// NewDBPathsFromData creates a slice with allocated DBPath objects +// from paths and target_sizes. +func NewDBPathsFromData(paths []string, target_sizes []uint64) []*DBPath { + dbpaths := make([]*DBPath, len(paths)) + for i, path := range paths { + targetSize := target_sizes[i] + dbpaths[i] = NewDBPath(path, targetSize) + } + + return dbpaths +} + +// DestroyDBPaths deallocates all DBPath objects in dbpaths. +func DestroyDBPaths(dbpaths []*DBPath) { + for _, dbpath := range dbpaths { + dbpath.Destroy() + } +} diff --git a/gorocksdb/doc.go b/gorocksdb/doc.go new file mode 100644 index 000000000..f56a0926b --- /dev/null +++ b/gorocksdb/doc.go @@ -0,0 +1,74 @@ +/* +Package gorocksdb provides the ability to create and access RocksDB databases. + +gorocksdb.OpenDb opens and creates databases. + + bbto := gorocksdb.NewDefaultBlockBasedTableOptions() + bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30)) + opts := gorocksdb.NewDefaultOptions() + opts.SetBlockBasedTableFactory(bbto) + opts.SetCreateIfMissing(true) + db, err := gorocksdb.OpenDb(opts, "/path/to/db") + +The DB struct returned by OpenDb provides DB.Get, DB.Put, DB.Merge and DB.Delete to modify +and query the database. + + ro := gorocksdb.NewDefaultReadOptions() + wo := gorocksdb.NewDefaultWriteOptions() + // if ro and wo are not used again, be sure to Close them. + err = db.Put(wo, []byte("foo"), []byte("bar")) + ... + value, err := db.Get(ro, []byte("foo")) + defer value.Free() + ... + err = db.Delete(wo, []byte("foo")) + +For bulk reads, use an Iterator. If you want to avoid disturbing your live +traffic while doing the bulk read, be sure to call SetFillCache(false) on the +ReadOptions you use when creating the Iterator. + + ro := gorocksdb.NewDefaultReadOptions() + ro.SetFillCache(false) + it := db.NewIterator(ro) + defer it.Close() + it.Seek([]byte("foo")) + for it = it; it.Valid(); it.Next() { + key := it.Key() + value := it.Value() + fmt.Printf("Key: %v Value: %v\n", key.Data(), value.Data()) + key.Free() + value.Free() + } + if err := it.Err(); err != nil { + ... + } + +Batched, atomic writes can be performed with a WriteBatch and +DB.Write. + + wb := gorocksdb.NewWriteBatch() + // defer wb.Close or use wb.Clear and reuse. + wb.Delete([]byte("foo")) + wb.Put([]byte("foo"), []byte("bar")) + wb.Put([]byte("bar"), []byte("foo")) + err := db.Write(wo, wb) + +If your working dataset does not fit in memory, you'll want to add a bloom +filter to your database. NewBloomFilter and +BlockBasedTableOptions.SetFilterPolicy is what you want. NewBloomFilter is +amount of bits in the filter to use per key in your database. + + filter := gorocksdb.NewBloomFilter(10) + bbto := gorocksdb.NewDefaultBlockBasedTableOptions() + bbto.SetFilterPolicy(filter) + opts.SetBlockBasedTableFactory(bbto) + db, err := gorocksdb.OpenDb(opts, "/path/to/db") + +If you're using a custom comparator in your code, be aware you may have to +make your own filter policy object. + +This documentation is not a complete discussion of RocksDB. Please read the +RocksDB documentation for information on its +operation. You'll find lots of goodies there. +*/ +package gorocksdb diff --git a/gorocksdb/dynflag.go b/gorocksdb/dynflag.go new file mode 100644 index 000000000..18c18f404 --- /dev/null +++ b/gorocksdb/dynflag.go @@ -0,0 +1,6 @@ +// +build !linux !static + +package gorocksdb + +// #cgo LDFLAGS: -lrocksdb -lstdc++ -lm -ldl +import "C" diff --git a/gorocksdb/env.go b/gorocksdb/env.go new file mode 100644 index 000000000..11e84ef8c --- /dev/null +++ b/gorocksdb/env.go @@ -0,0 +1,45 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// Env is a system call environment used by a database. +type Env struct { + c *C.rocksdb_env_t +} + +// NewDefaultEnv creates a default environment. +func NewDefaultEnv() *Env { + return NewNativeEnv(C.rocksdb_create_default_env()) +} + +// NewMemEnv creates MemEnv for in-memory testing. +func NewMemEnv() *Env { + return NewNativeEnv(C.rocksdb_create_mem_env()) +} + +// NewNativeEnv creates a Environment object. +func NewNativeEnv(c *C.rocksdb_env_t) *Env { + return &Env{c} +} + +// SetBackgroundThreads sets the number of background worker threads +// of a specific thread pool for this environment. +// 'LOW' is the default pool. +// Default: 1 +func (env *Env) SetBackgroundThreads(n int) { + C.rocksdb_env_set_background_threads(env.c, C.int(n)) +} + +// SetHighPriorityBackgroundThreads sets the size of the high priority +// thread pool that can be used to prevent compactions from stalling +// memtable flushes. +func (env *Env) SetHighPriorityBackgroundThreads(n int) { + C.rocksdb_env_set_high_priority_background_threads(env.c, C.int(n)) +} + +// Destroy deallocates the Env object. +func (env *Env) Destroy() { + C.rocksdb_env_destroy(env.c) + env.c = nil +} diff --git a/gorocksdb/filter_policy.go b/gorocksdb/filter_policy.go new file mode 100644 index 000000000..696bd993d --- /dev/null +++ b/gorocksdb/filter_policy.go @@ -0,0 +1,94 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// FilterPolicy is a factory type that allows the RocksDB database to create a +// filter, such as a bloom filter, which will used to reduce reads. +type FilterPolicy interface { + // keys contains a list of keys (potentially with duplicates) + // that are ordered according to the user supplied comparator. + CreateFilter(keys [][]byte) []byte + + // "filter" contains the data appended by a preceding call to + // CreateFilter(). This method must return true if + // the key was in the list of keys passed to CreateFilter(). + // This method may return true or false if the key was not on the + // list, but it should aim to return false with a high probability. + KeyMayMatch(key []byte, filter []byte) bool + + // Return the name of this policy. + Name() string +} + +// NewNativeFilterPolicy creates a FilterPolicy object. +func NewNativeFilterPolicy(c *C.rocksdb_filterpolicy_t) FilterPolicy { + return nativeFilterPolicy{c} +} + +type nativeFilterPolicy struct { + c *C.rocksdb_filterpolicy_t +} + +func (fp nativeFilterPolicy) CreateFilter(keys [][]byte) []byte { return nil } +func (fp nativeFilterPolicy) KeyMayMatch(key []byte, filter []byte) bool { return false } +func (fp nativeFilterPolicy) Name() string { return "" } + +// NewBloomFilter returns a new filter policy that uses a bloom filter with approximately +// the specified number of bits per key. A good value for bits_per_key +// is 10, which yields a filter with ~1% false positive rate. +// +// Note: if you are using a custom comparator that ignores some parts +// of the keys being compared, you must not use NewBloomFilterPolicy() +// and must provide your own FilterPolicy that also ignores the +// corresponding parts of the keys. For example, if the comparator +// ignores trailing spaces, it would be incorrect to use a +// FilterPolicy (like NewBloomFilterPolicy) that does not ignore +// trailing spaces in keys. +func NewBloomFilter(bitsPerKey int) FilterPolicy { + return NewNativeFilterPolicy(C.rocksdb_filterpolicy_create_bloom(C.double(bitsPerKey))) +} + +// NewBloomFilterFull returns a new filter policy created with use_block_based_builder=false +// (use full or partitioned filter). +func NewBloomFilterFull(bitsPerKey int) FilterPolicy { + return NewNativeFilterPolicy(C.rocksdb_filterpolicy_create_bloom_full(C.double(bitsPerKey))) +} + +// Hold references to filter policies. +var filterPolicies = NewCOWList() + +type filterPolicyWrapper struct { + name *C.char + filterPolicy FilterPolicy +} + +func registerFilterPolicy(fp FilterPolicy) int { + return filterPolicies.Append(filterPolicyWrapper{C.CString(fp.Name()), fp}) +} + +//export gorocksdb_filterpolicy_create_filter +func gorocksdb_filterpolicy_create_filter(idx int, cKeys **C.char, cKeysLen *C.size_t, cNumKeys C.int, cDstLen *C.size_t) *C.char { + rawKeys := charSlice(cKeys, cNumKeys) + keysLen := sizeSlice(cKeysLen, cNumKeys) + keys := make([][]byte, int(cNumKeys)) + for i, len := range keysLen { + keys[i] = charToByte(rawKeys[i], len) + } + + dst := filterPolicies.Get(idx).(filterPolicyWrapper).filterPolicy.CreateFilter(keys) + *cDstLen = C.size_t(len(dst)) + return cByteSlice(dst) +} + +//export gorocksdb_filterpolicy_key_may_match +func gorocksdb_filterpolicy_key_may_match(idx int, cKey *C.char, cKeyLen C.size_t, cFilter *C.char, cFilterLen C.size_t) C.uchar { + key := charToByte(cKey, cKeyLen) + filter := charToByte(cFilter, cFilterLen) + return boolToChar(filterPolicies.Get(idx).(filterPolicyWrapper).filterPolicy.KeyMayMatch(key, filter)) +} + +//export gorocksdb_filterpolicy_name +func gorocksdb_filterpolicy_name(idx int) *C.char { + return filterPolicies.Get(idx).(filterPolicyWrapper).name +} diff --git a/gorocksdb/filter_policy_test.go b/gorocksdb/filter_policy_test.go new file mode 100644 index 000000000..c15aa1b66 --- /dev/null +++ b/gorocksdb/filter_policy_test.go @@ -0,0 +1,76 @@ +package gorocksdb + +import ( + "testing" + + "github.com/facebookgo/ensure" +) + +// fatalAsError is used as a wrapper to make it possible to use ensure +// also if C calls Go otherwise it will throw a internal lockOSThread error. +type fatalAsError struct { + t *testing.T +} + +func (f *fatalAsError) Fatal(a ...interface{}) { + f.t.Error(a...) +} + +func TestFilterPolicy(t *testing.T) { + var ( + givenKeys = [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} + givenFilter = []byte("key") + createFilterCalled = false + keyMayMatchCalled = false + ) + policy := &mockFilterPolicy{ + createFilter: func(keys [][]byte) []byte { + createFilterCalled = true + ensure.DeepEqual(&fatalAsError{t}, keys, givenKeys) + return givenFilter + }, + keyMayMatch: func(key, filter []byte) bool { + keyMayMatchCalled = true + ensure.DeepEqual(&fatalAsError{t}, key, givenKeys[0]) + ensure.DeepEqual(&fatalAsError{t}, filter, givenFilter) + return true + }, + } + + db := newTestDB(t, "TestFilterPolicy", func(opts *Options) { + blockOpts := NewDefaultBlockBasedTableOptions() + blockOpts.SetFilterPolicy(policy) + opts.SetBlockBasedTableFactory(blockOpts) + }) + defer db.Close() + + // insert keys + wo := NewDefaultWriteOptions() + for _, k := range givenKeys { + ensure.Nil(t, db.Put(wo, k, []byte("val"))) + } + + // flush to trigger the filter creation + ensure.Nil(t, db.Flush(NewDefaultFlushOptions())) + ensure.True(t, createFilterCalled) + + // test key may match call + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, givenKeys[0]) + defer v1.Free() + ensure.Nil(t, err) + ensure.True(t, keyMayMatchCalled) +} + +type mockFilterPolicy struct { + createFilter func(keys [][]byte) []byte + keyMayMatch func(key, filter []byte) bool +} + +func (m *mockFilterPolicy) Name() string { return "gorocksdb.test" } +func (m *mockFilterPolicy) CreateFilter(keys [][]byte) []byte { + return m.createFilter(keys) +} +func (m *mockFilterPolicy) KeyMayMatch(key, filter []byte) bool { + return m.keyMayMatch(key, filter) +} diff --git a/gorocksdb/go.mod b/gorocksdb/go.mod new file mode 100644 index 000000000..4fa450a1c --- /dev/null +++ b/gorocksdb/go.mod @@ -0,0 +1,16 @@ +module gorocksdb + +go 1.18 + +require ( + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c + github.com/stretchr/testify v1.8.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/gorocksdb/go.sum b/gorocksdb/go.sum new file mode 100644 index 000000000..622d18db7 --- /dev/null +++ b/gorocksdb/go.sum @@ -0,0 +1,23 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/gorocksdb/gorocksdb.c b/gorocksdb/gorocksdb.c new file mode 100644 index 000000000..efebbe510 --- /dev/null +++ b/gorocksdb/gorocksdb.c @@ -0,0 +1,70 @@ +#include "gorocksdb.h" +#include "_cgo_export.h" + +/* Base */ + +void gorocksdb_destruct_handler(void* state) { } + +/* Comparator */ + +rocksdb_comparator_t* gorocksdb_comparator_create(uintptr_t idx) { + return rocksdb_comparator_create( + (void*)idx, + gorocksdb_destruct_handler, + (int (*)(void*, const char*, size_t, const char*, size_t))(gorocksdb_comparator_compare), + (const char *(*)(void*))(gorocksdb_comparator_name)); +} + +/* CompactionFilter */ + +rocksdb_compactionfilter_t* gorocksdb_compactionfilter_create(uintptr_t idx) { + return rocksdb_compactionfilter_create( + (void*)idx, + gorocksdb_destruct_handler, + (unsigned char (*)(void*, int, const char*, size_t, const char*, size_t, char**, size_t*, unsigned char*))(gorocksdb_compactionfilter_filter), + (const char *(*)(void*))(gorocksdb_compactionfilter_name)); +} + +/* Filter Policy */ + +rocksdb_filterpolicy_t* gorocksdb_filterpolicy_create(uintptr_t idx) { + return rocksdb_filterpolicy_create( + (void*)idx, + gorocksdb_destruct_handler, + (char* (*)(void*, const char* const*, const size_t*, int, size_t*))(gorocksdb_filterpolicy_create_filter), + (unsigned char (*)(void*, const char*, size_t, const char*, size_t))(gorocksdb_filterpolicy_key_may_match), + gorocksdb_filterpolicy_delete_filter, + (const char *(*)(void*))(gorocksdb_filterpolicy_name)); +} + +void gorocksdb_filterpolicy_delete_filter(void* state, const char* v, size_t s) { + free((char*)v); +} + +/* Merge Operator */ + +rocksdb_mergeoperator_t* gorocksdb_mergeoperator_create(uintptr_t idx) { + return rocksdb_mergeoperator_create( + (void*)idx, + gorocksdb_destruct_handler, + (char* (*)(void*, const char*, size_t, const char*, size_t, const char* const*, const size_t*, int, unsigned char*, size_t*))(gorocksdb_mergeoperator_full_merge), + (char* (*)(void*, const char*, size_t, const char* const*, const size_t*, int, unsigned char*, size_t*))(gorocksdb_mergeoperator_partial_merge_multi), + gorocksdb_mergeoperator_delete_value, + (const char* (*)(void*))(gorocksdb_mergeoperator_name)); +} + +void gorocksdb_mergeoperator_delete_value(void* id, const char* v, size_t s) { + free((char*)v); +} + +/* Slice Transform */ + +rocksdb_slicetransform_t* gorocksdb_slicetransform_create(uintptr_t idx) { + return rocksdb_slicetransform_create( + (void*)idx, + gorocksdb_destruct_handler, + (char* (*)(void*, const char*, size_t, size_t*))(gorocksdb_slicetransform_transform), + (unsigned char (*)(void*, const char*, size_t))(gorocksdb_slicetransform_in_domain), + (unsigned char (*)(void*, const char*, size_t))(gorocksdb_slicetransform_in_range), + (const char* (*)(void*))(gorocksdb_slicetransform_name)); +} diff --git a/gorocksdb/gorocksdb.h b/gorocksdb/gorocksdb.h new file mode 100644 index 000000000..4a9968f08 --- /dev/null +++ b/gorocksdb/gorocksdb.h @@ -0,0 +1,30 @@ +#include +#include "rocksdb/c.h" + +// This API provides convenient C wrapper functions for rocksdb client. + +/* Base */ + +extern void gorocksdb_destruct_handler(void* state); + +/* CompactionFilter */ + +extern rocksdb_compactionfilter_t* gorocksdb_compactionfilter_create(uintptr_t idx); + +/* Comparator */ + +extern rocksdb_comparator_t* gorocksdb_comparator_create(uintptr_t idx); + +/* Filter Policy */ + +extern rocksdb_filterpolicy_t* gorocksdb_filterpolicy_create(uintptr_t idx); +extern void gorocksdb_filterpolicy_delete_filter(void* state, const char* v, size_t s); + +/* Merge Operator */ + +extern rocksdb_mergeoperator_t* gorocksdb_mergeoperator_create(uintptr_t idx); +extern void gorocksdb_mergeoperator_delete_value(void* state, const char* v, size_t s); + +/* Slice Transform */ + +extern rocksdb_slicetransform_t* gorocksdb_slicetransform_create(uintptr_t idx); diff --git a/gorocksdb/iterator.go b/gorocksdb/iterator.go new file mode 100644 index 000000000..fefb82f1a --- /dev/null +++ b/gorocksdb/iterator.go @@ -0,0 +1,126 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import ( + "bytes" + "errors" + "unsafe" +) + +// Iterator provides a way to seek to specific keys and iterate through +// the keyspace from that point, as well as access the values of those keys. +// +// For example: +// +// it := db.NewIterator(readOpts) +// defer it.Close() +// +// it.Seek([]byte("foo")) +// for ; it.Valid(); it.Next() { +// fmt.Printf("Key: %v Value: %v\n", it.Key().Data(), it.Value().Data()) +// } +// +// if err := it.Err(); err != nil { +// return err +// } +// +type Iterator struct { + c *C.rocksdb_iterator_t +} + +// NewNativeIterator creates a Iterator object. +func NewNativeIterator(c unsafe.Pointer) *Iterator { + return &Iterator{(*C.rocksdb_iterator_t)(c)} +} + +// Valid returns false only when an Iterator has iterated past either the +// first or the last key in the database. +func (iter *Iterator) Valid() bool { + return C.rocksdb_iter_valid(iter.c) != 0 +} + +// ValidForPrefix returns false only when an Iterator has iterated past the +// first or the last key in the database or the specified prefix. +func (iter *Iterator) ValidForPrefix(prefix []byte) bool { + if C.rocksdb_iter_valid(iter.c) == 0 { + return false + } + + key := iter.Key() + result := bytes.HasPrefix(key.Data(), prefix) + key.Free() + return result +} + +// Key returns the key the iterator currently holds. +func (iter *Iterator) Key() *Slice { + var cLen C.size_t + cKey := C.rocksdb_iter_key(iter.c, &cLen) + if cKey == nil { + return nil + } + return &Slice{cKey, cLen, true} +} + +// Value returns the value in the database the iterator currently holds. +func (iter *Iterator) Value() *Slice { + var cLen C.size_t + cVal := C.rocksdb_iter_value(iter.c, &cLen) + if cVal == nil { + return nil + } + return &Slice{cVal, cLen, true} +} + +// Next moves the iterator to the next sequential key in the database. +func (iter *Iterator) Next() { + C.rocksdb_iter_next(iter.c) +} + +// Prev moves the iterator to the previous sequential key in the database. +func (iter *Iterator) Prev() { + C.rocksdb_iter_prev(iter.c) +} + +// SeekToFirst moves the iterator to the first key in the database. +func (iter *Iterator) SeekToFirst() { + C.rocksdb_iter_seek_to_first(iter.c) +} + +// SeekToLast moves the iterator to the last key in the database. +func (iter *Iterator) SeekToLast() { + C.rocksdb_iter_seek_to_last(iter.c) +} + +// Seek moves the iterator to the position greater than or equal to the key. +func (iter *Iterator) Seek(key []byte) { + cKey := byteToChar(key) + C.rocksdb_iter_seek(iter.c, cKey, C.size_t(len(key))) +} + +// SeekForPrev moves the iterator to the last key that less than or equal +// to the target key, in contrast with Seek. +func (iter *Iterator) SeekForPrev(key []byte) { + cKey := byteToChar(key) + C.rocksdb_iter_seek_for_prev(iter.c, cKey, C.size_t(len(key))) +} + +// Err returns nil if no errors happened during iteration, or the actual +// error otherwise. +func (iter *Iterator) Err() error { + var cErr *C.char + C.rocksdb_iter_get_error(iter.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Close closes the iterator. +func (iter *Iterator) Close() { + C.rocksdb_iter_destroy(iter.c) + iter.c = nil +} diff --git a/gorocksdb/iterator_test.go b/gorocksdb/iterator_test.go new file mode 100644 index 000000000..358400ba7 --- /dev/null +++ b/gorocksdb/iterator_test.go @@ -0,0 +1,31 @@ +package gorocksdb + +import ( + "testing" + + "github.com/facebookgo/ensure" +) + +func TestIterator(t *testing.T) { + db := newTestDB(t, "TestIterator", nil) + defer db.Close() + + // insert keys + givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} + wo := NewDefaultWriteOptions() + for _, k := range givenKeys { + ensure.Nil(t, db.Put(wo, k, []byte("val"))) + } + + ro := NewDefaultReadOptions() + iter := db.NewIterator(ro) + defer iter.Close() + var actualKeys [][]byte + for iter.SeekToFirst(); iter.Valid(); iter.Next() { + key := make([]byte, 4) + copy(key, iter.Key().Data()) + actualKeys = append(actualKeys, key) + } + ensure.Nil(t, iter.Err()) + ensure.DeepEqual(t, actualKeys, givenKeys) +} diff --git a/gorocksdb/memory_usage.go b/gorocksdb/memory_usage.go new file mode 100644 index 000000000..7b9a6ad69 --- /dev/null +++ b/gorocksdb/memory_usage.go @@ -0,0 +1,58 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import ( + "errors" + "unsafe" +) + +// MemoryUsage contains memory usage statistics provided by RocksDB +type MemoryUsage struct { + // MemTableTotal estimates memory usage of all mem-tables + MemTableTotal uint64 + // MemTableUnflushed estimates memory usage of unflushed mem-tables + MemTableUnflushed uint64 + // MemTableReadersTotal memory usage of table readers (indexes and bloom filters) + MemTableReadersTotal uint64 + // CacheTotal memory usage of cache + CacheTotal uint64 +} + +// GetApproximateMemoryUsageByType returns summary +// memory usage stats for given databases and caches. +func GetApproximateMemoryUsageByType(dbs []*DB, caches []*Cache) (*MemoryUsage, error) { + // register memory consumers + consumers := C.rocksdb_memory_consumers_create() + defer C.rocksdb_memory_consumers_destroy(consumers) + + for _, db := range dbs { + if db != nil { + C.rocksdb_memory_consumers_add_db(consumers, db.c) + } + } + for _, cache := range caches { + if cache != nil { + C.rocksdb_memory_consumers_add_cache(consumers, cache.c) + } + } + + // obtain memory usage stats + var cErr *C.char + memoryUsage := C.rocksdb_approximate_memory_usage_create(consumers, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + + defer C.rocksdb_approximate_memory_usage_destroy(memoryUsage) + + result := &MemoryUsage{ + MemTableTotal: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_total(memoryUsage)), + MemTableUnflushed: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_unflushed(memoryUsage)), + MemTableReadersTotal: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_readers_total(memoryUsage)), + CacheTotal: uint64(C.rocksdb_approximate_memory_usage_get_cache_total(memoryUsage)), + } + return result, nil +} diff --git a/gorocksdb/memory_usage_test.go b/gorocksdb/memory_usage_test.go new file mode 100644 index 000000000..7fc6eaa3f --- /dev/null +++ b/gorocksdb/memory_usage_test.go @@ -0,0 +1,56 @@ +package gorocksdb + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/facebookgo/ensure" +) + +func TestMemoryUsage(t *testing.T) { + // create database with cache + cache := NewLRUCache(8 * 1024 * 1024) + bbto := NewDefaultBlockBasedTableOptions() + bbto.SetBlockCache(cache) + defer cache.Destroy() + + applyOpts := func(opts *Options) { + opts.SetBlockBasedTableFactory(bbto) + } + + db := newTestDB(t, "TestMemoryUsage", applyOpts) + defer db.Close() + + // take first memory usage snapshot + mu1, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) + ensure.Nil(t, err) + + // perform IO operations that will affect in-memory tables (and maybe cache as well) + wo := NewDefaultWriteOptions() + defer wo.Destroy() + ro := NewDefaultReadOptions() + defer ro.Destroy() + + key := []byte("key") + value := make([]byte, 1024) + _, err = rand.Read(value) + ensure.Nil(t, err) + + err = db.Put(wo, key, value) + ensure.Nil(t, err) + _, err = db.Get(ro, key) + ensure.Nil(t, err) + + // take second memory usage snapshot + mu2, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) + ensure.Nil(t, err) + + // the amount of memory used by memtables should increase after write/read; + // cache memory usage is not likely to be changed, perhaps because requested key is kept by memtable + assert.True(t, mu2.MemTableTotal > mu1.MemTableTotal) + assert.True(t, mu2.MemTableUnflushed > mu1.MemTableUnflushed) + assert.True(t, mu2.CacheTotal >= mu1.CacheTotal) + assert.True(t, mu2.MemTableReadersTotal >= mu1.MemTableReadersTotal) +} diff --git a/gorocksdb/merge_operator.go b/gorocksdb/merge_operator.go new file mode 100644 index 000000000..2de7f9abc --- /dev/null +++ b/gorocksdb/merge_operator.go @@ -0,0 +1,168 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// A MergeOperator specifies the SEMANTICS of a merge, which only +// client knows. It could be numeric addition, list append, string +// concatenation, edit data structure, ... , anything. +// The library, on the other hand, is concerned with the exercise of this +// interface, at the right time (during get, iteration, compaction...) +// +// Please read the RocksDB documentation for +// more details and example implementations. +type MergeOperator interface { + // Gives the client a way to express the read -> modify -> write semantics + // key: The key that's associated with this merge operation. + // Client could multiplex the merge operator based on it + // if the key space is partitioned and different subspaces + // refer to different types of data which have different + // merge operation semantics. + // existingValue: null indicates that the key does not exist before this op. + // operands: the sequence of merge operations to apply, front() first. + // + // Return true on success. + // + // All values passed in will be client-specific values. So if this method + // returns false, it is because client specified bad data or there was + // internal corruption. This will be treated as an error by the library. + FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) + + // The name of the MergeOperator. + Name() string +} + +// PartialMerger implements PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, err) +// When a MergeOperator implements this interface, PartialMerge will be called in addition +// to FullMerge for compactions across levels +type PartialMerger interface { + // This function performs merge(left_op, right_op) + // when both the operands are themselves merge operation types + // that you would have passed to a db.Merge() call in the same order + // (i.e.: db.Merge(key,left_op), followed by db.Merge(key,right_op)). + // + // PartialMerge should combine them into a single merge operation. + // The return value should be constructed such that a call to + // db.Merge(key, new_value) would yield the same result as a call + // to db.Merge(key, left_op) followed by db.Merge(key, right_op). + // + // If it is impossible or infeasible to combine the two operations, return false. + // The library will internally keep track of the operations, and apply them in the + // correct order once a base-value (a Put/Delete/End-of-Database) is seen. + PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) +} + +// MultiMerger implements PartialMergeMulti(key []byte, operands [][]byte) ([]byte, err) +// When a MergeOperator implements this interface, PartialMergeMulti will be called in addition +// to FullMerge for compactions across levels +type MultiMerger interface { + // PartialMerge performs merge on multiple operands + // when all of the operands are themselves merge operation types + // that you would have passed to a db.Merge() call in the same order + // (i.e.: db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), + // ... db.Merge(key, operand[n])). + // + // PartialMerge should combine them into a single merge operation. + // The return value should be constructed such that a call to + // db.Merge(key, new_value) would yield the same result as a call + // to db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), + // ... db.Merge(key, operand[n])). + // + // If it is impossible or infeasible to combine the operations, return false. + // The library will internally keep track of the operations, and apply them in the + // correct order once a base-value (a Put/Delete/End-of-Database) is seen. + PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) +} + +// NewNativeMergeOperator creates a MergeOperator object. +func NewNativeMergeOperator(c *C.rocksdb_mergeoperator_t) MergeOperator { + return nativeMergeOperator{c} +} + +type nativeMergeOperator struct { + c *C.rocksdb_mergeoperator_t +} + +func (mo nativeMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + return nil, false +} +func (mo nativeMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { + return nil, false +} +func (mo nativeMergeOperator) Name() string { return "" } + +// Hold references to merge operators. +var mergeOperators = NewCOWList() + +type mergeOperatorWrapper struct { + name *C.char + mergeOperator MergeOperator +} + +func registerMergeOperator(merger MergeOperator) int { + return mergeOperators.Append(mergeOperatorWrapper{C.CString(merger.Name()), merger}) +} + +//export gorocksdb_mergeoperator_full_merge +func gorocksdb_mergeoperator_full_merge(idx int, cKey *C.char, cKeyLen C.size_t, cExistingValue *C.char, cExistingValueLen C.size_t, cOperands **C.char, cOperandsLen *C.size_t, cNumOperands C.int, cSuccess *C.uchar, cNewValueLen *C.size_t) *C.char { + key := charToByte(cKey, cKeyLen) + rawOperands := charSlice(cOperands, cNumOperands) + operandsLen := sizeSlice(cOperandsLen, cNumOperands) + existingValue := charToByte(cExistingValue, cExistingValueLen) + operands := make([][]byte, int(cNumOperands)) + for i, len := range operandsLen { + operands[i] = charToByte(rawOperands[i], len) + } + + newValue, success := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator.FullMerge(key, existingValue, operands) + newValueLen := len(newValue) + + *cNewValueLen = C.size_t(newValueLen) + *cSuccess = boolToChar(success) + + return cByteSlice(newValue) +} + +//export gorocksdb_mergeoperator_partial_merge_multi +func gorocksdb_mergeoperator_partial_merge_multi(idx int, cKey *C.char, cKeyLen C.size_t, cOperands **C.char, cOperandsLen *C.size_t, cNumOperands C.int, cSuccess *C.uchar, cNewValueLen *C.size_t) *C.char { + key := charToByte(cKey, cKeyLen) + rawOperands := charSlice(cOperands, cNumOperands) + operandsLen := sizeSlice(cOperandsLen, cNumOperands) + operands := make([][]byte, int(cNumOperands)) + for i, len := range operandsLen { + operands[i] = charToByte(rawOperands[i], len) + } + + var newValue []byte + success := true + + merger := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator + + // check if this MergeOperator supports partial or multi merges + switch v := merger.(type) { + case MultiMerger: + newValue, success = v.PartialMergeMulti(key, operands) + case PartialMerger: + leftOperand := operands[0] + for i := 1; i < int(cNumOperands); i++ { + newValue, success = v.PartialMerge(key, leftOperand, operands[i]) + if !success { + break + } + leftOperand = newValue + } + default: + success = false + } + + newValueLen := len(newValue) + *cNewValueLen = C.size_t(newValueLen) + *cSuccess = boolToChar(success) + + return cByteSlice(newValue) +} + +//export gorocksdb_mergeoperator_name +func gorocksdb_mergeoperator_name(idx int) *C.char { + return mergeOperators.Get(idx).(mergeOperatorWrapper).name +} diff --git a/gorocksdb/merge_operator_test.go b/gorocksdb/merge_operator_test.go new file mode 100644 index 000000000..9dad6f78a --- /dev/null +++ b/gorocksdb/merge_operator_test.go @@ -0,0 +1,185 @@ +package gorocksdb + +import ( + "testing" + + "github.com/facebookgo/ensure" +) + +func TestMergeOperator(t *testing.T) { + var ( + givenKey = []byte("hello") + givenVal1 = []byte("foo") + givenVal2 = []byte("bar") + givenMerged = []byte("foobar") + ) + merger := &mockMergeOperator{ + fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, existingValue, givenVal1) + ensure.DeepEqual(&fatalAsError{t}, operands, [][]byte{givenVal2}) + return givenMerged, true + }, + } + db := newTestDB(t, "TestMergeOperator", func(opts *Options) { + opts.SetMergeOperator(merger) + }) + defer db.Close() + + wo := NewDefaultWriteOptions() + ensure.Nil(t, db.Put(wo, givenKey, givenVal1)) + ensure.Nil(t, db.Merge(wo, givenKey, givenVal2)) + + // trigger a compaction to ensure that a merge is performed + db.CompactRange(Range{nil, nil}) + + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), givenMerged) +} + +func TestPartialMergeOperator(t *testing.T) { + var ( + givenKey = []byte("hello") + startingVal = []byte("foo") + mergeVal1 = []byte("bar") + mergeVal2 = []byte("baz") + fMergeResult = []byte("foobarbaz") + pMergeResult = []byte("barbaz") + ) + + merger := &mockMergePartialOperator{ + fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, existingValue, startingVal) + ensure.DeepEqual(&fatalAsError{t}, operands[0], pMergeResult) + return fMergeResult, true + }, + partialMerge: func(key, leftOperand, rightOperand []byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, leftOperand, mergeVal1) + ensure.DeepEqual(&fatalAsError{t}, rightOperand, mergeVal2) + return pMergeResult, true + }, + } + db := newTestDB(t, "TestMergeOperator", func(opts *Options) { + opts.SetMergeOperator(merger) + }) + defer db.Close() + + wo := NewDefaultWriteOptions() + defer wo.Destroy() + + // insert a starting value and compact to trigger merges + ensure.Nil(t, db.Put(wo, givenKey, startingVal)) + + // trigger a compaction to ensure that a merge is performed + db.CompactRange(Range{nil, nil}) + + // we expect these two operands to be passed to merge partial + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal1)) + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal2)) + + // trigger a compaction to ensure that a + // partial and full merge are performed + db.CompactRange(Range{nil, nil}) + + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), fMergeResult) + +} + +func TestMergeMultiOperator(t *testing.T) { + var ( + givenKey = []byte("hello") + startingVal = []byte("foo") + mergeVal1 = []byte("bar") + mergeVal2 = []byte("baz") + fMergeResult = []byte("foobarbaz") + pMergeResult = []byte("barbaz") + ) + + merger := &mockMergeMultiOperator{ + fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, existingValue, startingVal) + ensure.DeepEqual(&fatalAsError{t}, operands[0], pMergeResult) + return fMergeResult, true + }, + partialMergeMulti: func(key []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, operands[0], mergeVal1) + ensure.DeepEqual(&fatalAsError{t}, operands[1], mergeVal2) + return pMergeResult, true + }, + } + db := newTestDB(t, "TestMergeOperator", func(opts *Options) { + opts.SetMergeOperator(merger) + }) + defer db.Close() + + wo := NewDefaultWriteOptions() + defer wo.Destroy() + + // insert a starting value and compact to trigger merges + ensure.Nil(t, db.Put(wo, givenKey, startingVal)) + + // trigger a compaction to ensure that a merge is performed + db.CompactRange(Range{nil, nil}) + + // we expect these two operands to be passed to merge multi + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal1)) + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal2)) + + // trigger a compaction to ensure that a + // partial and full merge are performed + db.CompactRange(Range{nil, nil}) + + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), fMergeResult) + +} + +// Mock Objects +type mockMergeOperator struct { + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) +} + +func (m *mockMergeOperator) Name() string { return "gorocksdb.test" } +func (m *mockMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + return m.fullMerge(key, existingValue, operands) +} + +type mockMergeMultiOperator struct { + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) + partialMergeMulti func(key []byte, operands [][]byte) ([]byte, bool) +} + +func (m *mockMergeMultiOperator) Name() string { return "gorocksdb.multi" } +func (m *mockMergeMultiOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + return m.fullMerge(key, existingValue, operands) +} +func (m *mockMergeMultiOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { + return m.partialMergeMulti(key, operands) +} + +type mockMergePartialOperator struct { + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) + partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) +} + +func (m *mockMergePartialOperator) Name() string { return "gorocksdb.partial" } +func (m *mockMergePartialOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + return m.fullMerge(key, existingValue, operands) +} +func (m *mockMergePartialOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { + return m.partialMerge(key, leftOperand, rightOperand) +} diff --git a/gorocksdb/options.go b/gorocksdb/options.go new file mode 100644 index 000000000..070002154 --- /dev/null +++ b/gorocksdb/options.go @@ -0,0 +1,1213 @@ +package gorocksdb + +// #include "rocksdb/c.h" +// #include "gorocksdb.h" +import "C" +import ( + "errors" + "unsafe" +) + +// CompressionType specifies the block compression. +// DB contents are stored in a set of blocks, each of which holds a +// sequence of key,value pairs. Each block may be compressed before +// being stored in a file. The following enum describes which +// compression method (if any) is used to compress a block. +type CompressionType uint + +// Compression types. +const ( + NoCompression = CompressionType(C.rocksdb_no_compression) + SnappyCompression = CompressionType(C.rocksdb_snappy_compression) + ZLibCompression = CompressionType(C.rocksdb_zlib_compression) + Bz2Compression = CompressionType(C.rocksdb_bz2_compression) + LZ4Compression = CompressionType(C.rocksdb_lz4_compression) + LZ4HCCompression = CompressionType(C.rocksdb_lz4hc_compression) + XpressCompression = CompressionType(C.rocksdb_xpress_compression) + ZSTDCompression = CompressionType(C.rocksdb_zstd_compression) +) + +// CompactionStyle specifies the compaction style. +type CompactionStyle uint + +// Compaction styles. +const ( + LevelCompactionStyle = CompactionStyle(C.rocksdb_level_compaction) + UniversalCompactionStyle = CompactionStyle(C.rocksdb_universal_compaction) + FIFOCompactionStyle = CompactionStyle(C.rocksdb_fifo_compaction) +) + +// CompactionAccessPattern specifies the access patern in compaction. +type CompactionAccessPattern uint + +// Access patterns for compaction. +const ( + NoneCompactionAccessPattern = CompactionAccessPattern(0) + NormalCompactionAccessPattern = CompactionAccessPattern(1) + SequentialCompactionAccessPattern = CompactionAccessPattern(2) + WillneedCompactionAccessPattern = CompactionAccessPattern(3) +) + +// InfoLogLevel describes the log level. +type InfoLogLevel uint + +// Log leves. +const ( + DebugInfoLogLevel = InfoLogLevel(0) + InfoInfoLogLevel = InfoLogLevel(1) + WarnInfoLogLevel = InfoLogLevel(2) + ErrorInfoLogLevel = InfoLogLevel(3) + FatalInfoLogLevel = InfoLogLevel(4) +) + +type WALRecoveryMode int + +const ( + TolerateCorruptedTailRecordsRecovery = WALRecoveryMode(0) + AbsoluteConsistencyRecovery = WALRecoveryMode(1) + PointInTimeRecovery = WALRecoveryMode(2) + SkipAnyCorruptedRecordsRecovery = WALRecoveryMode(3) +) + +// Options represent all of the available options when opening a database with Open. +type Options struct { + c *C.rocksdb_options_t + + // Hold references for GC. + env *Env + bbto *BlockBasedTableOptions + + // We keep these so we can free their memory in Destroy. + ccmp *C.rocksdb_comparator_t + cmo *C.rocksdb_mergeoperator_t + cst *C.rocksdb_slicetransform_t + ccf *C.rocksdb_compactionfilter_t +} + +// NewDefaultOptions creates the default Options. +func NewDefaultOptions() *Options { + return NewNativeOptions(C.rocksdb_options_create()) +} + +// NewNativeOptions creates a Options object. +func NewNativeOptions(c *C.rocksdb_options_t) *Options { + return &Options{c: c} +} + +// GetOptionsFromString creates a Options object from existing opt and string. +// If base is nil, a default opt create by NewDefaultOptions will be used as base opt. +func GetOptionsFromString(base *Options, optStr string) (*Options, error) { + if base == nil { + base = NewDefaultOptions() + defer base.Destroy() + } + + var ( + cErr *C.char + cOptStr = C.CString(optStr) + ) + defer C.free(unsafe.Pointer(cOptStr)) + + newOpt := NewDefaultOptions() + C.rocksdb_get_options_from_string(base.c, cOptStr, newOpt.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + + return newOpt, nil +} + +// ------------------- +// Parameters that affect behavior + +// SetCompactionFilter sets the specified compaction filter +// which will be applied on compactions. +// Default: nil +func (opts *Options) SetCompactionFilter(value CompactionFilter) { + if nc, ok := value.(nativeCompactionFilter); ok { + opts.ccf = nc.c + } else { + idx := registerCompactionFilter(value) + opts.ccf = C.gorocksdb_compactionfilter_create(C.uintptr_t(idx)) + } + C.rocksdb_options_set_compaction_filter(opts.c, opts.ccf) +} + +// SetComparator sets the comparator which define the order of keys in the table. +// Default: a comparator that uses lexicographic byte-wise ordering +func (opts *Options) SetComparator(value Comparator) { + if nc, ok := value.(nativeComparator); ok { + opts.ccmp = nc.c + } else { + idx := registerComperator(value) + opts.ccmp = C.gorocksdb_comparator_create(C.uintptr_t(idx)) + } + C.rocksdb_options_set_comparator(opts.c, opts.ccmp) +} + +// SetMergeOperator sets the merge operator which will be called +// if a merge operations are used. +// Default: nil +func (opts *Options) SetMergeOperator(value MergeOperator) { + if nmo, ok := value.(nativeMergeOperator); ok { + opts.cmo = nmo.c + } else { + idx := registerMergeOperator(value) + opts.cmo = C.gorocksdb_mergeoperator_create(C.uintptr_t(idx)) + } + C.rocksdb_options_set_merge_operator(opts.c, opts.cmo) +} + +// A single CompactionFilter instance to call into during compaction. +// Allows an application to modify/delete a key-value during background +// compaction. +// +// If the client requires a new compaction filter to be used for different +// compaction runs, it can specify compaction_filter_factory instead of this +// option. The client should specify only one of the two. +// compaction_filter takes precedence over compaction_filter_factory if +// client specifies both. +// +// If multithreaded compaction is being used, the supplied CompactionFilter +// instance may be used from different threads concurrently and so should be +// thread-safe. +// +// Default: nil +// TODO: implement in C +//func (opts *Options) SetCompactionFilter(value *CompactionFilter) { +// C.rocksdb_options_set_compaction_filter(opts.c, value.filter) +//} + +// This is a factory that provides compaction filter objects which allow +// an application to modify/delete a key-value during background compaction. +// +// A new filter will be created on each compaction run. If multithreaded +// compaction is being used, each created CompactionFilter will only be used +// from a single thread and so does not need to be thread-safe. +// +// Default: a factory that doesn't provide any object +// std::shared_ptr compaction_filter_factory; +// TODO: implement in C and Go + +// Version TWO of the compaction_filter_factory +// It supports rolling compaction +// +// Default: a factory that doesn't provide any object +// std::shared_ptr compaction_filter_factory_v2; +// TODO: implement in C and Go + +// SetCreateIfMissing specifies whether the database +// should be created if it is missing. +// Default: false +func (opts *Options) SetCreateIfMissing(value bool) { + C.rocksdb_options_set_create_if_missing(opts.c, boolToChar(value)) +} + +// SetErrorIfExists specifies whether an error should be raised +// if the database already exists. +// Default: false +func (opts *Options) SetErrorIfExists(value bool) { + C.rocksdb_options_set_error_if_exists(opts.c, boolToChar(value)) +} + +// SetParanoidChecks enable/disable paranoid checks. +// +// If true, the implementation will do aggressive checking of the +// data it is processing and will stop early if it detects any +// errors. This may have unforeseen ramifications: for example, a +// corruption of one DB entry may cause a large number of entries to +// become unreadable or for the entire DB to become unopenable. +// If any of the writes to the database fails (Put, Delete, Merge, Write), +// the database will switch to read-only mode and fail all other +// Write operations. +// Default: false +func (opts *Options) SetParanoidChecks(value bool) { + C.rocksdb_options_set_paranoid_checks(opts.c, boolToChar(value)) +} + +// SetDBPaths sets the DBPaths of the options. +// +// A list of paths where SST files can be put into, with its target size. +// Newer data is placed into paths specified earlier in the vector while +// older data gradually moves to paths specified later in the vector. +// +// For example, you have a flash device with 10GB allocated for the DB, +// as well as a hard drive of 2TB, you should config it to be: +// [{"/flash_path", 10GB}, {"/hard_drive", 2TB}] +// +// The system will try to guarantee data under each path is close to but +// not larger than the target size. But current and future file sizes used +// by determining where to place a file are based on best-effort estimation, +// which means there is a chance that the actual size under the directory +// is slightly more than target size under some workloads. User should give +// some buffer room for those cases. +// +// If none of the paths has sufficient room to place a file, the file will +// be placed to the last path anyway, despite to the target size. +// +// Placing newer data to earlier paths is also best-efforts. User should +// expect user files to be placed in higher levels in some extreme cases. +// +// If left empty, only one path will be used, which is db_name passed when +// opening the DB. +// Default: empty +func (opts *Options) SetDBPaths(dbpaths []*DBPath) { + l := len(dbpaths) + cDbpaths := make([]*C.rocksdb_dbpath_t, l) + for i, v := range dbpaths { + cDbpaths[i] = v.c + } + + C.rocksdb_options_set_db_paths(opts.c, &cDbpaths[0], C.size_t(l)) +} + +// SetEnv sets the specified object to interact with the environment, +// e.g. to read/write files, schedule background work, etc. +// Default: DefaultEnv +func (opts *Options) SetEnv(value *Env) { + opts.env = value + + C.rocksdb_options_set_env(opts.c, value.c) +} + +// SetInfoLogLevel sets the info log level. +// Default: InfoInfoLogLevel +func (opts *Options) SetInfoLogLevel(value InfoLogLevel) { + C.rocksdb_options_set_info_log_level(opts.c, C.int(value)) +} + +// IncreaseParallelism sets the parallelism. +// +// By default, RocksDB uses only one background thread for flush and +// compaction. Calling this function will set it up such that total of +// `total_threads` is used. Good value for `total_threads` is the number of +// cores. You almost definitely want to call this function if your system is +// bottlenecked by RocksDB. +func (opts *Options) IncreaseParallelism(total_threads int) { + C.rocksdb_options_increase_parallelism(opts.c, C.int(total_threads)) +} + +// OptimizeForPointLookup optimize the DB for point lookups. +// +// Use this if you don't need to keep the data sorted, i.e. you'll never use +// an iterator, only Put() and Get() API calls +// +// If you use this with rocksdb >= 5.0.2, you must call `SetAllowConcurrentMemtableWrites(false)` +// to avoid an assertion error immediately on opening the db. +func (opts *Options) OptimizeForPointLookup(block_cache_size_mb uint64) { + C.rocksdb_options_optimize_for_point_lookup(opts.c, C.uint64_t(block_cache_size_mb)) +} + +// Set whether to allow concurrent memtable writes. Conccurent writes are +// not supported by all memtable factories (currently only SkipList memtables). +// As of rocksdb 5.0.2 you must call `SetAllowConcurrentMemtableWrites(false)` +// if you use `OptimizeForPointLookup`. +func (opts *Options) SetAllowConcurrentMemtableWrites(allow bool) { + C.rocksdb_options_set_allow_concurrent_memtable_write(opts.c, boolToChar(allow)) +} + +// OptimizeLevelStyleCompaction optimize the DB for leveld compaction. +// +// Default values for some parameters in ColumnFamilyOptions are not +// optimized for heavy workloads and big datasets, which means you might +// observe write stalls under some conditions. As a starting point for tuning +// RocksDB options, use the following two functions: +// * OptimizeLevelStyleCompaction -- optimizes level style compaction +// * OptimizeUniversalStyleCompaction -- optimizes universal style compaction +// Universal style compaction is focused on reducing Write Amplification +// Factor for big data sets, but increases Space Amplification. You can learn +// more about the different styles here: +// https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide +// Make sure to also call IncreaseParallelism(), which will provide the +// biggest performance gains. +// Note: we might use more memory than memtable_memory_budget during high +// write rate period +func (opts *Options) OptimizeLevelStyleCompaction(memtable_memory_budget uint64) { + C.rocksdb_options_optimize_level_style_compaction(opts.c, C.uint64_t(memtable_memory_budget)) +} + +// OptimizeUniversalStyleCompaction optimize the DB for universal compaction. +// See note on OptimizeLevelStyleCompaction. +func (opts *Options) OptimizeUniversalStyleCompaction(memtable_memory_budget uint64) { + C.rocksdb_options_optimize_universal_style_compaction(opts.c, C.uint64_t(memtable_memory_budget)) +} + +// SetWriteBufferSize sets the amount of data to build up in memory +// (backed by an unsorted log on disk) before converting to a sorted on-disk file. +// +// Larger values increase performance, especially during bulk loads. +// Up to max_write_buffer_number write buffers may be held in memory +// at the same time, +// so you may wish to adjust this parameter to control memory usage. +// Also, a larger write buffer will result in a longer recovery time +// the next time the database is opened. +// Default: 64MB +func (opts *Options) SetWriteBufferSize(value int) { + C.rocksdb_options_set_write_buffer_size(opts.c, C.size_t(value)) +} + +// SetMaxWriteBufferNumber sets the maximum number of write buffers +// that are built up in memory. +// +// The default is 2, so that when 1 write buffer is being flushed to +// storage, new writes can continue to the other write buffer. +// Default: 2 +func (opts *Options) SetMaxWriteBufferNumber(value int) { + C.rocksdb_options_set_max_write_buffer_number(opts.c, C.int(value)) +} + +// SetMinWriteBufferNumberToMerge sets the minimum number of write buffers +// that will be merged together before writing to storage. +// +// If set to 1, then all write buffers are flushed to L0 as individual files +// and this increases read amplification because a get request has to check +// in all of these files. Also, an in-memory merge may result in writing lesser +// data to storage if there are duplicate records in each of these +// individual write buffers. +// Default: 1 +func (opts *Options) SetMinWriteBufferNumberToMerge(value int) { + C.rocksdb_options_set_min_write_buffer_number_to_merge(opts.c, C.int(value)) +} + +// SetMaxOpenFiles sets the number of open files that can be used by the DB. +// +// You may need to increase this if your database has a large working set +// (budget one open file per 2MB of working set). +// Default: 1000 +func (opts *Options) SetMaxOpenFiles(value int) { + C.rocksdb_options_set_max_open_files(opts.c, C.int(value)) +} + +// SetMaxFileOpeningThreads sets the maximum number of file opening threads. +// If max_open_files is -1, DB will open all files on DB::Open(). You can +// use this option to increase the number of threads used to open the files. +// Default: 16 +func (opts *Options) SetMaxFileOpeningThreads(value int) { + C.rocksdb_options_set_max_file_opening_threads(opts.c, C.int(value)) +} + +// SetMaxTotalWalSize sets the maximum total wal size in bytes. +// Once write-ahead logs exceed this size, we will start forcing the flush of +// column families whose memtables are backed by the oldest live WAL file +// (i.e. the ones that are causing all the space amplification). If set to 0 +// (default), we will dynamically choose the WAL size limit to be +// [sum of all write_buffer_size * max_write_buffer_number] * 4 +// Default: 0 +func (opts *Options) SetMaxTotalWalSize(value uint64) { + C.rocksdb_options_set_max_total_wal_size(opts.c, C.uint64_t(value)) +} + +// SetCompression sets the compression algorithm. +// Default: SnappyCompression, which gives lightweight but fast +// compression. +func (opts *Options) SetCompression(value CompressionType) { + C.rocksdb_options_set_compression(opts.c, C.int(value)) +} + +// SetCompressionPerLevel sets different compression algorithm per level. +// +// Different levels can have different compression policies. There +// are cases where most lower levels would like to quick compression +// algorithm while the higher levels (which have more data) use +// compression algorithms that have better compression but could +// be slower. This array should have an entry for +// each level of the database. This array overrides the +// value specified in the previous field 'compression'. +func (opts *Options) SetCompressionPerLevel(value []CompressionType) { + cLevels := make([]C.int, len(value)) + for i, v := range value { + cLevels[i] = C.int(v) + } + + C.rocksdb_options_set_compression_per_level(opts.c, &cLevels[0], C.size_t(len(value))) +} + +// SetMinLevelToCompress sets the start level to use compression. +func (opts *Options) SetMinLevelToCompress(value int) { + C.rocksdb_options_set_min_level_to_compress(opts.c, C.int(value)) +} + +// SetCompressionOptions sets different options for compression algorithms. +// Default: nil +func (opts *Options) SetCompressionOptions(value *CompressionOptions) { + C.rocksdb_options_set_compression_options(opts.c, C.int(value.WindowBits), C.int(value.Level), C.int(value.Strategy), C.int(value.MaxDictBytes)) +} + +// SetPrefixExtractor sets the prefic extractor. +// +// If set, use the specified function to determine the +// prefixes for keys. These prefixes will be placed in the filter. +// Depending on the workload, this can reduce the number of read-IOP +// cost for scans when a prefix is passed via ReadOptions to +// db.NewIterator(). +// Default: nil +func (opts *Options) SetPrefixExtractor(value SliceTransform) { + if nst, ok := value.(nativeSliceTransform); ok { + opts.cst = nst.c + } else { + idx := registerSliceTransform(value) + opts.cst = C.gorocksdb_slicetransform_create(C.uintptr_t(idx)) + } + C.rocksdb_options_set_prefix_extractor(opts.c, opts.cst) +} + +// SetNumLevels sets the number of levels for this database. +// Default: 7 +func (opts *Options) SetNumLevels(value int) { + C.rocksdb_options_set_num_levels(opts.c, C.int(value)) +} + +// SetLevel0FileNumCompactionTrigger sets the number of files +// to trigger level-0 compaction. +// +// A value <0 means that level-0 compaction will not be +// triggered by number of files at all. +// Default: 4 +func (opts *Options) SetLevel0FileNumCompactionTrigger(value int) { + C.rocksdb_options_set_level0_file_num_compaction_trigger(opts.c, C.int(value)) +} + +// SetLevel0SlowdownWritesTrigger sets the soft limit on number of level-0 files. +// +// We start slowing down writes at this point. +// A value <0 means that no writing slow down will be triggered by +// number of files in level-0. +// Default: 8 +func (opts *Options) SetLevel0SlowdownWritesTrigger(value int) { + C.rocksdb_options_set_level0_slowdown_writes_trigger(opts.c, C.int(value)) +} + +// SetLevel0StopWritesTrigger sets the maximum number of level-0 files. +// We stop writes at this point. +// Default: 12 +func (opts *Options) SetLevel0StopWritesTrigger(value int) { + C.rocksdb_options_set_level0_stop_writes_trigger(opts.c, C.int(value)) +} + +// SetMaxMemCompactionLevel sets the maximum level +// to which a new compacted memtable is pushed if it does not create overlap. +// +// We try to push to level 2 to avoid the +// relatively expensive level 0=>1 compactions and to avoid some +// expensive manifest file operations. We do not push all the way to +// the largest level since that can generate a lot of wasted disk +// space if the same key space is being repeatedly overwritten. +// Default: 2 +func (opts *Options) SetMaxMemCompactionLevel(value int) { + C.rocksdb_options_set_max_mem_compaction_level(opts.c, C.int(value)) +} + +// SetTargetFileSizeBase sets the target file size for compaction. +// +// Target file size is per-file size for level-1. +// Target file size for level L can be calculated by +// target_file_size_base * (target_file_size_multiplier ^ (L-1)) +// +// For example, if target_file_size_base is 2MB and +// target_file_size_multiplier is 10, then each file on level-1 will +// be 2MB, and each file on level 2 will be 20MB, +// and each file on level-3 will be 200MB. +// Default: 2MB +func (opts *Options) SetTargetFileSizeBase(value uint64) { + C.rocksdb_options_set_target_file_size_base(opts.c, C.uint64_t(value)) +} + +// SetTargetFileSizeMultiplier sets the target file size multiplier for compaction. +// Default: 1 +func (opts *Options) SetTargetFileSizeMultiplier(value int) { + C.rocksdb_options_set_target_file_size_multiplier(opts.c, C.int(value)) +} + +// SetMaxBytesForLevelBase sets the maximum total data size for a level. +// +// It is the max total for level-1. +// Maximum number of bytes for level L can be calculated as +// (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1)) +// +// For example, if max_bytes_for_level_base is 20MB, and if +// max_bytes_for_level_multiplier is 10, total data size for level-1 +// will be 20MB, total file size for level-2 will be 200MB, +// and total file size for level-3 will be 2GB. +// Default: 10MB +func (opts *Options) SetMaxBytesForLevelBase(value uint64) { + C.rocksdb_options_set_max_bytes_for_level_base(opts.c, C.uint64_t(value)) +} + +// SetMaxBytesForLevelMultiplier sets the max Bytes for level multiplier. +// Default: 10 +func (opts *Options) SetMaxBytesForLevelMultiplier(value float64) { + C.rocksdb_options_set_max_bytes_for_level_multiplier(opts.c, C.double(value)) +} + +// SetLevelCompactiondynamiclevelbytes specifies whether to pick +// target size of each level dynamically. +// +// We will pick a base level b >= 1. L0 will be directly merged into level b, +// instead of always into level 1. Level 1 to b-1 need to be empty. +// We try to pick b and its target size so that +// 1. target size is in the range of +// (max_bytes_for_level_base / max_bytes_for_level_multiplier, +// max_bytes_for_level_base] +// 2. target size of the last level (level num_levels-1) equals to extra size +// of the level. +// At the same time max_bytes_for_level_multiplier and +// max_bytes_for_level_multiplier_additional are still satisfied. +// +// With this option on, from an empty DB, we make last level the base level, +// which means merging L0 data into the last level, until it exceeds +// max_bytes_for_level_base. And then we make the second last level to be +// base level, to start to merge L0 data to second last level, with its +// target size to be 1/max_bytes_for_level_multiplier of the last level's +// extra size. After the data accumulates more so that we need to move the +// base level to the third last one, and so on. +// +// For example, assume max_bytes_for_level_multiplier=10, num_levels=6, +// and max_bytes_for_level_base=10MB. +// Target sizes of level 1 to 5 starts with: +// [- - - - 10MB] +// with base level is level. Target sizes of level 1 to 4 are not applicable +// because they will not be used. +// Until the size of Level 5 grows to more than 10MB, say 11MB, we make +// base target to level 4 and now the targets looks like: +// [- - - 1.1MB 11MB] +// While data are accumulated, size targets are tuned based on actual data +// of level 5. When level 5 has 50MB of data, the target is like: +// [- - - 5MB 50MB] +// Until level 5's actual size is more than 100MB, say 101MB. Now if we keep +// level 4 to be the base level, its target size needs to be 10.1MB, which +// doesn't satisfy the target size range. So now we make level 3 the target +// size and the target sizes of the levels look like: +// [- - 1.01MB 10.1MB 101MB] +// In the same way, while level 5 further grows, all levels' targets grow, +// like +// [- - 5MB 50MB 500MB] +// Until level 5 exceeds 1000MB and becomes 1001MB, we make level 2 the +// base level and make levels' target sizes like this: +// [- 1.001MB 10.01MB 100.1MB 1001MB] +// and go on... +// +// By doing it, we give max_bytes_for_level_multiplier a priority against +// max_bytes_for_level_base, for a more predictable LSM tree shape. It is +// useful to limit worse case space amplification. +// +// max_bytes_for_level_multiplier_additional is ignored with this flag on. +// +// Turning this feature on or off for an existing DB can cause unexpected +// LSM tree structure so it's not recommended. +// +// Default: false +func (opts *Options) SetLevelCompactionDynamicLevelBytes(value bool) { + C.rocksdb_options_set_level_compaction_dynamic_level_bytes(opts.c, boolToChar(value)) +} + +// SetMaxCompactionBytes sets the maximum number of bytes in all compacted files. +// We try to limit number of bytes in one compaction to be lower than this +// threshold. But it's not guaranteed. +// Value 0 will be sanitized. +// Default: result.target_file_size_base * 25 +func (opts *Options) SetMaxCompactionBytes(value uint64) { + C.rocksdb_options_set_max_compaction_bytes(opts.c, C.uint64_t(value)) +} + +// SetSoftPendingCompactionBytesLimit sets the threshold at which +// all writes will be slowed down to at least delayed_write_rate if estimated +// bytes needed to be compaction exceed this threshold. +// +// Default: 64GB +func (opts *Options) SetSoftPendingCompactionBytesLimit(value uint64) { + C.rocksdb_options_set_soft_pending_compaction_bytes_limit(opts.c, C.size_t(value)) +} + +// SetHardPendingCompactionBytesLimit sets the bytes threshold at which +// all writes are stopped if estimated bytes needed to be compaction exceed +// this threshold. +// +// Default: 256GB +func (opts *Options) SetHardPendingCompactionBytesLimit(value uint64) { + C.rocksdb_options_set_hard_pending_compaction_bytes_limit(opts.c, C.size_t(value)) +} + +// SetMaxBytesForLevelMultiplierAdditional sets different max-size multipliers +// for different levels. +// +// These are multiplied by max_bytes_for_level_multiplier to arrive +// at the max-size of each level. +// Default: 1 for each level +func (opts *Options) SetMaxBytesForLevelMultiplierAdditional(value []int) { + cLevels := make([]C.int, len(value)) + for i, v := range value { + cLevels[i] = C.int(v) + } + + C.rocksdb_options_set_max_bytes_for_level_multiplier_additional(opts.c, &cLevels[0], C.size_t(len(value))) +} + +// SetUseFsync enable/disable fsync. +// +// If true, then every store to stable storage will issue a fsync. +// If false, then every store to stable storage will issue a fdatasync. +// This parameter should be set to true while storing data to +// filesystem like ext3 that can lose files after a reboot. +// Default: false +func (opts *Options) SetUseFsync(value bool) { + C.rocksdb_options_set_use_fsync(opts.c, C.int(btoi(value))) +} + +// SetDbLogDir specifies the absolute info LOG dir. +// +// If it is empty, the log files will be in the same dir as data. +// If it is non empty, the log files will be in the specified dir, +// and the db data dir's absolute path will be used as the log file +// name's prefix. +// Default: empty +func (opts *Options) SetDbLogDir(value string) { + cvalue := C.CString(value) + defer C.free(unsafe.Pointer(cvalue)) + C.rocksdb_options_set_db_log_dir(opts.c, cvalue) +} + +// SetWalDir specifies the absolute dir path for write-ahead logs (WAL). +// +// If it is empty, the log files will be in the same dir as data. +// If it is non empty, the log files will be in the specified dir, +// When destroying the db, all log files and the dir itopts is deleted. +// Default: empty +func (opts *Options) SetWalDir(value string) { + cvalue := C.CString(value) + defer C.free(unsafe.Pointer(cvalue)) + C.rocksdb_options_set_wal_dir(opts.c, cvalue) +} + +// SetDeleteObsoleteFilesPeriodMicros sets the periodicity +// when obsolete files get deleted. +// +// The files that get out of scope by compaction +// process will still get automatically delete on every compaction, +// regardless of this setting. +// Default: 6 hours +func (opts *Options) SetDeleteObsoleteFilesPeriodMicros(value uint64) { + C.rocksdb_options_set_delete_obsolete_files_period_micros(opts.c, C.uint64_t(value)) +} + +// SetMaxBackgroundCompactions sets the maximum number of +// concurrent background jobs, submitted to +// the default LOW priority thread pool +// Default: 1 +func (opts *Options) SetMaxBackgroundCompactions(value int) { + C.rocksdb_options_set_max_background_compactions(opts.c, C.int(value)) +} + +// SetMaxBackgroundFlushes sets the maximum number of +// concurrent background memtable flush jobs, submitted to +// the HIGH priority thread pool. +// +// By default, all background jobs (major compaction and memtable flush) go +// to the LOW priority pool. If this option is set to a positive number, +// memtable flush jobs will be submitted to the HIGH priority pool. +// It is important when the same Env is shared by multiple db instances. +// Without a separate pool, long running major compaction jobs could +// potentially block memtable flush jobs of other db instances, leading to +// unnecessary Put stalls. +// Default: 0 +func (opts *Options) SetMaxBackgroundFlushes(value int) { + C.rocksdb_options_set_max_background_flushes(opts.c, C.int(value)) +} + +// SetMaxLogFileSize sets the maximal size of the info log file. +// +// If the log file is larger than `max_log_file_size`, a new info log +// file will be created. +// If max_log_file_size == 0, all logs will be written to one log file. +// Default: 0 +func (opts *Options) SetMaxLogFileSize(value int) { + C.rocksdb_options_set_max_log_file_size(opts.c, C.size_t(value)) +} + +// SetLogFileTimeToRoll sets the time for the info log file to roll (in seconds). +// +// If specified with non-zero value, log file will be rolled +// if it has been active longer than `log_file_time_to_roll`. +// Default: 0 (disabled) +func (opts *Options) SetLogFileTimeToRoll(value int) { + C.rocksdb_options_set_log_file_time_to_roll(opts.c, C.size_t(value)) +} + +// SetKeepLogFileNum sets the maximal info log files to be kept. +// Default: 1000 +func (opts *Options) SetKeepLogFileNum(value int) { + C.rocksdb_options_set_keep_log_file_num(opts.c, C.size_t(value)) +} + +// SetSoftRateLimit sets the soft rate limit. +// +// Puts are delayed 0-1 ms when any level has a compaction score that exceeds +// soft_rate_limit. This is ignored when == 0.0. +// CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not +// hold, RocksDB will set soft_rate_limit = hard_rate_limit +// Default: 0.0 (disabled) +func (opts *Options) SetSoftRateLimit(value float64) { + C.rocksdb_options_set_soft_rate_limit(opts.c, C.double(value)) +} + +// SetHardRateLimit sets the hard rate limit. +// +// Puts are delayed 1ms at a time when any level has a compaction score that +// exceeds hard_rate_limit. This is ignored when <= 1.0. +// Default: 0.0 (disabled) +func (opts *Options) SetHardRateLimit(value float64) { + C.rocksdb_options_set_hard_rate_limit(opts.c, C.double(value)) +} + +// SetRateLimitDelayMaxMilliseconds sets the max time +// a put will be stalled when hard_rate_limit is enforced. +// If 0, then there is no limit. +// Default: 1000 +func (opts *Options) SetRateLimitDelayMaxMilliseconds(value uint) { + C.rocksdb_options_set_rate_limit_delay_max_milliseconds(opts.c, C.uint(value)) +} + +// SetMaxManifestFileSize sets the maximal manifest file size until is rolled over. +// The older manifest file be deleted. +// Default: MAX_INT so that roll-over does not take place. +func (opts *Options) SetMaxManifestFileSize(value uint64) { + C.rocksdb_options_set_max_manifest_file_size(opts.c, C.size_t(value)) +} + +// SetTableCacheNumshardbits sets the number of shards used for table cache. +// Default: 4 +func (opts *Options) SetTableCacheNumshardbits(value int) { + C.rocksdb_options_set_table_cache_numshardbits(opts.c, C.int(value)) +} + +// SetTableCacheRemoveScanCountLimit sets the count limit during a scan. +// +// During data eviction of table's LRU cache, it would be inefficient +// to strictly follow LRU because this piece of memory will not really +// be released unless its refcount falls to zero. Instead, make two +// passes: the first pass will release items with refcount = 1, +// and if not enough space releases after scanning the number of +// elements specified by this parameter, we will remove items in LRU order. +// Default: 16 +func (opts *Options) SetTableCacheRemoveScanCountLimit(value int) { + C.rocksdb_options_set_table_cache_remove_scan_count_limit(opts.c, C.int(value)) +} + +// SetArenaBlockSize sets the size of one block in arena memory allocation. +// +// If <= 0, a proper value is automatically calculated (usually 1/10 of +// writer_buffer_size). +// Default: 0 +func (opts *Options) SetArenaBlockSize(value int) { + C.rocksdb_options_set_arena_block_size(opts.c, C.size_t(value)) +} + +// SetDisableAutoCompactions enable/disable automatic compactions. +// +// Manual compactions can still be issued on this database. +// Default: false +func (opts *Options) SetDisableAutoCompactions(value bool) { + C.rocksdb_options_set_disable_auto_compactions(opts.c, C.int(btoi(value))) +} + +// SetWALRecoveryMode sets the recovery mode +// +// Recovery mode to control the consistency while replaying WAL +// Default: TolerateCorruptedTailRecordsRecovery +func (opts *Options) SetWALRecoveryMode(mode WALRecoveryMode) { + C.rocksdb_options_set_wal_recovery_mode(opts.c, C.int(mode)) +} + +// SetWALTtlSeconds sets the WAL ttl in seconds. +// +// The following two options affect how archived logs will be deleted. +// 1. If both set to 0, logs will be deleted asap and will not get into +// the archive. +// 2. If wal_ttl_seconds is 0 and wal_size_limit_mb is not 0, +// WAL files will be checked every 10 min and if total size is greater +// then wal_size_limit_mb, they will be deleted starting with the +// earliest until size_limit is met. All empty files will be deleted. +// 3. If wal_ttl_seconds is not 0 and wall_size_limit_mb is 0, then +// WAL files will be checked every wal_ttl_seconds / 2 and those that +// are older than wal_ttl_seconds will be deleted. +// 4. If both are not 0, WAL files will be checked every 10 min and both +// checks will be performed with ttl being first. +// Default: 0 +func (opts *Options) SetWALTtlSeconds(value uint64) { + C.rocksdb_options_set_WAL_ttl_seconds(opts.c, C.uint64_t(value)) +} + +// SetWalSizeLimitMb sets the WAL size limit in MB. +// +// If total size of WAL files is greater then wal_size_limit_mb, +// they will be deleted starting with the earliest until size_limit is met +// Default: 0 +func (opts *Options) SetWalSizeLimitMb(value uint64) { + C.rocksdb_options_set_WAL_size_limit_MB(opts.c, C.uint64_t(value)) +} + +// SetEnablePipelinedWrite enables pipelined write +// +// Default: false +func (opts *Options) SetEnablePipelinedWrite(value bool) { + C.rocksdb_options_set_enable_pipelined_write(opts.c, boolToChar(value)) +} + +// SetManifestPreallocationSize sets the number of bytes +// to preallocate (via fallocate) the manifest files. +// +// Default is 4mb, which is reasonable to reduce random IO +// as well as prevent overallocation for mounts that preallocate +// large amounts of data (such as xfs's allocsize option). +// Default: 4mb +func (opts *Options) SetManifestPreallocationSize(value int) { + C.rocksdb_options_set_manifest_preallocation_size(opts.c, C.size_t(value)) +} + +// SetPurgeRedundantKvsWhileFlush enable/disable purging of +// duplicate/deleted keys when a memtable is flushed to storage. +// Default: true +func (opts *Options) SetPurgeRedundantKvsWhileFlush(value bool) { + C.rocksdb_options_set_purge_redundant_kvs_while_flush(opts.c, boolToChar(value)) +} + +// SetAllowMmapReads enable/disable mmap reads for reading sst tables. +// Default: false +func (opts *Options) SetAllowMmapReads(value bool) { + C.rocksdb_options_set_allow_mmap_reads(opts.c, boolToChar(value)) +} + +// SetAllowMmapWrites enable/disable mmap writes for writing sst tables. +// Default: false +func (opts *Options) SetAllowMmapWrites(value bool) { + C.rocksdb_options_set_allow_mmap_writes(opts.c, boolToChar(value)) +} + +// SetUseDirectReads enable/disable direct I/O mode (O_DIRECT) for reads +// Default: false +func (opts *Options) SetUseDirectReads(value bool) { + C.rocksdb_options_set_use_direct_reads(opts.c, boolToChar(value)) +} + +// SetUseDirectIOForFlushAndCompaction enable/disable direct I/O mode (O_DIRECT) for both reads and writes in background flush and compactions +// When true, new_table_reader_for_compaction_inputs is forced to true. +// Default: false +func (opts *Options) SetUseDirectIOForFlushAndCompaction(value bool) { + C.rocksdb_options_set_use_direct_io_for_flush_and_compaction(opts.c, boolToChar(value)) +} + +// SetIsFdCloseOnExec enable/dsiable child process inherit open files. +// Default: true +func (opts *Options) SetIsFdCloseOnExec(value bool) { + C.rocksdb_options_set_is_fd_close_on_exec(opts.c, boolToChar(value)) +} + +// SetSkipLogErrorOnRecovery enable/disable skipping of +// log corruption error on recovery (If client is ok with +// losing most recent changes) +// Default: false +func (opts *Options) SetSkipLogErrorOnRecovery(value bool) { + C.rocksdb_options_set_skip_log_error_on_recovery(opts.c, boolToChar(value)) +} + +// SetStatsDumpPeriodSec sets the stats dump period in seconds. +// +// If not zero, dump stats to LOG every stats_dump_period_sec +// Default: 3600 (1 hour) +func (opts *Options) SetStatsDumpPeriodSec(value uint) { + C.rocksdb_options_set_stats_dump_period_sec(opts.c, C.uint(value)) +} + +// SetAdviseRandomOnOpen specifies whether we will hint the underlying +// file system that the file access pattern is random, when a sst file is opened. +// Default: true +func (opts *Options) SetAdviseRandomOnOpen(value bool) { + C.rocksdb_options_set_advise_random_on_open(opts.c, boolToChar(value)) +} + +// SetDbWriteBufferSize sets the amount of data to build up +// in memtables across all column families before writing to disk. +// +// This is distinct from write_buffer_size, which enforces a limit +// for a single memtable. +// +// This feature is disabled by default. Specify a non-zero value +// to enable it. +// +// Default: 0 (disabled) +func (opts *Options) SetDbWriteBufferSize(value int) { + C.rocksdb_options_set_db_write_buffer_size(opts.c, C.size_t(value)) +} + +// SetAccessHintOnCompactionStart specifies the file access pattern +// once a compaction is started. +// +// It will be applied to all input files of a compaction. +// Default: NormalCompactionAccessPattern +func (opts *Options) SetAccessHintOnCompactionStart(value CompactionAccessPattern) { + C.rocksdb_options_set_access_hint_on_compaction_start(opts.c, C.int(value)) +} + +// SetUseAdaptiveMutex enable/disable adaptive mutex, which spins +// in the user space before resorting to kernel. +// +// This could reduce context switch when the mutex is not +// heavily contended. However, if the mutex is hot, we could end up +// wasting spin time. +// Default: false +func (opts *Options) SetUseAdaptiveMutex(value bool) { + C.rocksdb_options_set_use_adaptive_mutex(opts.c, boolToChar(value)) +} + +// SetBytesPerSync sets the bytes per sync. +// +// Allows OS to incrementally sync files to disk while they are being +// written, asynchronously, in the background. +// Issue one request for every bytes_per_sync written. +// Default: 0 (disabled) +func (opts *Options) SetBytesPerSync(value uint64) { + C.rocksdb_options_set_bytes_per_sync(opts.c, C.uint64_t(value)) +} + +// SetCompactionStyle sets the compaction style. +// Default: LevelCompactionStyle +func (opts *Options) SetCompactionStyle(value CompactionStyle) { + C.rocksdb_options_set_compaction_style(opts.c, C.int(value)) +} + +// SetUniversalCompactionOptions sets the options needed +// to support Universal Style compactions. +// Default: nil +func (opts *Options) SetUniversalCompactionOptions(value *UniversalCompactionOptions) { + C.rocksdb_options_set_universal_compaction_options(opts.c, value.c) +} + +// SetFIFOCompactionOptions sets the options for FIFO compaction style. +// Default: nil +func (opts *Options) SetFIFOCompactionOptions(value *FIFOCompactionOptions) { + C.rocksdb_options_set_fifo_compaction_options(opts.c, value.c) +} + +// GetStatisticsString returns the statistics as a string. +func (opts *Options) GetStatisticsString() string { + sString := C.rocksdb_options_statistics_get_string(opts.c) + defer C.rocksdb_free(unsafe.Pointer(sString)) + return C.GoString(sString) +} + +// SetRateLimiter sets the rate limiter of the options. +// Use to control write rate of flush and compaction. Flush has higher +// priority than compaction. Rate limiting is disabled if nullptr. +// If rate limiter is enabled, bytes_per_sync is set to 1MB by default. +// Default: nullptr +func (opts *Options) SetRateLimiter(rateLimiter *RateLimiter) { + C.rocksdb_options_set_ratelimiter(opts.c, rateLimiter.c) +} + +// SetMaxSequentialSkipInIterations specifies whether an iteration->Next() +// sequentially skips over keys with the same user-key or not. +// +// This number specifies the number of keys (with the same userkey) +// that will be sequentially skipped before a reseek is issued. +// Default: 8 +func (opts *Options) SetMaxSequentialSkipInIterations(value uint64) { + C.rocksdb_options_set_max_sequential_skip_in_iterations(opts.c, C.uint64_t(value)) +} + +// SetInplaceUpdateSupport enable/disable thread-safe inplace updates. +// +// Requires updates if +// * key exists in current memtable +// * new sizeof(new_value) <= sizeof(old_value) +// * old_value for that key is a put i.e. kTypeValue +// Default: false. +func (opts *Options) SetInplaceUpdateSupport(value bool) { + C.rocksdb_options_set_inplace_update_support(opts.c, boolToChar(value)) +} + +// SetInplaceUpdateNumLocks sets the number of locks used for inplace update. +// Default: 10000, if inplace_update_support = true, else 0. +func (opts *Options) SetInplaceUpdateNumLocks(value int) { + C.rocksdb_options_set_inplace_update_num_locks(opts.c, C.size_t(value)) +} + +// SetMemtableHugePageSize sets the page size for huge page for +// arena used by the memtable. +// If <=0, it won't allocate from huge page but from malloc. +// Users are responsible to reserve huge pages for it to be allocated. For +// example: +// sysctl -w vm.nr_hugepages=20 +// See linux doc Documentation/vm/hugetlbpage.txt +// If there isn't enough free huge page available, it will fall back to +// malloc. +// +// Dynamically changeable through SetOptions() API +func (opts *Options) SetMemtableHugePageSize(value int) { + C.rocksdb_options_set_memtable_huge_page_size(opts.c, C.size_t(value)) +} + +// SetBloomLocality sets the bloom locality. +// +// Control locality of bloom filter probes to improve cache miss rate. +// This option only applies to memtable prefix bloom and plaintable +// prefix bloom. It essentially limits the max number of cache lines each +// bloom filter check can touch. +// This optimization is turned off when set to 0. The number should never +// be greater than number of probes. This option can boost performance +// for in-memory workload but should use with care since it can cause +// higher false positive rate. +// Default: 0 +func (opts *Options) SetBloomLocality(value uint32) { + C.rocksdb_options_set_bloom_locality(opts.c, C.uint32_t(value)) +} + +// SetMaxSuccessiveMerges sets the maximum number of +// successive merge operations on a key in the memtable. +// +// When a merge operation is added to the memtable and the maximum number of +// successive merges is reached, the value of the key will be calculated and +// inserted into the memtable instead of the merge operation. This will +// ensure that there are never more than max_successive_merges merge +// operations in the memtable. +// Default: 0 (disabled) +func (opts *Options) SetMaxSuccessiveMerges(value int) { + C.rocksdb_options_set_max_successive_merges(opts.c, C.size_t(value)) +} + +// EnableStatistics enable statistics. +func (opts *Options) EnableStatistics() { + C.rocksdb_options_enable_statistics(opts.c) +} + +// PrepareForBulkLoad prepare the DB for bulk loading. +// +// All data will be in level 0 without any automatic compaction. +// It's recommended to manually call CompactRange(NULL, NULL) before reading +// from the database, because otherwise the read can be very slow. +func (opts *Options) PrepareForBulkLoad() { + C.rocksdb_options_prepare_for_bulk_load(opts.c) +} + +// SetMemtableVectorRep sets a MemTableRep which is backed by a vector. +// +// On iteration, the vector is sorted. This is useful for workloads where +// iteration is very rare and writes are generally not issued after reads begin. +func (opts *Options) SetMemtableVectorRep() { + C.rocksdb_options_set_memtable_vector_rep(opts.c) +} + +// SetHashSkipListRep sets a hash skip list as MemTableRep. +// +// It contains a fixed array of buckets, each +// pointing to a skiplist (null if the bucket is empty). +// +// bucketCount: number of fixed array buckets +// skiplistHeight: the max height of the skiplist +// skiplistBranchingFactor: probabilistic size ratio between adjacent +// link lists in the skiplist +func (opts *Options) SetHashSkipListRep(bucketCount int, skiplistHeight, skiplistBranchingFactor int32) { + C.rocksdb_options_set_hash_skip_list_rep(opts.c, C.size_t(bucketCount), C.int32_t(skiplistHeight), C.int32_t(skiplistBranchingFactor)) +} + +// SetHashLinkListRep sets a hashed linked list as MemTableRep. +// +// It contains a fixed array of buckets, each pointing to a sorted single +// linked list (null if the bucket is empty). +// +// bucketCount: number of fixed array buckets +func (opts *Options) SetHashLinkListRep(bucketCount int) { + C.rocksdb_options_set_hash_link_list_rep(opts.c, C.size_t(bucketCount)) +} + +// SetPlainTableFactory sets a plain table factory with prefix-only seek. +// +// For this factory, you need to set prefix_extractor properly to make it +// work. Look-up will starts with prefix hash lookup for key prefix. Inside the +// hash bucket found, a binary search is executed for hash conflicts. Finally, +// a linear search is used. +// +// keyLen: plain table has optimization for fix-sized keys, +// which can be specified via keyLen. +// bloomBitsPerKey: the number of bits used for bloom filer per prefix. You +// may disable it by passing a zero. +// hashTableRatio: the desired utilization of the hash table used for prefix +// hashing. hashTableRatio = number of prefixes / #buckets +// in the hash table +// indexSparseness: inside each prefix, need to build one index record for how +// many keys for binary search inside each hash bucket. +func (opts *Options) SetPlainTableFactory(keyLen uint32, bloomBitsPerKey int, hashTableRatio float64, indexSparseness int) { + C.rocksdb_options_set_plain_table_factory(opts.c, C.uint32_t(keyLen), C.int(bloomBitsPerKey), C.double(hashTableRatio), C.size_t(indexSparseness)) +} + +// SetCreateIfMissingColumnFamilies specifies whether the column families +// should be created if they are missing. +func (opts *Options) SetCreateIfMissingColumnFamilies(value bool) { + C.rocksdb_options_set_create_missing_column_families(opts.c, boolToChar(value)) +} + +// SetBlockBasedTableFactory sets the block based table factory. +func (opts *Options) SetBlockBasedTableFactory(value *BlockBasedTableOptions) { + opts.bbto = value + C.rocksdb_options_set_block_based_table_factory(opts.c, value.c) +} + +// SetAllowIngestBehind sets allow_ingest_behind +// Set this option to true during creation of database if you want +// to be able to ingest behind (call IngestExternalFile() skipping keys +// that already exist, rather than overwriting matching keys). +// Setting this option to true will affect 2 things: +// 1) Disable some internal optimizations around SST file compression +// 2) Reserve bottom-most level for ingested files only. +// 3) Note that num_levels should be >= 3 if this option is turned on. +// +// DEFAULT: false +// Immutable. +func (opts *Options) SetAllowIngestBehind(value bool) { + C.rocksdb_options_set_allow_ingest_behind(opts.c, boolToChar(value)) +} + +// SetMemTablePrefixBloomSizeRatio sets memtable_prefix_bloom_size_ratio +// if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0, +// create prefix bloom for memtable with the size of +// write_buffer_size * memtable_prefix_bloom_size_ratio. +// If it is larger than 0.25, it is sanitized to 0.25. +// +// Default: 0 (disable) +func (opts *Options) SetMemTablePrefixBloomSizeRatio(value float64) { + C.rocksdb_options_set_memtable_prefix_bloom_size_ratio(opts.c, C.double(value)) +} + +// SetOptimizeFiltersForHits sets optimize_filters_for_hits +// This flag specifies that the implementation should optimize the filters +// mainly for cases where keys are found rather than also optimize for keys +// missed. This would be used in cases where the application knows that +// there are very few misses or the performance in the case of misses is not +// important. +// +// For now, this flag allows us to not store filters for the last level i.e +// the largest level which contains data of the LSM store. For keys which +// are hits, the filters in this level are not useful because we will search +// for the data anyway. NOTE: the filters in other levels are still useful +// even for key hit because they tell us whether to look in that level or go +// to the higher level. +// +// Default: false +func (opts *Options) SetOptimizeFiltersForHits(value bool) { + C.rocksdb_options_set_optimize_filters_for_hits(opts.c, C.int(btoi(value))) +} + +// Destroy deallocates the Options object. +func (opts *Options) Destroy() { + C.rocksdb_options_destroy(opts.c) + if opts.ccmp != nil { + C.rocksdb_comparator_destroy(opts.ccmp) + } + // don't destroy the opts.cst here, it has already been + // associated with a PrefixExtractor and this will segfault + if opts.ccf != nil { + C.rocksdb_compactionfilter_destroy(opts.ccf) + } + opts.c = nil + opts.env = nil + opts.bbto = nil +} diff --git a/gorocksdb/options_block_based_table.go b/gorocksdb/options_block_based_table.go new file mode 100644 index 000000000..80244132f --- /dev/null +++ b/gorocksdb/options_block_based_table.go @@ -0,0 +1,237 @@ +package gorocksdb + +// #include "rocksdb/c.h" +// #include "gorocksdb.h" +import "C" + +// IndexType specifies the index type that will be used for this table. +type IndexType uint + +const ( + // A space efficient index block that is optimized for + // binary-search-based index. + KBinarySearchIndexType = 0 + // The hash index, if enabled, will do the hash lookup when + // `Options.prefix_extractor` is provided. + KHashSearchIndexType = 1 + // A two-level index implementation. Both levels are binary search indexes. + KTwoLevelIndexSearchIndexType = 2 +) + +// BlockBasedTableOptions represents block-based table options. +type BlockBasedTableOptions struct { + c *C.rocksdb_block_based_table_options_t + + // Hold references for GC. + cache *Cache + compCache *Cache + + // We keep these so we can free their memory in Destroy. + cFp *C.rocksdb_filterpolicy_t +} + +// NewDefaultBlockBasedTableOptions creates a default BlockBasedTableOptions object. +func NewDefaultBlockBasedTableOptions() *BlockBasedTableOptions { + return NewNativeBlockBasedTableOptions(C.rocksdb_block_based_options_create()) +} + +// NewNativeBlockBasedTableOptions creates a BlockBasedTableOptions object. +func NewNativeBlockBasedTableOptions(c *C.rocksdb_block_based_table_options_t) *BlockBasedTableOptions { + return &BlockBasedTableOptions{c: c} +} + +// Destroy deallocates the BlockBasedTableOptions object. +func (opts *BlockBasedTableOptions) Destroy() { + C.rocksdb_block_based_options_destroy(opts.c) + opts.c = nil + opts.cache = nil + opts.compCache = nil +} + +// SetCacheIndexAndFilterBlocks is indicating if we'd put index/filter blocks to the block cache. +// If not specified, each "table reader" object will pre-load index/filter +// block during table initialization. +// Default: false +func (opts *BlockBasedTableOptions) SetCacheIndexAndFilterBlocks(value bool) { + C.rocksdb_block_based_options_set_cache_index_and_filter_blocks(opts.c, boolToChar(value)) +} + +// SetCacheIndexAndFilterBlocksWithHighPriority sets cache index and filter +// blocks with high priority (if cache_index_and_filter_blocks is enabled). +// If set to true, depending on implementation of block cache, +// index and filter blocks may be less likely to be evicted than data blocks. +func (opts *BlockBasedTableOptions) SetCacheIndexAndFilterBlocksWithHighPriority(value bool) { + C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(opts.c, boolToChar(value)) +} + +// SetPinL0FilterAndIndexBlocksInCache sets cache_index_and_filter_blocks. +// If is true and the below is true (hash_index_allow_collision), then +// filter and index blocks are stored in the cache, but a reference is +// held in the "table reader" object so the blocks are pinned and only +// evicted from cache when the table reader is freed. +func (opts *BlockBasedTableOptions) SetPinL0FilterAndIndexBlocksInCache(value bool) { + C.rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(opts.c, boolToChar(value)) +} + +// SetPinTopLevelIndexAndFilter set that if cache_index_and_filter_blocks is true, then +// the top-level index of partitioned filter and index blocks are stored in +// the cache, but a reference is held in the "table reader" object so the +// blocks are pinned and only evicted from cache when the table reader is +// freed. This is not limited to l0 in LSM tree. +func (opts *BlockBasedTableOptions) SetPinTopLevelIndexAndFilter(value bool) { + C.rocksdb_block_based_options_set_pin_top_level_index_and_filter(opts.c, boolToChar(value)) +} + +// SetBlockSize sets the approximate size of user data packed per block. +// Note that the block size specified here corresponds opts uncompressed data. +// The actual size of the unit read from disk may be smaller if +// compression is enabled. This parameter can be changed dynamically. +// Default: 4K +func (opts *BlockBasedTableOptions) SetBlockSize(blockSize int) { + C.rocksdb_block_based_options_set_block_size(opts.c, C.size_t(blockSize)) +} + +// SetBlockSizeDeviation sets the block size deviation. +// This is used opts close a block before it reaches the configured +// 'block_size'. If the percentage of free space in the current block is less +// than this specified number and adding a new record opts the block will +// exceed the configured block size, then this block will be closed and the +// new record will be written opts the next block. +// Default: 10 +func (opts *BlockBasedTableOptions) SetBlockSizeDeviation(blockSizeDeviation int) { + C.rocksdb_block_based_options_set_block_size_deviation(opts.c, C.int(blockSizeDeviation)) +} + +// SetBlockRestartInterval sets the number of keys between +// restart points for delta encoding of keys. +// This parameter can be changed dynamically. Most clients should +// leave this parameter alone. +// Default: 16 +func (opts *BlockBasedTableOptions) SetBlockRestartInterval(blockRestartInterval int) { + C.rocksdb_block_based_options_set_block_restart_interval(opts.c, C.int(blockRestartInterval)) +} + +// SetIndexBlockRestartInterval is the same as SetBlockRestartInterval but used for the index block. +// Default: 1 +func (opts *BlockBasedTableOptions) SetIndexBlockRestartInterval(indexBlockRestartInterval int) { + C.rocksdb_block_based_options_set_index_block_restart_interval(opts.c, C.int(indexBlockRestartInterval)) +} + +// SetMetadataBlockSize sets the block size for partitioned metadata. +// Currently applied to indexes when +// kTwoLevelIndexSearch is used and to filters when partition_filters is used. +// Note: Since in the current implementation the filters and index partitions +// are aligned, an index/filter block is created when either index or filter +// block size reaches the specified limit. +// Note: this limit is currently applied to only index blocks; a filter +// partition is cut right after an index block is cut +// Default: 4096 +func (opts *BlockBasedTableOptions) SetMetadataBlockSize(metadataBlockSize uint64) { + C.rocksdb_block_based_options_set_metadata_block_size(opts.c, C.uint64_t(metadataBlockSize)) +} + +// SetPartitionFilters sets using partitioned full filters for each SST file. +// This option is incompatible with block-based filters. +// Note: currently this option requires kTwoLevelIndexSearch to be set as well. +// Default: false +func (opts *BlockBasedTableOptions) SetPartitionFilters(value bool) { + C.rocksdb_block_based_options_set_partition_filters(opts.c, boolToChar(value)) +} + +// SetUseDeltaEncoding sets using delta encoding to compress keys in blocks. +// ReadOptions::pin_data requires this option to be disabled. +func (opts *BlockBasedTableOptions) SetUseDeltaEncoding(value bool) { + C.rocksdb_block_based_options_set_use_delta_encoding(opts.c, boolToChar(value)) +} + +// SetFilterPolicy sets the filter policy opts reduce disk reads. +// Many applications will benefit from passing the result of +// NewBloomFilterPolicy() here. +// Default: nil +func (opts *BlockBasedTableOptions) SetFilterPolicy(fp FilterPolicy) { + if nfp, ok := fp.(nativeFilterPolicy); ok { + opts.cFp = nfp.c + } else { + idx := registerFilterPolicy(fp) + opts.cFp = C.gorocksdb_filterpolicy_create(C.uintptr_t(idx)) + } + C.rocksdb_block_based_options_set_filter_policy(opts.c, opts.cFp) +} + +// SetNoBlockCache specify whether block cache should be used or not. +// Default: false +func (opts *BlockBasedTableOptions) SetNoBlockCache(value bool) { + C.rocksdb_block_based_options_set_no_block_cache(opts.c, boolToChar(value)) +} + +// SetBlockCache sets the control over blocks (user data is stored in a set of blocks, and +// a block is the unit of reading from disk). +// +// If set, use the specified cache for blocks. +// If nil, rocksdb will auoptsmatically create and use an 8MB internal cache. +// Default: nil +func (opts *BlockBasedTableOptions) SetBlockCache(cache *Cache) { + opts.cache = cache + C.rocksdb_block_based_options_set_block_cache(opts.c, cache.c) +} + +// SetBlockCacheCompressed sets the cache for compressed blocks. +// If nil, rocksdb will not use a compressed block cache. +// Default: nil +func (opts *BlockBasedTableOptions) SetBlockCacheCompressed(cache *Cache) { + opts.compCache = cache + C.rocksdb_block_based_options_set_block_cache_compressed(opts.c, cache.c) +} + +// SetWholeKeyFiltering specify if whole keys in the filter (not just prefixes) +// should be placed. +// This must generally be true for gets opts be efficient. +// Default: true +func (opts *BlockBasedTableOptions) SetWholeKeyFiltering(value bool) { + C.rocksdb_block_based_options_set_whole_key_filtering(opts.c, boolToChar(value)) +} + +// SetFormatVersion sets the format version. +// We currently have five versions: +// 0 -- This version is currently written out by all RocksDB's versions by +// default. Can be read by really old RocksDB's. Doesn't support changing +// checksum (default is CRC32). +// 1 -- Can be read by RocksDB's versions since 3.0. Supports non-default +// checksum, like xxHash. It is written by RocksDB when +// BlockBasedTableOptions::checksum is something other than kCRC32c. (version +// 0 is silently upconverted) +// 2 -- Can be read by RocksDB's versions since 3.10. Changes the way we +// encode compressed blocks with LZ4, BZip2 and Zlib compression. If you +// don't plan to run RocksDB before version 3.10, you should probably use +// this. +// 3 -- Can be read by RocksDB's versions since 5.15. Changes the way we +// encode the keys in index blocks. If you don't plan to run RocksDB before +// version 5.15, you should probably use this. +// This option only affects newly written tables. When reading existing +// tables, the information about version is read from the footer. +// 4 -- Can be read by RocksDB's versions since 5.16. Changes the way we +// encode the values in index blocks. If you don't plan to run RocksDB before +// version 5.16 and you are using index_block_restart_interval > 1, you should +// probably use this as it would reduce the index size. +// This option only affects newly written tables. When reading existing +// tables, the information about version is read from the footer. +// Default: 2 +func (opts *BlockBasedTableOptions) SetFormatVersion(version int) { + C.rocksdb_block_based_options_set_format_version(opts.c, C.int(version)) +} + +// SetIndexType sets the index type used for this table. +// kBinarySearch: +// A space efficient index block that is optimized for +// binary-search-based index. +// +// kHashSearch: +// The hash index, if enabled, will do the hash lookup when +// `Options.prefix_extractor` is provided. +// +// kTwoLevelIndexSearch: +// A two-level index implementation. Both levels are binary search indexes. +// Default: kBinarySearch +func (opts *BlockBasedTableOptions) SetIndexType(value IndexType) { + C.rocksdb_block_based_options_set_index_type(opts.c, C.int(value)) +} diff --git a/gorocksdb/options_compaction.go b/gorocksdb/options_compaction.go new file mode 100644 index 000000000..a7db21367 --- /dev/null +++ b/gorocksdb/options_compaction.go @@ -0,0 +1,130 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// UniversalCompactionStopStyle describes a algorithm used to make a +// compaction request stop picking new files into a single compaction run. +type UniversalCompactionStopStyle uint + +// Compaction stop style types. +const ( + CompactionStopStyleSimilarSize = UniversalCompactionStopStyle(C.rocksdb_similar_size_compaction_stop_style) + CompactionStopStyleTotalSize = UniversalCompactionStopStyle(C.rocksdb_total_size_compaction_stop_style) +) + +// FIFOCompactionOptions represent all of the available options for +// FIFO compaction. +type FIFOCompactionOptions struct { + c *C.rocksdb_fifo_compaction_options_t +} + +// NewDefaultFIFOCompactionOptions creates a default FIFOCompactionOptions object. +func NewDefaultFIFOCompactionOptions() *FIFOCompactionOptions { + return NewNativeFIFOCompactionOptions(C.rocksdb_fifo_compaction_options_create()) +} + +// NewNativeFIFOCompactionOptions creates a native FIFOCompactionOptions object. +func NewNativeFIFOCompactionOptions(c *C.rocksdb_fifo_compaction_options_t) *FIFOCompactionOptions { + return &FIFOCompactionOptions{c} +} + +// SetMaxTableFilesSize sets the max table file size. +// Once the total sum of table files reaches this, we will delete the oldest +// table file +// Default: 1GB +func (opts *FIFOCompactionOptions) SetMaxTableFilesSize(value uint64) { + C.rocksdb_fifo_compaction_options_set_max_table_files_size(opts.c, C.uint64_t(value)) +} + +// Destroy deallocates the FIFOCompactionOptions object. +func (opts *FIFOCompactionOptions) Destroy() { + C.rocksdb_fifo_compaction_options_destroy(opts.c) +} + +// UniversalCompactionOptions represent all of the available options for +// universal compaction. +type UniversalCompactionOptions struct { + c *C.rocksdb_universal_compaction_options_t +} + +// NewDefaultUniversalCompactionOptions creates a default UniversalCompactionOptions +// object. +func NewDefaultUniversalCompactionOptions() *UniversalCompactionOptions { + return NewNativeUniversalCompactionOptions(C.rocksdb_universal_compaction_options_create()) +} + +// NewNativeUniversalCompactionOptions creates a UniversalCompactionOptions +// object. +func NewNativeUniversalCompactionOptions(c *C.rocksdb_universal_compaction_options_t) *UniversalCompactionOptions { + return &UniversalCompactionOptions{c} +} + +// SetSizeRatio sets the percentage flexibilty while comparing file size. +// If the candidate file(s) size is 1% smaller than the next file's size, +// then include next file into this candidate set. +// Default: 1 +func (opts *UniversalCompactionOptions) SetSizeRatio(value uint) { + C.rocksdb_universal_compaction_options_set_size_ratio(opts.c, C.int(value)) +} + +// SetMinMergeWidth sets the minimum number of files in a single compaction run. +// Default: 2 +func (opts *UniversalCompactionOptions) SetMinMergeWidth(value uint) { + C.rocksdb_universal_compaction_options_set_min_merge_width(opts.c, C.int(value)) +} + +// SetMaxMergeWidth sets the maximum number of files in a single compaction run. +// Default: UINT_MAX +func (opts *UniversalCompactionOptions) SetMaxMergeWidth(value uint) { + C.rocksdb_universal_compaction_options_set_max_merge_width(opts.c, C.int(value)) +} + +// SetMaxSizeAmplificationPercent sets the size amplification. +// It is defined as the amount (in percentage) of +// additional storage needed to store a single byte of data in the database. +// For example, a size amplification of 2% means that a database that +// contains 100 bytes of user-data may occupy upto 102 bytes of +// physical storage. By this definition, a fully compacted database has +// a size amplification of 0%. Rocksdb uses the following heuristic +// to calculate size amplification: it assumes that all files excluding +// the earliest file contribute to the size amplification. +// Default: 200, which means that a 100 byte database could require upto +// 300 bytes of storage. +func (opts *UniversalCompactionOptions) SetMaxSizeAmplificationPercent(value uint) { + C.rocksdb_universal_compaction_options_set_max_size_amplification_percent(opts.c, C.int(value)) +} + +// SetCompressionSizePercent sets the percentage of compression size. +// +// If this option is set to be -1, all the output files +// will follow compression type specified. +// +// If this option is not negative, we will try to make sure compressed +// size is just above this value. In normal cases, at least this percentage +// of data will be compressed. +// When we are compacting to a new file, here is the criteria whether +// it needs to be compressed: assuming here are the list of files sorted +// by generation time: +// A1...An B1...Bm C1...Ct +// where A1 is the newest and Ct is the oldest, and we are going to compact +// B1...Bm, we calculate the total size of all the files as total_size, as +// well as the total size of C1...Ct as total_C, the compaction output file +// will be compressed iff +// total_C / total_size < this percentage +// Default: -1 +func (opts *UniversalCompactionOptions) SetCompressionSizePercent(value int) { + C.rocksdb_universal_compaction_options_set_compression_size_percent(opts.c, C.int(value)) +} + +// SetStopStyle sets the algorithm used to stop picking files into a single compaction run. +// Default: CompactionStopStyleTotalSize +func (opts *UniversalCompactionOptions) SetStopStyle(value UniversalCompactionStopStyle) { + C.rocksdb_universal_compaction_options_set_stop_style(opts.c, C.int(value)) +} + +// Destroy deallocates the UniversalCompactionOptions object. +func (opts *UniversalCompactionOptions) Destroy() { + C.rocksdb_universal_compaction_options_destroy(opts.c) + opts.c = nil +} diff --git a/gorocksdb/options_compression.go b/gorocksdb/options_compression.go new file mode 100644 index 000000000..7165cedaf --- /dev/null +++ b/gorocksdb/options_compression.go @@ -0,0 +1,24 @@ +package gorocksdb + +// CompressionOptions represents options for different compression algorithms like Zlib. +type CompressionOptions struct { + WindowBits int + Level int + Strategy int + MaxDictBytes int +} + +// NewDefaultCompressionOptions creates a default CompressionOptions object. +func NewDefaultCompressionOptions() *CompressionOptions { + return NewCompressionOptions(-14, -1, 0, 0) +} + +// NewCompressionOptions creates a CompressionOptions object. +func NewCompressionOptions(windowBits, level, strategy, maxDictBytes int) *CompressionOptions { + return &CompressionOptions{ + WindowBits: windowBits, + Level: level, + Strategy: strategy, + MaxDictBytes: maxDictBytes, + } +} diff --git a/gorocksdb/options_env.go b/gorocksdb/options_env.go new file mode 100644 index 000000000..cdac98f24 --- /dev/null +++ b/gorocksdb/options_env.go @@ -0,0 +1,25 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// EnvOptions represents options for env. +type EnvOptions struct { + c *C.rocksdb_envoptions_t +} + +// NewDefaultEnvOptions creates a default EnvOptions object. +func NewDefaultEnvOptions() *EnvOptions { + return NewNativeEnvOptions(C.rocksdb_envoptions_create()) +} + +// NewNativeEnvOptions creates a EnvOptions object. +func NewNativeEnvOptions(c *C.rocksdb_envoptions_t) *EnvOptions { + return &EnvOptions{c: c} +} + +// Destroy deallocates the EnvOptions object. +func (opts *EnvOptions) Destroy() { + C.rocksdb_envoptions_destroy(opts.c) + opts.c = nil +} diff --git a/gorocksdb/options_flush.go b/gorocksdb/options_flush.go new file mode 100644 index 000000000..518236a5d --- /dev/null +++ b/gorocksdb/options_flush.go @@ -0,0 +1,32 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// FlushOptions represent all of the available options when manual flushing the +// database. +type FlushOptions struct { + c *C.rocksdb_flushoptions_t +} + +// NewDefaultFlushOptions creates a default FlushOptions object. +func NewDefaultFlushOptions() *FlushOptions { + return NewNativeFlushOptions(C.rocksdb_flushoptions_create()) +} + +// NewNativeFlushOptions creates a FlushOptions object. +func NewNativeFlushOptions(c *C.rocksdb_flushoptions_t) *FlushOptions { + return &FlushOptions{c} +} + +// SetWait specify if the flush will wait until the flush is done. +// Default: true +func (opts *FlushOptions) SetWait(value bool) { + C.rocksdb_flushoptions_set_wait(opts.c, boolToChar(value)) +} + +// Destroy deallocates the FlushOptions object. +func (opts *FlushOptions) Destroy() { + C.rocksdb_flushoptions_destroy(opts.c) + opts.c = nil +} diff --git a/gorocksdb/options_ingest.go b/gorocksdb/options_ingest.go new file mode 100644 index 000000000..89efb547e --- /dev/null +++ b/gorocksdb/options_ingest.go @@ -0,0 +1,65 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// IngestExternalFileOptions represents available options when ingesting external files. +type IngestExternalFileOptions struct { + c *C.rocksdb_ingestexternalfileoptions_t +} + +// NewDefaultIngestExternalFileOptions creates a default IngestExternalFileOptions object. +func NewDefaultIngestExternalFileOptions() *IngestExternalFileOptions { + return NewNativeIngestExternalFileOptions(C.rocksdb_ingestexternalfileoptions_create()) +} + +// NewNativeIngestExternalFileOptions creates a IngestExternalFileOptions object. +func NewNativeIngestExternalFileOptions(c *C.rocksdb_ingestexternalfileoptions_t) *IngestExternalFileOptions { + return &IngestExternalFileOptions{c: c} +} + +// SetMoveFiles specifies if it should move the files instead of copying them. +// Default to false. +func (opts *IngestExternalFileOptions) SetMoveFiles(flag bool) { + C.rocksdb_ingestexternalfileoptions_set_move_files(opts.c, boolToChar(flag)) +} + +// SetSnapshotConsistency if specifies the consistency. +// If set to false, an ingested file key could appear in existing snapshots that were created before the +// file was ingested. +// Default to true. +func (opts *IngestExternalFileOptions) SetSnapshotConsistency(flag bool) { + C.rocksdb_ingestexternalfileoptions_set_snapshot_consistency(opts.c, boolToChar(flag)) +} + +// SetAllowGlobalSeqNo sets allow_global_seqno. If set to false,IngestExternalFile() will fail if the file key +// range overlaps with existing keys or tombstones in the DB. +// Default true. +func (opts *IngestExternalFileOptions) SetAllowGlobalSeqNo(flag bool) { + C.rocksdb_ingestexternalfileoptions_set_allow_global_seqno(opts.c, boolToChar(flag)) +} + +// SetAllowBlockingFlush sets allow_blocking_flush. If set to false and the file key range overlaps with +// the memtable key range (memtable flush required), IngestExternalFile will fail. +// Default to true. +func (opts *IngestExternalFileOptions) SetAllowBlockingFlush(flag bool) { + C.rocksdb_ingestexternalfileoptions_set_allow_blocking_flush(opts.c, boolToChar(flag)) +} + +// SetIngestionBehind sets ingest_behind +// Set to true if you would like duplicate keys in the file being ingested +// to be skipped rather than overwriting existing data under that key. +// Usecase: back-fill of some historical data in the database without +// over-writing existing newer version of data. +// This option could only be used if the DB has been running +// with allow_ingest_behind=true since the dawn of time. +// All files will be ingested at the bottommost level with seqno=0. +func (opts *IngestExternalFileOptions) SetIngestionBehind(flag bool) { + C.rocksdb_ingestexternalfileoptions_set_ingest_behind(opts.c, boolToChar(flag)) +} + +// Destroy deallocates the IngestExternalFileOptions object. +func (opts *IngestExternalFileOptions) Destroy() { + C.rocksdb_ingestexternalfileoptions_destroy(opts.c) + opts.c = nil +} diff --git a/gorocksdb/options_read.go b/gorocksdb/options_read.go new file mode 100644 index 000000000..6a37cc486 --- /dev/null +++ b/gorocksdb/options_read.go @@ -0,0 +1,136 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" +import "unsafe" + +// ReadTier controls fetching of data during a read request. +// An application can issue a read request (via Get/Iterators) and specify +// if that read should process data that ALREADY resides on a specified cache +// level. For example, if an application specifies BlockCacheTier then the +// Get call will process data that is already processed in the memtable or +// the block cache. It will not page in data from the OS cache or data that +// resides in storage. +type ReadTier uint + +const ( + // ReadAllTier reads data in memtable, block cache, OS cache or storage. + ReadAllTier = ReadTier(0) + // BlockCacheTier reads data in memtable or block cache. + BlockCacheTier = ReadTier(1) +) + +// ReadOptions represent all of the available options when reading from a +// database. +type ReadOptions struct { + c *C.rocksdb_readoptions_t +} + +// NewDefaultReadOptions creates a default ReadOptions object. +func NewDefaultReadOptions() *ReadOptions { + return NewNativeReadOptions(C.rocksdb_readoptions_create()) +} + +// NewNativeReadOptions creates a ReadOptions object. +func NewNativeReadOptions(c *C.rocksdb_readoptions_t) *ReadOptions { + return &ReadOptions{c} +} + +// UnsafeGetReadOptions returns the underlying c read options object. +func (opts *ReadOptions) UnsafeGetReadOptions() unsafe.Pointer { + return unsafe.Pointer(opts.c) +} + +// SetVerifyChecksums speciy if all data read from underlying storage will be +// verified against corresponding checksums. +// Default: false +func (opts *ReadOptions) SetVerifyChecksums(value bool) { + C.rocksdb_readoptions_set_verify_checksums(opts.c, boolToChar(value)) +} + +// SetPrefixSameAsStart Enforce that the iterator only iterates over the same +// prefix as the seek. +// This option is effective only for prefix seeks, i.e. prefix_extractor is +// non-null for the column family and total_order_seek is false. Unlike +// iterate_upper_bound, prefix_same_as_start only works within a prefix +// but in both directions. +// Default: false +func (opts *ReadOptions) SetPrefixSameAsStart(value bool) { + C.rocksdb_readoptions_set_prefix_same_as_start(opts.c, boolToChar(value)) +} + +// SetFillCache specify whether the "data block"/"index block"/"filter block" +// read for this iteration should be cached in memory? +// Callers may wish to set this field to false for bulk scans. +// Default: true +func (opts *ReadOptions) SetFillCache(value bool) { + C.rocksdb_readoptions_set_fill_cache(opts.c, boolToChar(value)) +} + +// SetSnapshot sets the snapshot which should be used for the read. +// The snapshot must belong to the DB that is being read and must +// not have been released. +// Default: nil +func (opts *ReadOptions) SetSnapshot(snap *Snapshot) { + C.rocksdb_readoptions_set_snapshot(opts.c, snap.c) +} + +// SetReadTier specify if this read request should process data that ALREADY +// resides on a particular cache. If the required data is not +// found at the specified cache, then Status::Incomplete is returned. +// Default: ReadAllTier +func (opts *ReadOptions) SetReadTier(value ReadTier) { + C.rocksdb_readoptions_set_read_tier(opts.c, C.int(value)) +} + +// SetTailing specify if to create a tailing iterator. +// A special iterator that has a view of the complete database +// (i.e. it can also be used to read newly added data) and +// is optimized for sequential reads. It will return records +// that were inserted into the database after the creation of the iterator. +// Default: false +func (opts *ReadOptions) SetTailing(value bool) { + C.rocksdb_readoptions_set_tailing(opts.c, boolToChar(value)) +} + +// SetIterateUpperBound specifies "iterate_upper_bound", which defines +// the extent upto which the forward iterator can returns entries. +// Once the bound is reached, Valid() will be false. +// "iterate_upper_bound" is exclusive ie the bound value is +// not a valid entry. If iterator_extractor is not null, the Seek target +// and iterator_upper_bound need to have the same prefix. +// This is because ordering is not guaranteed outside of prefix domain. +// There is no lower bound on the iterator. If needed, that can be easily +// implemented. +// Default: nullptr +func (opts *ReadOptions) SetIterateUpperBound(key []byte) { + cKey := byteToChar(key) + cKeyLen := C.size_t(len(key)) + C.rocksdb_readoptions_set_iterate_upper_bound(opts.c, cKey, cKeyLen) +} + +// SetPinData specifies the value of "pin_data". If true, it keeps the blocks +// loaded by the iterator pinned in memory as long as the iterator is not deleted, +// If used when reading from tables created with +// BlockBasedTableOptions::use_delta_encoding = false, +// Iterator's property "rocksdb.iterator.is-key-pinned" is guaranteed to +// return 1. +// Default: false +func (opts *ReadOptions) SetPinData(value bool) { + C.rocksdb_readoptions_set_pin_data(opts.c, boolToChar(value)) +} + +// SetReadaheadSize specifies the value of "readahead_size". +// If non-zero, NewIterator will create a new table reader which +// performs reads of the given size. Using a large size (> 2MB) can +// improve the performance of forward iteration on spinning disks. +// Default: 0 +func (opts *ReadOptions) SetReadaheadSize(value uint64) { + C.rocksdb_readoptions_set_readahead_size(opts.c, C.size_t(value)) +} + +// Destroy deallocates the ReadOptions object. +func (opts *ReadOptions) Destroy() { + C.rocksdb_readoptions_destroy(opts.c) + opts.c = nil +} diff --git a/gorocksdb/options_transaction.go b/gorocksdb/options_transaction.go new file mode 100644 index 000000000..cb72bff8f --- /dev/null +++ b/gorocksdb/options_transaction.go @@ -0,0 +1,66 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// TransactionOptions represent all of the available options options for +// a transaction on the database. +type TransactionOptions struct { + c *C.rocksdb_transaction_options_t +} + +// NewDefaultTransactionOptions creates a default TransactionOptions object. +func NewDefaultTransactionOptions() *TransactionOptions { + return NewNativeTransactionOptions(C.rocksdb_transaction_options_create()) +} + +// NewNativeTransactionOptions creates a TransactionOptions object. +func NewNativeTransactionOptions(c *C.rocksdb_transaction_options_t) *TransactionOptions { + return &TransactionOptions{c} +} + +// SetSetSnapshot to true is the same as calling +// Transaction::SetSnapshot(). +func (opts *TransactionOptions) SetSetSnapshot(value bool) { + C.rocksdb_transaction_options_set_set_snapshot(opts.c, boolToChar(value)) +} + +// SetDeadlockDetect to true means that before acquiring locks, this transaction will +// check if doing so will cause a deadlock. If so, it will return with +// Status::Busy. The user should retry their transaction. +func (opts *TransactionOptions) SetDeadlockDetect(value bool) { + C.rocksdb_transaction_options_set_deadlock_detect(opts.c, boolToChar(value)) +} + +// SetLockTimeout positive, specifies the wait timeout in milliseconds when +// a transaction attempts to lock a key. +// If 0, no waiting is done if a lock cannot instantly be acquired. +// If negative, TransactionDBOptions::transaction_lock_timeout will be used +func (opts *TransactionOptions) SetLockTimeout(lock_timeout int64) { + C.rocksdb_transaction_options_set_lock_timeout(opts.c, C.int64_t(lock_timeout)) +} + +// SetExpiration sets the Expiration duration in milliseconds. +// If non-negative, transactions that last longer than this many milliseconds will fail to commit. +// If not set, a forgotten transaction that is never committed, rolled back, or deleted +// will never relinquish any locks it holds. This could prevent keys from +// being written by other writers. +func (opts *TransactionOptions) SetExpiration(expiration int64) { + C.rocksdb_transaction_options_set_expiration(opts.c, C.int64_t(expiration)) +} + +// SetDeadlockDetectDepth sets the number of traversals to make during deadlock detection. +func (opts *TransactionOptions) SetDeadlockDetectDepth(depth int64) { + C.rocksdb_transaction_options_set_deadlock_detect_depth(opts.c, C.int64_t(depth)) +} + +// SetMaxWriteBatchSize sets the maximum number of bytes used for the write batch. 0 means no limit. +func (opts *TransactionOptions) SetMaxWriteBatchSize(size uint64) { + C.rocksdb_transaction_options_set_max_write_batch_size(opts.c, C.size_t(size)) +} + +// Destroy deallocates the TransactionOptions object. +func (opts *TransactionOptions) Destroy() { + C.rocksdb_transaction_options_destroy(opts.c) + opts.c = nil +} diff --git a/gorocksdb/options_transactiondb.go b/gorocksdb/options_transactiondb.go new file mode 100644 index 000000000..6c0b1cd96 --- /dev/null +++ b/gorocksdb/options_transactiondb.go @@ -0,0 +1,72 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// TransactionDBOptions represent all of the available options when opening a transactional database +// with OpenTransactionDb. +type TransactionDBOptions struct { + c *C.rocksdb_transactiondb_options_t +} + +// NewDefaultTransactionDBOptions creates a default TransactionDBOptions object. +func NewDefaultTransactionDBOptions() *TransactionDBOptions { + return NewNativeTransactionDBOptions(C.rocksdb_transactiondb_options_create()) +} + +// NewDefaultTransactionDBOptions creates a TransactionDBOptions object. +func NewNativeTransactionDBOptions(c *C.rocksdb_transactiondb_options_t) *TransactionDBOptions { + return &TransactionDBOptions{c} +} + +// SetMaxNumLocks sets the maximum number of keys that can be locked at the same time +// per column family. +// If the number of locked keys is greater than max_num_locks, transaction +// writes (or GetForUpdate) will return an error. +// If this value is not positive, no limit will be enforced. +func (opts *TransactionDBOptions) SetMaxNumLocks(max_num_locks int64) { + C.rocksdb_transactiondb_options_set_max_num_locks(opts.c, C.int64_t(max_num_locks)) +} + +// SetNumStripes sets the concurrency level. +// Increasing this value will increase the concurrency by dividing the lock +// table (per column family) into more sub-tables, each with their own +// separate +// mutex. +func (opts *TransactionDBOptions) SetNumStripes(num_stripes uint64) { + C.rocksdb_transactiondb_options_set_num_stripes(opts.c, C.size_t(num_stripes)) +} + +// SetTransactionLockTimeout if positive, specifies the default wait timeout in milliseconds when +// a transaction attempts to lock a key if not specified by +// TransactionOptions::lock_timeout. +// +// If 0, no waiting is done if a lock cannot instantly be acquired. +// If negative, there is no timeout. Not using a timeout is not recommended +// as it can lead to deadlocks. Currently, there is no deadlock-detection to +// recover from a deadlock. +func (opts *TransactionDBOptions) SetTransactionLockTimeout(txn_lock_timeout int64) { + C.rocksdb_transactiondb_options_set_transaction_lock_timeout(opts.c, C.int64_t(txn_lock_timeout)) +} + +// SetDefaultLockTimeout if posititve, specifies the wait timeout in milliseconds when writing a key +// OUTSIDE of a transaction (ie by calling DB::Put(),Merge(),Delete(),Write() +// directly). +// If 0, no waiting is done if a lock cannot instantly be acquired. +// If negative, there is no timeout and will block indefinitely when acquiring +// a lock. +// +// Not using a timeout can lead to deadlocks. Currently, there +// is no deadlock-detection to recover from a deadlock. While DB writes +// cannot deadlock with other DB writes, they can deadlock with a transaction. +// A negative timeout should only be used if all transactions have a small +// expiration set. +func (opts *TransactionDBOptions) SetDefaultLockTimeout(default_lock_timeout int64) { + C.rocksdb_transactiondb_options_set_default_lock_timeout(opts.c, C.int64_t(default_lock_timeout)) +} + +// Destroy deallocates the TransactionDBOptions object. +func (opts *TransactionDBOptions) Destroy() { + C.rocksdb_transactiondb_options_destroy(opts.c) + opts.c = nil +} diff --git a/gorocksdb/options_write.go b/gorocksdb/options_write.go new file mode 100644 index 000000000..01cd9c9ab --- /dev/null +++ b/gorocksdb/options_write.go @@ -0,0 +1,42 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// WriteOptions represent all of the available options when writing to a +// database. +type WriteOptions struct { + c *C.rocksdb_writeoptions_t +} + +// NewDefaultWriteOptions creates a default WriteOptions object. +func NewDefaultWriteOptions() *WriteOptions { + return NewNativeWriteOptions(C.rocksdb_writeoptions_create()) +} + +// NewNativeWriteOptions creates a WriteOptions object. +func NewNativeWriteOptions(c *C.rocksdb_writeoptions_t) *WriteOptions { + return &WriteOptions{c} +} + +// SetSync sets the sync mode. If true, the write will be flushed +// from the operating system buffer cache before the write is considered complete. +// If this flag is true, writes will be slower. +// Default: false +func (opts *WriteOptions) SetSync(value bool) { + C.rocksdb_writeoptions_set_sync(opts.c, boolToChar(value)) +} + +// DisableWAL sets whether WAL should be active or not. +// If true, writes will not first go to the write ahead log, +// and the write may got lost after a crash. +// Default: false +func (opts *WriteOptions) DisableWAL(value bool) { + C.rocksdb_writeoptions_disable_WAL(opts.c, C.int(btoi(value))) +} + +// Destroy deallocates the WriteOptions object. +func (opts *WriteOptions) Destroy() { + C.rocksdb_writeoptions_destroy(opts.c) + opts.c = nil +} diff --git a/gorocksdb/ratelimiter.go b/gorocksdb/ratelimiter.go new file mode 100644 index 000000000..72b539515 --- /dev/null +++ b/gorocksdb/ratelimiter.go @@ -0,0 +1,31 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" + +// RateLimiter, is used to control write rate of flush and +// compaction. +type RateLimiter struct { + c *C.rocksdb_ratelimiter_t +} + +// NewDefaultRateLimiter creates a default RateLimiter object. +func NewRateLimiter(rate_bytes_per_sec, refill_period_us int64, fairness int32) *RateLimiter { + return NewNativeRateLimiter(C.rocksdb_ratelimiter_create( + C.int64_t(rate_bytes_per_sec), + C.int64_t(refill_period_us), + C.int32_t(fairness), + )) +} + +// NewNativeRateLimiter creates a native RateLimiter object. +func NewNativeRateLimiter(c *C.rocksdb_ratelimiter_t) *RateLimiter { + return &RateLimiter{c} +} + +// Destroy deallocates the RateLimiter object. +func (self *RateLimiter) Destroy() { + C.rocksdb_ratelimiter_destroy(self.c) + self.c = nil +} diff --git a/gorocksdb/slice.go b/gorocksdb/slice.go new file mode 100644 index 000000000..707a1f2e9 --- /dev/null +++ b/gorocksdb/slice.go @@ -0,0 +1,84 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import "unsafe" + +// Slice is used as a wrapper for non-copy values +type Slice struct { + data *C.char + size C.size_t + freed bool +} + +type Slices []*Slice + +func (slices Slices) Destroy() { + for _, s := range slices { + s.Free() + } +} + +// NewSlice returns a slice with the given data. +func NewSlice(data *C.char, size C.size_t) *Slice { + return &Slice{data, size, false} +} + +// StringToSlice is similar to NewSlice, but can be called with +// a Go string type. This exists to make testing integration +// with Gorocksdb easier. +func StringToSlice(data string) *Slice { + return NewSlice(C.CString(data), C.size_t(len(data))) +} + +// Data returns the data of the slice. If the key doesn't exist this will be a +// nil slice. +func (s *Slice) Data() []byte { + return charToByte(s.data, s.size) +} + +// Size returns the size of the data. +func (s *Slice) Size() int { + return int(s.size) +} + +// Exists returns if the key exists +func (s *Slice) Exists() bool { + return s.data != nil +} + +// Free frees the slice data. +func (s *Slice) Free() { + if !s.freed { + C.rocksdb_free(unsafe.Pointer(s.data)) + s.freed = true + } +} + +// PinnableSliceHandle represents a handle to a PinnableSlice. +type PinnableSliceHandle struct { + c *C.rocksdb_pinnableslice_t +} + +// NewNativePinnableSliceHandle creates a PinnableSliceHandle object. +func NewNativePinnableSliceHandle(c *C.rocksdb_pinnableslice_t) *PinnableSliceHandle { + return &PinnableSliceHandle{c} +} + +// Data returns the data of the slice. +func (h *PinnableSliceHandle) Data() []byte { + if h.c == nil { + return nil + } + + var cValLen C.size_t + cValue := C.rocksdb_pinnableslice_value(h.c, &cValLen) + + return charToByte(cValue, cValLen) +} + +// Destroy calls the destructor of the underlying pinnable slice handle. +func (h *PinnableSliceHandle) Destroy() { + C.rocksdb_pinnableslice_destroy(h.c) +} diff --git a/gorocksdb/slice_transform.go b/gorocksdb/slice_transform.go new file mode 100644 index 000000000..8b9b2362b --- /dev/null +++ b/gorocksdb/slice_transform.go @@ -0,0 +1,82 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// A SliceTransform can be used as a prefix extractor. +type SliceTransform interface { + // Transform a src in domain to a dst in the range. + Transform(src []byte) []byte + + // Determine whether this is a valid src upon the function applies. + InDomain(src []byte) bool + + // Determine whether dst=Transform(src) for some src. + InRange(src []byte) bool + + // Return the name of this transformation. + Name() string +} + +// NewFixedPrefixTransform creates a new fixed prefix transform. +func NewFixedPrefixTransform(prefixLen int) SliceTransform { + return NewNativeSliceTransform(C.rocksdb_slicetransform_create_fixed_prefix(C.size_t(prefixLen))) +} + +// NewNoopPrefixTransform creates a new no-op prefix transform. +func NewNoopPrefixTransform() SliceTransform { + return NewNativeSliceTransform(C.rocksdb_slicetransform_create_noop()) +} + +// NewNativeSliceTransform creates a SliceTransform object. +func NewNativeSliceTransform(c *C.rocksdb_slicetransform_t) SliceTransform { + return nativeSliceTransform{c} +} + +type nativeSliceTransform struct { + c *C.rocksdb_slicetransform_t +} + +func (st nativeSliceTransform) Transform(src []byte) []byte { return nil } +func (st nativeSliceTransform) InDomain(src []byte) bool { return false } +func (st nativeSliceTransform) InRange(src []byte) bool { return false } +func (st nativeSliceTransform) Name() string { return "" } + +// Hold references to slice transforms. +var sliceTransforms = NewCOWList() + +type sliceTransformWrapper struct { + name *C.char + sliceTransform SliceTransform +} + +func registerSliceTransform(st SliceTransform) int { + return sliceTransforms.Append(sliceTransformWrapper{C.CString(st.Name()), st}) +} + +//export gorocksdb_slicetransform_transform +func gorocksdb_slicetransform_transform(idx int, cKey *C.char, cKeyLen C.size_t, cDstLen *C.size_t) *C.char { + key := charToByte(cKey, cKeyLen) + dst := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.Transform(key) + *cDstLen = C.size_t(len(dst)) + return cByteSlice(dst) +} + +//export gorocksdb_slicetransform_in_domain +func gorocksdb_slicetransform_in_domain(idx int, cKey *C.char, cKeyLen C.size_t) C.uchar { + key := charToByte(cKey, cKeyLen) + inDomain := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.InDomain(key) + return boolToChar(inDomain) +} + +//export gorocksdb_slicetransform_in_range +func gorocksdb_slicetransform_in_range(idx int, cKey *C.char, cKeyLen C.size_t) C.uchar { + key := charToByte(cKey, cKeyLen) + inRange := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.InRange(key) + return boolToChar(inRange) +} + +//export gorocksdb_slicetransform_name +func gorocksdb_slicetransform_name(idx int) *C.char { + return sliceTransforms.Get(idx).(sliceTransformWrapper).name +} diff --git a/gorocksdb/slice_transform_test.go b/gorocksdb/slice_transform_test.go new file mode 100644 index 000000000..d60c7326b --- /dev/null +++ b/gorocksdb/slice_transform_test.go @@ -0,0 +1,52 @@ +package gorocksdb + +import ( + "testing" + + "github.com/facebookgo/ensure" +) + +func TestSliceTransform(t *testing.T) { + db := newTestDB(t, "TestSliceTransform", func(opts *Options) { + opts.SetPrefixExtractor(&testSliceTransform{}) + }) + defer db.Close() + + wo := NewDefaultWriteOptions() + ensure.Nil(t, db.Put(wo, []byte("foo1"), []byte("foo"))) + ensure.Nil(t, db.Put(wo, []byte("foo2"), []byte("foo"))) + ensure.Nil(t, db.Put(wo, []byte("bar1"), []byte("bar"))) + + iter := db.NewIterator(NewDefaultReadOptions()) + defer iter.Close() + prefix := []byte("foo") + numFound := 0 + for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() { + numFound++ + } + ensure.Nil(t, iter.Err()) + ensure.DeepEqual(t, numFound, 2) +} + +func TestFixedPrefixTransformOpen(t *testing.T) { + db := newTestDB(t, "TestFixedPrefixTransformOpen", func(opts *Options) { + opts.SetPrefixExtractor(NewFixedPrefixTransform(3)) + }) + defer db.Close() +} + +func TestNewNoopPrefixTransform(t *testing.T) { + db := newTestDB(t, "TestNewNoopPrefixTransform", func(opts *Options) { + opts.SetPrefixExtractor(NewNoopPrefixTransform()) + }) + defer db.Close() +} + +type testSliceTransform struct { + initiated bool +} + +func (st *testSliceTransform) Name() string { return "gorocksdb.test" } +func (st *testSliceTransform) Transform(src []byte) []byte { return src[0:3] } +func (st *testSliceTransform) InDomain(src []byte) bool { return len(src) >= 3 } +func (st *testSliceTransform) InRange(src []byte) bool { return len(src) == 3 } diff --git a/gorocksdb/snapshot.go b/gorocksdb/snapshot.go new file mode 100644 index 000000000..2ea169096 --- /dev/null +++ b/gorocksdb/snapshot.go @@ -0,0 +1,14 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// Snapshot provides a consistent view of read operations in a DB. +type Snapshot struct { + c *C.rocksdb_snapshot_t +} + +// NewNativeSnapshot creates a Snapshot object. +func NewNativeSnapshot(c *C.rocksdb_snapshot_t) *Snapshot { + return &Snapshot{c} +} diff --git a/gorocksdb/sst_file_writer.go b/gorocksdb/sst_file_writer.go new file mode 100644 index 000000000..54f2c1393 --- /dev/null +++ b/gorocksdb/sst_file_writer.go @@ -0,0 +1,67 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" + +import ( + "errors" + "unsafe" +) + +// SSTFileWriter is used to create sst files that can be added to database later. +// All keys in files generated by SstFileWriter will have sequence number = 0. +type SSTFileWriter struct { + c *C.rocksdb_sstfilewriter_t +} + +// NewSSTFileWriter creates an SSTFileWriter object. +func NewSSTFileWriter(opts *EnvOptions, dbOpts *Options) *SSTFileWriter { + c := C.rocksdb_sstfilewriter_create(opts.c, dbOpts.c) + return &SSTFileWriter{c: c} +} + +// Open prepares SstFileWriter to write into file located at "path". +func (w *SSTFileWriter) Open(path string) error { + var ( + cErr *C.char + cPath = C.CString(path) + ) + defer C.free(unsafe.Pointer(cPath)) + C.rocksdb_sstfilewriter_open(w.c, cPath, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Add adds key, value to currently opened file. +// REQUIRES: key is after any previously added key according to comparator. +func (w *SSTFileWriter) Add(key, value []byte) error { + cKey := byteToChar(key) + cValue := byteToChar(value) + var cErr *C.char + C.rocksdb_sstfilewriter_add(w.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Finish finishes writing to sst file and close file. +func (w *SSTFileWriter) Finish() error { + var cErr *C.char + C.rocksdb_sstfilewriter_finish(w.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Destroy destroys the SSTFileWriter object. +func (w *SSTFileWriter) Destroy() { + C.rocksdb_sstfilewriter_destroy(w.c) +} diff --git a/gorocksdb/staticflag_linux.go b/gorocksdb/staticflag_linux.go new file mode 100644 index 000000000..3af044ef8 --- /dev/null +++ b/gorocksdb/staticflag_linux.go @@ -0,0 +1,6 @@ +// +build static + +package gorocksdb + +// #cgo LDFLAGS: -l:librocksdb.a -l:libstdc++.a -lm -ldl +import "C" diff --git a/gorocksdb/transaction.go b/gorocksdb/transaction.go new file mode 100644 index 000000000..67c9ef09c --- /dev/null +++ b/gorocksdb/transaction.go @@ -0,0 +1,125 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" + +import ( + "errors" + "unsafe" +) + +// Transaction is used with TransactionDB for transaction support. +type Transaction struct { + c *C.rocksdb_transaction_t +} + +// NewNativeTransaction creates a Transaction object. +func NewNativeTransaction(c *C.rocksdb_transaction_t) *Transaction { + return &Transaction{c} +} + +// Commit commits the transaction to the database. +func (transaction *Transaction) Commit() error { + var ( + cErr *C.char + ) + C.rocksdb_transaction_commit(transaction.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Rollback performs a rollback on the transaction. +func (transaction *Transaction) Rollback() error { + var ( + cErr *C.char + ) + C.rocksdb_transaction_rollback(transaction.c, &cErr) + + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Get returns the data associated with the key from the database given this transaction. +func (transaction *Transaction) Get(opts *ReadOptions, key []byte) (*Slice, error) { + var ( + cErr *C.char + cValLen C.size_t + cKey = byteToChar(key) + ) + cValue := C.rocksdb_transaction_get( + transaction.c, opts.c, cKey, C.size_t(len(key)), &cValLen, &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return NewSlice(cValue, cValLen), nil +} + +// GetForUpdate queries the data associated with the key and puts an exclusive lock on the key from the database given this transaction. +func (transaction *Transaction) GetForUpdate(opts *ReadOptions, key []byte) (*Slice, error) { + var ( + cErr *C.char + cValLen C.size_t + cKey = byteToChar(key) + ) + cValue := C.rocksdb_transaction_get_for_update( + transaction.c, opts.c, cKey, C.size_t(len(key)), &cValLen, C.uchar(byte(1)) /*exclusive*/, &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return NewSlice(cValue, cValLen), nil +} + +// Put writes data associated with a key to the transaction. +func (transaction *Transaction) Put(key, value []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + cValue = byteToChar(value) + ) + C.rocksdb_transaction_put( + transaction.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Delete removes the data associated with the key from the transaction. +func (transaction *Transaction) Delete(key []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + ) + C.rocksdb_transaction_delete(transaction.c, cKey, C.size_t(len(key)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// NewIterator returns an Iterator over the database that uses the +// ReadOptions given. +func (transaction *Transaction) NewIterator(opts *ReadOptions) *Iterator { + return NewNativeIterator( + unsafe.Pointer(C.rocksdb_transaction_create_iterator(transaction.c, opts.c))) +} + +// Destroy deallocates the transaction object. +func (transaction *Transaction) Destroy() { + C.rocksdb_transaction_destroy(transaction.c) + transaction.c = nil +} diff --git a/gorocksdb/transactiondb.go b/gorocksdb/transactiondb.go new file mode 100644 index 000000000..cfdeac9c1 --- /dev/null +++ b/gorocksdb/transactiondb.go @@ -0,0 +1,143 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import ( + "errors" + "unsafe" +) + +// TransactionDB is a reusable handle to a RocksDB transactional database on disk, created by OpenTransactionDb. +type TransactionDB struct { + c *C.rocksdb_transactiondb_t + name string + opts *Options + transactionDBOpts *TransactionDBOptions +} + +// OpenTransactionDb opens a database with the specified options. +func OpenTransactionDb( + opts *Options, + transactionDBOpts *TransactionDBOptions, + name string, +) (*TransactionDB, error) { + var ( + cErr *C.char + cName = C.CString(name) + ) + defer C.free(unsafe.Pointer(cName)) + db := C.rocksdb_transactiondb_open( + opts.c, transactionDBOpts.c, cName, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return &TransactionDB{ + name: name, + c: db, + opts: opts, + transactionDBOpts: transactionDBOpts, + }, nil +} + +// NewSnapshot creates a new snapshot of the database. +func (db *TransactionDB) NewSnapshot() *Snapshot { + return NewNativeSnapshot(C.rocksdb_transactiondb_create_snapshot(db.c)) +} + +// ReleaseSnapshot releases the snapshot and its resources. +func (db *TransactionDB) ReleaseSnapshot(snapshot *Snapshot) { + C.rocksdb_transactiondb_release_snapshot(db.c, snapshot.c) + snapshot.c = nil +} + +// TransactionBegin begins a new transaction +// with the WriteOptions and TransactionOptions given. +func (db *TransactionDB) TransactionBegin( + opts *WriteOptions, + transactionOpts *TransactionOptions, + oldTransaction *Transaction, +) *Transaction { + if oldTransaction != nil { + return NewNativeTransaction(C.rocksdb_transaction_begin( + db.c, + opts.c, + transactionOpts.c, + oldTransaction.c, + )) + } + + return NewNativeTransaction(C.rocksdb_transaction_begin( + db.c, opts.c, transactionOpts.c, nil)) +} + +// Get returns the data associated with the key from the database. +func (db *TransactionDB) Get(opts *ReadOptions, key []byte) (*Slice, error) { + var ( + cErr *C.char + cValLen C.size_t + cKey = byteToChar(key) + ) + cValue := C.rocksdb_transactiondb_get( + db.c, opts.c, cKey, C.size_t(len(key)), &cValLen, &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + return NewSlice(cValue, cValLen), nil +} + +// Put writes data associated with a key to the database. +func (db *TransactionDB) Put(opts *WriteOptions, key, value []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + cValue = byteToChar(value) + ) + C.rocksdb_transactiondb_put( + db.c, opts.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// Delete removes the data associated with the key from the database. +func (db *TransactionDB) Delete(opts *WriteOptions, key []byte) error { + var ( + cErr *C.char + cKey = byteToChar(key) + ) + C.rocksdb_transactiondb_delete(db.c, opts.c, cKey, C.size_t(len(key)), &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +// NewCheckpoint creates a new Checkpoint for this db. +func (db *TransactionDB) NewCheckpoint() (*Checkpoint, error) { + var ( + cErr *C.char + ) + cCheckpoint := C.rocksdb_transactiondb_checkpoint_object_create( + db.c, &cErr, + ) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return nil, errors.New(C.GoString(cErr)) + } + + return NewNativeCheckpoint(cCheckpoint), nil +} + +// Close closes the database. +func (transactionDB *TransactionDB) Close() { + C.rocksdb_transactiondb_close(transactionDB.c) + transactionDB.c = nil +} diff --git a/gorocksdb/transactiondb_test.go b/gorocksdb/transactiondb_test.go new file mode 100644 index 000000000..48fb382b1 --- /dev/null +++ b/gorocksdb/transactiondb_test.go @@ -0,0 +1,139 @@ +package gorocksdb + +import ( + "io/ioutil" + "testing" + + "github.com/facebookgo/ensure" +) + +func TestOpenTransactionDb(t *testing.T) { + db := newTestTransactionDB(t, "TestOpenTransactionDb", nil) + defer db.Close() +} + +func TestTransactionDBCRUD(t *testing.T) { + db := newTestTransactionDB(t, "TestTransactionDBGet", nil) + defer db.Close() + + var ( + givenKey = []byte("hello") + givenVal1 = []byte("world1") + givenVal2 = []byte("world2") + givenTxnKey = []byte("hello2") + givenTxnKey2 = []byte("hello3") + givenTxnVal1 = []byte("whatawonderful") + wo = NewDefaultWriteOptions() + ro = NewDefaultReadOptions() + to = NewDefaultTransactionOptions() + ) + + // create + ensure.Nil(t, db.Put(wo, givenKey, givenVal1)) + + // retrieve + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), givenVal1) + + // update + ensure.Nil(t, db.Put(wo, givenKey, givenVal2)) + v2, err := db.Get(ro, givenKey) + defer v2.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v2.Data(), givenVal2) + + // delete + ensure.Nil(t, db.Delete(wo, givenKey)) + v3, err := db.Get(ro, givenKey) + defer v3.Free() + ensure.Nil(t, err) + ensure.True(t, v3.Data() == nil) + + // transaction + txn := db.TransactionBegin(wo, to, nil) + defer txn.Destroy() + // create + ensure.Nil(t, txn.Put(givenTxnKey, givenTxnVal1)) + v4, err := txn.Get(ro, givenTxnKey) + defer v4.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v4.Data(), givenTxnVal1) + + ensure.Nil(t, txn.Commit()) + v5, err := db.Get(ro, givenTxnKey) + defer v5.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v5.Data(), givenTxnVal1) + + // transaction + txn2 := db.TransactionBegin(wo, to, nil) + defer txn2.Destroy() + // create + ensure.Nil(t, txn2.Put(givenTxnKey2, givenTxnVal1)) + // rollback + ensure.Nil(t, txn2.Rollback()) + + v6, err := txn2.Get(ro, givenTxnKey2) + defer v6.Free() + ensure.Nil(t, err) + ensure.True(t, v6.Data() == nil) + // transaction + txn3 := db.TransactionBegin(wo, to, nil) + defer txn3.Destroy() + // delete + ensure.Nil(t, txn3.Delete(givenTxnKey)) + ensure.Nil(t, txn3.Commit()) + + v7, err := db.Get(ro, givenTxnKey) + defer v7.Free() + ensure.Nil(t, err) + ensure.True(t, v7.Data() == nil) + +} + +func TestTransactionDBGetForUpdate(t *testing.T) { + lockTimeoutMilliSec := int64(50) + applyOpts := func(opts *Options, transactionDBOpts *TransactionDBOptions) { + transactionDBOpts.SetTransactionLockTimeout(lockTimeoutMilliSec) + } + db := newTestTransactionDB(t, "TestOpenTransactionDb", applyOpts) + defer db.Close() + + var ( + givenKey = []byte("hello") + givenVal = []byte("world") + wo = NewDefaultWriteOptions() + ro = NewDefaultReadOptions() + to = NewDefaultTransactionOptions() + ) + + txn := db.TransactionBegin(wo, to, nil) + defer txn.Destroy() + + v, err := txn.GetForUpdate(ro, givenKey) + defer v.Free() + ensure.Nil(t, err) + + // expect lock timeout error to be thrown + if err := db.Put(wo, givenKey, givenVal); err == nil { + t.Error("expect locktime out error, got nil error") + } +} + +func newTestTransactionDB(t *testing.T, name string, applyOpts func(opts *Options, transactionDBOpts *TransactionDBOptions)) *TransactionDB { + dir, err := ioutil.TempDir("", "gorockstransactiondb-"+name) + ensure.Nil(t, err) + + opts := NewDefaultOptions() + opts.SetCreateIfMissing(true) + transactionDBOpts := NewDefaultTransactionDBOptions() + if applyOpts != nil { + applyOpts(opts, transactionDBOpts) + } + db, err := OpenTransactionDb(opts, transactionDBOpts, dir) + ensure.Nil(t, err) + + return db +} diff --git a/gorocksdb/util.go b/gorocksdb/util.go new file mode 100644 index 000000000..b6637ec32 --- /dev/null +++ b/gorocksdb/util.go @@ -0,0 +1,74 @@ +package gorocksdb + +// #include + +import "C" +import ( + "reflect" + "unsafe" +) + +// btoi converts a bool value to int. +func btoi(b bool) int { + if b { + return 1 + } + return 0 +} + +// boolToChar converts a bool value to C.uchar. +func boolToChar(b bool) C.uchar { + if b { + return 1 + } + return 0 +} + +// charToByte converts a *C.char to a byte slice. +func charToByte(data *C.char, len C.size_t) []byte { + var value []byte + sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) + sH.Cap, sH.Len, sH.Data = int(len), int(len), uintptr(unsafe.Pointer(data)) + return value +} + +// byteToChar returns *C.char from byte slice. +func byteToChar(b []byte) *C.char { + var c *C.char + if len(b) > 0 { + c = (*C.char)(unsafe.Pointer(&b[0])) + } + return c +} + +// Go []byte to C string +// The C string is allocated in the C heap using malloc. +func cByteSlice(b []byte) *C.char { + var c *C.char + if len(b) > 0 { + c = (*C.char)(C.CBytes(b)) + } + return c +} + +// stringToChar returns *C.char from string. +func stringToChar(s string) *C.char { + ptrStr := (*reflect.StringHeader)(unsafe.Pointer(&s)) + return (*C.char)(unsafe.Pointer(ptrStr.Data)) +} + +// charSlice converts a C array of *char to a []*C.char. +func charSlice(data **C.char, len C.int) []*C.char { + var value []*C.char + sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) + sH.Cap, sH.Len, sH.Data = int(len), int(len), uintptr(unsafe.Pointer(data)) + return value +} + +// sizeSlice converts a C array of size_t to a []C.size_t. +func sizeSlice(data *C.size_t, len C.int) []C.size_t { + var value []C.size_t + sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) + sH.Cap, sH.Len, sH.Data = int(len), int(len), uintptr(unsafe.Pointer(data)) + return value +} diff --git a/gorocksdb/wal_iterator.go b/gorocksdb/wal_iterator.go new file mode 100755 index 000000000..7805d7c90 --- /dev/null +++ b/gorocksdb/wal_iterator.go @@ -0,0 +1,49 @@ +package gorocksdb + +// #include +// #include "rocksdb/c.h" +import "C" +import ( + "errors" + "unsafe" +) + +type WalIterator struct { + c *C.rocksdb_wal_iterator_t +} + +func NewNativeWalIterator(c unsafe.Pointer) *WalIterator { + return &WalIterator{(*C.rocksdb_wal_iterator_t)(c)} +} + +func (iter *WalIterator) Valid() bool { + return C.rocksdb_wal_iter_valid(iter.c) != 0 +} + +func (iter *WalIterator) Next() { + C.rocksdb_wal_iter_next(iter.c) +} + +func (iter *WalIterator) Err() error { + var cErr *C.char + C.rocksdb_wal_iter_status(iter.c, &cErr) + if cErr != nil { + defer C.rocksdb_free(unsafe.Pointer(cErr)) + return errors.New(C.GoString(cErr)) + } + return nil +} + +func (iter *WalIterator) Destroy() { + C.rocksdb_wal_iter_destroy(iter.c) + iter.c = nil +} + +// C.rocksdb_wal_iter_get_batch in the official rocksdb c wrapper has memory leak +// see https://github.com/facebook/rocksdb/pull/5515 +// https://github.com/facebook/rocksdb/issues/5536 +func (iter *WalIterator) GetBatch() (*WriteBatch, uint64) { + var cSeq C.uint64_t + cB := C.rocksdb_wal_iter_get_batch(iter.c, &cSeq) + return NewNativeWriteBatch(cB), uint64(cSeq) +} diff --git a/gorocksdb/write_batch.go b/gorocksdb/write_batch.go new file mode 100644 index 000000000..f894427bc --- /dev/null +++ b/gorocksdb/write_batch.go @@ -0,0 +1,283 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" +import ( + "errors" + "io" +) + +// WriteBatch is a batching of Puts, Merges and Deletes. +type WriteBatch struct { + c *C.rocksdb_writebatch_t +} + +// NewWriteBatch create a WriteBatch object. +func NewWriteBatch() *WriteBatch { + return NewNativeWriteBatch(C.rocksdb_writebatch_create()) +} + +// NewNativeWriteBatch create a WriteBatch object. +func NewNativeWriteBatch(c *C.rocksdb_writebatch_t) *WriteBatch { + return &WriteBatch{c} +} + +// WriteBatchFrom creates a write batch from a serialized WriteBatch. +func WriteBatchFrom(data []byte) *WriteBatch { + return NewNativeWriteBatch(C.rocksdb_writebatch_create_from(byteToChar(data), C.size_t(len(data)))) +} + +// Put queues a key-value pair. +func (wb *WriteBatch) Put(key, value []byte) { + cKey := byteToChar(key) + cValue := byteToChar(value) + C.rocksdb_writebatch_put(wb.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value))) +} + +// PutCF queues a key-value pair in a column family. +func (wb *WriteBatch) PutCF(cf *ColumnFamilyHandle, key, value []byte) { + cKey := byteToChar(key) + cValue := byteToChar(value) + C.rocksdb_writebatch_put_cf(wb.c, cf.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value))) +} + +// Append a blob of arbitrary size to the records in this batch. +func (wb *WriteBatch) PutLogData(blob []byte) { + cBlob := byteToChar(blob) + C.rocksdb_writebatch_put_log_data(wb.c, cBlob, C.size_t(len(blob))) +} + +// Merge queues a merge of "value" with the existing value of "key". +func (wb *WriteBatch) Merge(key, value []byte) { + cKey := byteToChar(key) + cValue := byteToChar(value) + C.rocksdb_writebatch_merge(wb.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value))) +} + +// MergeCF queues a merge of "value" with the existing value of "key" in a +// column family. +func (wb *WriteBatch) MergeCF(cf *ColumnFamilyHandle, key, value []byte) { + cKey := byteToChar(key) + cValue := byteToChar(value) + C.rocksdb_writebatch_merge_cf(wb.c, cf.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value))) +} + +// Delete queues a deletion of the data at key. +func (wb *WriteBatch) Delete(key []byte) { + cKey := byteToChar(key) + C.rocksdb_writebatch_delete(wb.c, cKey, C.size_t(len(key))) +} + +// DeleteCF queues a deletion of the data at key in a column family. +func (wb *WriteBatch) DeleteCF(cf *ColumnFamilyHandle, key []byte) { + cKey := byteToChar(key) + C.rocksdb_writebatch_delete_cf(wb.c, cf.c, cKey, C.size_t(len(key))) +} + +// DeleteRange deletes keys that are between [startKey, endKey) +func (wb *WriteBatch) DeleteRange(startKey []byte, endKey []byte) { + cStartKey := byteToChar(startKey) + cEndKey := byteToChar(endKey) + C.rocksdb_writebatch_delete_range(wb.c, cStartKey, C.size_t(len(startKey)), cEndKey, C.size_t(len(endKey))) +} + +// DeleteRangeCF deletes keys that are between [startKey, endKey) and +// belong to a given column family +func (wb *WriteBatch) DeleteRangeCF(cf *ColumnFamilyHandle, startKey []byte, endKey []byte) { + cStartKey := byteToChar(startKey) + cEndKey := byteToChar(endKey) + C.rocksdb_writebatch_delete_range_cf(wb.c, cf.c, cStartKey, C.size_t(len(startKey)), cEndKey, C.size_t(len(endKey))) +} + +// Data returns the serialized version of this batch. +func (wb *WriteBatch) Data() []byte { + var cSize C.size_t + cValue := C.rocksdb_writebatch_data(wb.c, &cSize) + return charToByte(cValue, cSize) +} + +// Count returns the number of updates in the batch. +func (wb *WriteBatch) Count() int { + return int(C.rocksdb_writebatch_count(wb.c)) +} + +// NewIterator returns a iterator to iterate over the records in the batch. +func (wb *WriteBatch) NewIterator() *WriteBatchIterator { + data := wb.Data() + if len(data) < 8+4 { + return &WriteBatchIterator{} + } + return &WriteBatchIterator{data: data[12:]} +} + +// Clear removes all the enqueued Put and Deletes. +func (wb *WriteBatch) Clear() { + C.rocksdb_writebatch_clear(wb.c) +} + +// Destroy deallocates the WriteBatch object. +func (wb *WriteBatch) Destroy() { + C.rocksdb_writebatch_destroy(wb.c) + wb.c = nil +} + +// WriteBatchRecordType describes the type of a batch record. +type WriteBatchRecordType byte + +// Types of batch records. +const ( + WriteBatchDeletionRecord WriteBatchRecordType = 0x0 + WriteBatchValueRecord WriteBatchRecordType = 0x1 + WriteBatchMergeRecord WriteBatchRecordType = 0x2 + WriteBatchLogDataRecord WriteBatchRecordType = 0x3 + WriteBatchCFDeletionRecord WriteBatchRecordType = 0x4 + WriteBatchCFValueRecord WriteBatchRecordType = 0x5 + WriteBatchCFMergeRecord WriteBatchRecordType = 0x6 + WriteBatchSingleDeletionRecord WriteBatchRecordType = 0x7 + WriteBatchCFSingleDeletionRecord WriteBatchRecordType = 0x8 + WriteBatchBeginPrepareXIDRecord WriteBatchRecordType = 0x9 + WriteBatchEndPrepareXIDRecord WriteBatchRecordType = 0xA + WriteBatchCommitXIDRecord WriteBatchRecordType = 0xB + WriteBatchRollbackXIDRecord WriteBatchRecordType = 0xC + WriteBatchNoopRecord WriteBatchRecordType = 0xD + WriteBatchRangeDeletion WriteBatchRecordType = 0xF + WriteBatchCFRangeDeletion WriteBatchRecordType = 0xE + WriteBatchCFBlobIndex WriteBatchRecordType = 0x10 + WriteBatchBlobIndex WriteBatchRecordType = 0x11 + WriteBatchBeginPersistedPrepareXIDRecord WriteBatchRecordType = 0x12 + WriteBatchNotUsedRecord WriteBatchRecordType = 0x7F +) + +// WriteBatchRecord represents a record inside a WriteBatch. +type WriteBatchRecord struct { + CF int + Key []byte + Value []byte + Type WriteBatchRecordType +} + +// WriteBatchIterator represents a iterator to iterator over records. +type WriteBatchIterator struct { + data []byte + record WriteBatchRecord + err error +} + +// Next returns the next record. +// Returns false if no further record exists. +func (iter *WriteBatchIterator) Next() bool { + if iter.err != nil || len(iter.data) == 0 { + return false + } + // reset the current record + iter.record.CF = 0 + iter.record.Key = nil + iter.record.Value = nil + + // parse the record type + iter.record.Type = iter.decodeRecType() + + switch iter.record.Type { + case + WriteBatchDeletionRecord, + WriteBatchSingleDeletionRecord: + iter.record.Key = iter.decodeSlice() + case + WriteBatchCFDeletionRecord, + WriteBatchCFSingleDeletionRecord: + iter.record.CF = int(iter.decodeVarint()) + if iter.err == nil { + iter.record.Key = iter.decodeSlice() + } + case + WriteBatchValueRecord, + WriteBatchMergeRecord, + WriteBatchRangeDeletion, + WriteBatchBlobIndex: + iter.record.Key = iter.decodeSlice() + if iter.err == nil { + iter.record.Value = iter.decodeSlice() + } + case + WriteBatchCFValueRecord, + WriteBatchCFRangeDeletion, + WriteBatchCFMergeRecord, + WriteBatchCFBlobIndex: + iter.record.CF = int(iter.decodeVarint()) + if iter.err == nil { + iter.record.Key = iter.decodeSlice() + } + if iter.err == nil { + iter.record.Value = iter.decodeSlice() + } + case WriteBatchLogDataRecord: + iter.record.Value = iter.decodeSlice() + case + WriteBatchNoopRecord, + WriteBatchBeginPrepareXIDRecord, + WriteBatchBeginPersistedPrepareXIDRecord: + case + WriteBatchEndPrepareXIDRecord, + WriteBatchCommitXIDRecord, + WriteBatchRollbackXIDRecord: + iter.record.Value = iter.decodeSlice() + default: + iter.err = errors.New("unsupported wal record type") + } + + return iter.err == nil + +} + +// Record returns the current record. +func (iter *WriteBatchIterator) Record() *WriteBatchRecord { + return &iter.record +} + +// Error returns the error if the iteration is failed. +func (iter *WriteBatchIterator) Error() error { + return iter.err +} + +func (iter *WriteBatchIterator) decodeSlice() []byte { + l := int(iter.decodeVarint()) + if l > len(iter.data) { + iter.err = io.ErrShortBuffer + } + if iter.err != nil { + return []byte{} + } + ret := iter.data[:l] + iter.data = iter.data[l:] + return ret +} + +func (iter *WriteBatchIterator) decodeRecType() WriteBatchRecordType { + if len(iter.data) == 0 { + iter.err = io.ErrShortBuffer + return WriteBatchNotUsedRecord + } + t := iter.data[0] + iter.data = iter.data[1:] + return WriteBatchRecordType(t) +} + +func (iter *WriteBatchIterator) decodeVarint() uint64 { + var n int + var x uint64 + for shift := uint(0); shift < 64 && n < len(iter.data); shift += 7 { + b := uint64(iter.data[n]) + n++ + x |= (b & 0x7F) << shift + if (b & 0x80) == 0 { + iter.data = iter.data[n:] + return x + } + } + if n == len(iter.data) { + iter.err = io.ErrShortBuffer + } else { + iter.err = errors.New("malformed varint") + } + return 0 +} diff --git a/gorocksdb/write_batch_test.go b/gorocksdb/write_batch_test.go new file mode 100644 index 000000000..72eeb36e7 --- /dev/null +++ b/gorocksdb/write_batch_test.go @@ -0,0 +1,87 @@ +package gorocksdb + +import ( + "testing" + + "github.com/facebookgo/ensure" +) + +func TestWriteBatch(t *testing.T) { + db := newTestDB(t, "TestWriteBatch", nil) + defer db.Close() + + var ( + givenKey1 = []byte("key1") + givenVal1 = []byte("val1") + givenKey2 = []byte("key2") + ) + wo := NewDefaultWriteOptions() + ensure.Nil(t, db.Put(wo, givenKey2, []byte("foo"))) + + // create and fill the write batch + wb := NewWriteBatch() + defer wb.Destroy() + wb.Put(givenKey1, givenVal1) + wb.Delete(givenKey2) + ensure.DeepEqual(t, wb.Count(), 2) + + // perform the batch + ensure.Nil(t, db.Write(wo, wb)) + + // check changes + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, givenKey1) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), givenVal1) + + v2, err := db.Get(ro, givenKey2) + defer v2.Free() + ensure.Nil(t, err) + ensure.True(t, v2.Data() == nil) + + // DeleteRange test + wb.Clear() + wb.DeleteRange(givenKey1, givenKey2) + + // perform the batch + ensure.Nil(t, db.Write(wo, wb)) + + v1, err = db.Get(ro, givenKey1) + defer v1.Free() + ensure.Nil(t, err) + ensure.True(t, v1.Data() == nil) +} + +func TestWriteBatchIterator(t *testing.T) { + db := newTestDB(t, "TestWriteBatchIterator", nil) + defer db.Close() + + var ( + givenKey1 = []byte("key1") + givenVal1 = []byte("val1") + givenKey2 = []byte("key2") + ) + // create and fill the write batch + wb := NewWriteBatch() + defer wb.Destroy() + wb.Put(givenKey1, givenVal1) + wb.Delete(givenKey2) + ensure.DeepEqual(t, wb.Count(), 2) + + // iterate over the batch + iter := wb.NewIterator() + ensure.True(t, iter.Next()) + record := iter.Record() + ensure.DeepEqual(t, record.Type, WriteBatchValueRecord) + ensure.DeepEqual(t, record.Key, givenKey1) + ensure.DeepEqual(t, record.Value, givenVal1) + + ensure.True(t, iter.Next()) + record = iter.Record() + ensure.DeepEqual(t, record.Type, WriteBatchDeletionRecord) + ensure.DeepEqual(t, record.Key, givenKey2) + + // there shouldn't be any left + ensure.False(t, iter.Next()) +} From edc139ee6c3bdff9bc2bfd8ba864b5611a906442 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Wed, 21 Dec 2022 20:20:28 +0800 Subject: [PATCH 06/22] add go-ycsb --- go-ycsb | 1 + 1 file changed, 1 insertion(+) create mode 160000 go-ycsb diff --git a/go-ycsb b/go-ycsb new file mode 160000 index 000000000..313a79e9c --- /dev/null +++ b/go-ycsb @@ -0,0 +1 @@ +Subproject commit 313a79e9c48a60772ee5305d56006874325306c6 From c0fca7e77a03faa2961b10b53426aacdb75bcf38 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Wed, 21 Dec 2022 20:21:30 +0800 Subject: [PATCH 07/22] add go-ycsb --- go-ycsb | 1 - go-ycsb/.dockerignore | 3 + go-ycsb/.github/workflows/docker.yml | 19 + .../workflows/github-release-publish.yml | 26 + go-ycsb/.github/workflows/go.yml | 56 + go-ycsb/.gitignore | 9 + go-ycsb/Dockerfile | 32 + go-ycsb/LICENSE | 201 +++ go-ycsb/Makefile | 41 + go-ycsb/README.md | 341 +++++ go-ycsb/YCSB-LICENSE | 202 +++ go-ycsb/cmd/go-ycsb/client.go | 111 ++ go-ycsb/cmd/go-ycsb/main.go | 200 +++ go-ycsb/cmd/go-ycsb/shell.go | 247 ++++ go-ycsb/db/aerospike/db.go | 176 +++ go-ycsb/db/badger/db.go | 263 ++++ go-ycsb/db/basic/db.go | 272 ++++ go-ycsb/db/boltdb/db.go | 244 ++++ go-ycsb/db/cassandra/db.go | 268 ++++ go-ycsb/db/dynamodb/db.go | 295 ++++ go-ycsb/db/elasticsearch/db.go | 346 +++++ go-ycsb/db/etcd/db.go | 163 +++ go-ycsb/db/etcd/doc.go | 3 + go-ycsb/db/foundationdb/db.go | 204 +++ go-ycsb/db/foundationdb/doc.go | 3 + go-ycsb/db/minio/db.go | 128 ++ go-ycsb/db/mongodb/db.go | 203 +++ go-ycsb/db/mysql/db.go | 512 +++++++ go-ycsb/db/pegasus/db.go | 176 +++ go-ycsb/db/pg/db.go | 361 +++++ go-ycsb/db/redis/db.go | 358 +++++ go-ycsb/db/rocksdb/db.go | 265 ++++ go-ycsb/db/rocksdb/doc.go | 3 + go-ycsb/db/spanner/db.go | 358 +++++ go-ycsb/db/sqlite/db.go | 400 ++++++ go-ycsb/db/sqlite/doc.go | 3 + go-ycsb/db/tinydb/db.go | 160 +++ go-ycsb/fdb-dockerfile | 37 + go-ycsb/go.mod | 176 +++ go-ycsb/go.sum | 1234 +++++++++++++++++ go-ycsb/pkg/client/client.go | 227 +++ go-ycsb/pkg/client/dbwrapper.go | 174 +++ go-ycsb/pkg/generator/acknowledged_counter.go | 110 ++ go-ycsb/pkg/generator/constant.go | 53 + go-ycsb/pkg/generator/counter.go | 58 + go-ycsb/pkg/generator/discrete.go | 77 + go-ycsb/pkg/generator/exponential.go | 64 + go-ycsb/pkg/generator/histogram.go | 134 ++ go-ycsb/pkg/generator/hotspot.go | 87 ++ go-ycsb/pkg/generator/number.go | 46 + go-ycsb/pkg/generator/scrambled_zipfian.go | 76 + go-ycsb/pkg/generator/sequential.go | 63 + go-ycsb/pkg/generator/skewedlatest.go | 68 + go-ycsb/pkg/generator/uniform.go | 57 + go-ycsb/pkg/generator/zipfian.go | 170 +++ go-ycsb/pkg/measurement/csv.go | 51 + go-ycsb/pkg/measurement/histogram.go | 97 ++ go-ycsb/pkg/measurement/histograms.go | 76 + go-ycsb/pkg/measurement/measurement.go | 126 ++ go-ycsb/pkg/prop/prop.go | 119 ++ go-ycsb/pkg/util/concurrent_map.go | 319 +++++ go-ycsb/pkg/util/core.go | 134 ++ go-ycsb/pkg/util/core_test.go | 24 + go-ycsb/pkg/util/hack.go | 43 + go-ycsb/pkg/util/hash.go | 46 + go-ycsb/pkg/util/output.go | 77 + go-ycsb/pkg/util/row.go | 115 ++ go-ycsb/pkg/util/row_test.go | 38 + go-ycsb/pkg/util/spinlock.go | 50 + go-ycsb/pkg/util/tls.go | 59 + go-ycsb/pkg/util/util.go | 73 + go-ycsb/pkg/workload/core.go | 705 ++++++++++ go-ycsb/pkg/ycsb/db.go | 120 ++ go-ycsb/pkg/ycsb/generator.go | 28 + go-ycsb/pkg/ycsb/measurement.go | 31 + go-ycsb/pkg/ycsb/workload.go | 71 + go-ycsb/tool/binary/bench.sh | 59 + go-ycsb/tool/binary/property/badger | 16 + go-ycsb/tool/binary/property/rocksdb | 27 + go-ycsb/tool/docker/bench.sh | 103 ++ go-ycsb/tool/docker/cassandra.yml | 24 + go-ycsb/tool/docker/clear.sh | 7 + go-ycsb/tool/docker/cockroach.yml | 26 + go-ycsb/tool/docker/config/init_cassandra.sh | 14 + go-ycsb/tool/docker/config/init_cockroach.sh | 8 + go-ycsb/tool/docker/config/pd.toml | 91 ++ go-ycsb/tool/docker/config/tidb.toml | 260 ++++ go-ycsb/tool/docker/config/tikv.toml | 689 +++++++++ go-ycsb/tool/docker/mariadb.yml | 17 + go-ycsb/tool/docker/mysql.yml | 17 + go-ycsb/tool/docker/mysql8.yml | 17 + go-ycsb/tool/docker/pg.yml | 18 + go-ycsb/tool/docker/raw.yml | 44 + go-ycsb/tool/docker/run_bench.sh | 10 + go-ycsb/tool/docker/scylla.yml | 24 + go-ycsb/tool/docker/sqlite.yml | 8 + go-ycsb/tool/docker/tidb.yml | 60 + go-ycsb/tool/docker/tikv.yml | 44 + go-ycsb/tool/report.go | 245 ++++ go-ycsb/workloads/minio | 4 + go-ycsb/workloads/workload_template | 206 +++ go-ycsb/workloads/workloada | 37 + go-ycsb/workloads/workloadb | 36 + go-ycsb/workloads/workloadc | 38 + go-ycsb/workloads/workloadd | 41 + go-ycsb/workloads/workloade | 46 + go-ycsb/workloads/workloadf | 37 + 107 files changed, 14238 insertions(+), 1 deletion(-) delete mode 160000 go-ycsb create mode 100644 go-ycsb/.dockerignore create mode 100644 go-ycsb/.github/workflows/docker.yml create mode 100644 go-ycsb/.github/workflows/github-release-publish.yml create mode 100644 go-ycsb/.github/workflows/go.yml create mode 100644 go-ycsb/.gitignore create mode 100644 go-ycsb/Dockerfile create mode 100644 go-ycsb/LICENSE create mode 100644 go-ycsb/Makefile create mode 100644 go-ycsb/README.md create mode 100644 go-ycsb/YCSB-LICENSE create mode 100644 go-ycsb/cmd/go-ycsb/client.go create mode 100644 go-ycsb/cmd/go-ycsb/main.go create mode 100644 go-ycsb/cmd/go-ycsb/shell.go create mode 100644 go-ycsb/db/aerospike/db.go create mode 100644 go-ycsb/db/badger/db.go create mode 100644 go-ycsb/db/basic/db.go create mode 100644 go-ycsb/db/boltdb/db.go create mode 100644 go-ycsb/db/cassandra/db.go create mode 100644 go-ycsb/db/dynamodb/db.go create mode 100644 go-ycsb/db/elasticsearch/db.go create mode 100644 go-ycsb/db/etcd/db.go create mode 100644 go-ycsb/db/etcd/doc.go create mode 100644 go-ycsb/db/foundationdb/db.go create mode 100644 go-ycsb/db/foundationdb/doc.go create mode 100644 go-ycsb/db/minio/db.go create mode 100644 go-ycsb/db/mongodb/db.go create mode 100644 go-ycsb/db/mysql/db.go create mode 100644 go-ycsb/db/pegasus/db.go create mode 100644 go-ycsb/db/pg/db.go create mode 100644 go-ycsb/db/redis/db.go create mode 100644 go-ycsb/db/rocksdb/db.go create mode 100644 go-ycsb/db/rocksdb/doc.go create mode 100644 go-ycsb/db/spanner/db.go create mode 100644 go-ycsb/db/sqlite/db.go create mode 100644 go-ycsb/db/sqlite/doc.go create mode 100644 go-ycsb/db/tinydb/db.go create mode 100644 go-ycsb/fdb-dockerfile create mode 100644 go-ycsb/go.mod create mode 100644 go-ycsb/go.sum create mode 100644 go-ycsb/pkg/client/client.go create mode 100644 go-ycsb/pkg/client/dbwrapper.go create mode 100644 go-ycsb/pkg/generator/acknowledged_counter.go create mode 100644 go-ycsb/pkg/generator/constant.go create mode 100644 go-ycsb/pkg/generator/counter.go create mode 100644 go-ycsb/pkg/generator/discrete.go create mode 100644 go-ycsb/pkg/generator/exponential.go create mode 100644 go-ycsb/pkg/generator/histogram.go create mode 100644 go-ycsb/pkg/generator/hotspot.go create mode 100644 go-ycsb/pkg/generator/number.go create mode 100644 go-ycsb/pkg/generator/scrambled_zipfian.go create mode 100644 go-ycsb/pkg/generator/sequential.go create mode 100644 go-ycsb/pkg/generator/skewedlatest.go create mode 100644 go-ycsb/pkg/generator/uniform.go create mode 100644 go-ycsb/pkg/generator/zipfian.go create mode 100644 go-ycsb/pkg/measurement/csv.go create mode 100644 go-ycsb/pkg/measurement/histogram.go create mode 100644 go-ycsb/pkg/measurement/histograms.go create mode 100644 go-ycsb/pkg/measurement/measurement.go create mode 100644 go-ycsb/pkg/prop/prop.go create mode 100644 go-ycsb/pkg/util/concurrent_map.go create mode 100644 go-ycsb/pkg/util/core.go create mode 100644 go-ycsb/pkg/util/core_test.go create mode 100644 go-ycsb/pkg/util/hack.go create mode 100644 go-ycsb/pkg/util/hash.go create mode 100644 go-ycsb/pkg/util/output.go create mode 100644 go-ycsb/pkg/util/row.go create mode 100644 go-ycsb/pkg/util/row_test.go create mode 100644 go-ycsb/pkg/util/spinlock.go create mode 100644 go-ycsb/pkg/util/tls.go create mode 100644 go-ycsb/pkg/util/util.go create mode 100644 go-ycsb/pkg/workload/core.go create mode 100644 go-ycsb/pkg/ycsb/db.go create mode 100644 go-ycsb/pkg/ycsb/generator.go create mode 100644 go-ycsb/pkg/ycsb/measurement.go create mode 100644 go-ycsb/pkg/ycsb/workload.go create mode 100755 go-ycsb/tool/binary/bench.sh create mode 100644 go-ycsb/tool/binary/property/badger create mode 100644 go-ycsb/tool/binary/property/rocksdb create mode 100755 go-ycsb/tool/docker/bench.sh create mode 100644 go-ycsb/tool/docker/cassandra.yml create mode 100755 go-ycsb/tool/docker/clear.sh create mode 100644 go-ycsb/tool/docker/cockroach.yml create mode 100755 go-ycsb/tool/docker/config/init_cassandra.sh create mode 100755 go-ycsb/tool/docker/config/init_cockroach.sh create mode 100644 go-ycsb/tool/docker/config/pd.toml create mode 100644 go-ycsb/tool/docker/config/tidb.toml create mode 100644 go-ycsb/tool/docker/config/tikv.toml create mode 100644 go-ycsb/tool/docker/mariadb.yml create mode 100644 go-ycsb/tool/docker/mysql.yml create mode 100644 go-ycsb/tool/docker/mysql8.yml create mode 100644 go-ycsb/tool/docker/pg.yml create mode 100644 go-ycsb/tool/docker/raw.yml create mode 100755 go-ycsb/tool/docker/run_bench.sh create mode 100644 go-ycsb/tool/docker/scylla.yml create mode 100644 go-ycsb/tool/docker/sqlite.yml create mode 100644 go-ycsb/tool/docker/tidb.yml create mode 100644 go-ycsb/tool/docker/tikv.yml create mode 100644 go-ycsb/tool/report.go create mode 100644 go-ycsb/workloads/minio create mode 100644 go-ycsb/workloads/workload_template create mode 100644 go-ycsb/workloads/workloada create mode 100644 go-ycsb/workloads/workloadb create mode 100644 go-ycsb/workloads/workloadc create mode 100644 go-ycsb/workloads/workloadd create mode 100644 go-ycsb/workloads/workloade create mode 100644 go-ycsb/workloads/workloadf diff --git a/go-ycsb b/go-ycsb deleted file mode 160000 index 313a79e9c..000000000 --- a/go-ycsb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 313a79e9c48a60772ee5305d56006874325306c6 diff --git a/go-ycsb/.dockerignore b/go-ycsb/.dockerignore new file mode 100644 index 000000000..8effa3eae --- /dev/null +++ b/go-ycsb/.dockerignore @@ -0,0 +1,3 @@ +.git +LICENSE +PATENTS diff --git a/go-ycsb/.github/workflows/docker.yml b/go-ycsb/.github/workflows/docker.yml new file mode 100644 index 000000000..d6a4ca036 --- /dev/null +++ b/go-ycsb/.github/workflows/docker.yml @@ -0,0 +1,19 @@ +name: Docker Image CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Build the Docker image + run: docker build . --file Dockerfile --tag my-image-name:$(date +%s) + diff --git a/go-ycsb/.github/workflows/github-release-publish.yml b/go-ycsb/.github/workflows/github-release-publish.yml new file mode 100644 index 000000000..436b2616c --- /dev/null +++ b/go-ycsb/.github/workflows/github-release-publish.yml @@ -0,0 +1,26 @@ +# .github/workflows/github-release-publish.yml +name: Publish artifacts to github release + +on: + release: + types: [published] + +jobs: + releases-matrix: + name: Release Go Binary + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux, darwin] + goarch: [amd64, arm64] + steps: + - uses: actions/checkout@v3 + - uses: wangyoucao577/go-release-action@v1.28 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + binary_name: "./bin/go-ycsb" + sha256sum: true + asset_name: go-ycsb-${{ matrix.goos }}-${{ matrix.goarch }} + build_command: "make" diff --git a/go-ycsb/.github/workflows/go.yml b/go-ycsb/.github/workflows/go.yml new file mode 100644 index 000000000..37c973ed3 --- /dev/null +++ b/go-ycsb/.github/workflows/go.yml @@ -0,0 +1,56 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + runs-on: ubuntu-latest + strategy: + matrix: + os: ["linux", "darwin"] + arch: ["amd64", "arm64"] + steps: + + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.18 + + - name: Build for ${{ matrix.os }}-${{ matrix.arch }} + env: + GOOS: ${{ matrix.os }} + GOARCH: ${{ matrix.arch }} + run: | + make + tar -C bin -zcf go-ycsb-$GOOS-$GOARCH.tar.gz go-ycsb + + - name: Release latest build + uses: softprops/action-gh-release@v1 + if: github.event_name == 'push' + with: + name: Latest Build + tag_name: latest-${{ github.sha }} + files: | + *.tar.gz + - name: Clean legacy latest releases + uses: actions/github-script@v6 + if: github.event_name == 'push' + with: + script: | + const { owner, repo } = context.repo; + const releases = (await github.rest.repos.listReleases({ owner, repo })).data.filter(r => r.draft && r.tag_name.startsWith('latest')); + for (const r of releases) { await github.rest.repos.deleteRelease({ owner, repo, release_id: r.id }).catch(_ => {}); } + - name: Clean legacy latest tags + if: github.event_name == 'push' + run: | + git tag -l | grep latest | grep -v latest-${{ github.sha }} | xargs -I{} git push -d origin {} || true diff --git a/go-ycsb/.gitignore b/go-ycsb/.gitignore new file mode 100644 index 000000000..6746da662 --- /dev/null +++ b/go-ycsb/.gitignore @@ -0,0 +1,9 @@ +bin +data +logs +log +.idea +.vscode +vendor +tool/tool +.DS_Store \ No newline at end of file diff --git a/go-ycsb/Dockerfile b/go-ycsb/Dockerfile new file mode 100644 index 000000000..689451513 --- /dev/null +++ b/go-ycsb/Dockerfile @@ -0,0 +1,32 @@ +FROM golang:1.18.4-alpine3.16 + +ENV GOPATH /go + +RUN apk update && apk upgrade && \ + apk add --no-cache git build-base wget + +RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 \ + && chmod +x /usr/local/bin/dumb-init + +RUN mkdir -p /go/src/github.com/pingcap/go-ycsb +WORKDIR /go/src/github.com/pingcap/go-ycsb + +COPY go.mod . +COPY go.sum . + +RUN GO111MODULE=on go mod download + +COPY . . + +RUN GO111MODULE=on go build -o /go-ycsb ./cmd/* + +FROM alpine:3.8 + +COPY --from=0 /go-ycsb /go-ycsb +COPY --from=0 /usr/local/bin/dumb-init /usr/local/bin/dumb-init + +ADD workloads /workloads + +EXPOSE 6060 + +ENTRYPOINT [ "/usr/local/bin/dumb-init", "/go-ycsb" ] diff --git a/go-ycsb/LICENSE b/go-ycsb/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/go-ycsb/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/go-ycsb/Makefile b/go-ycsb/Makefile new file mode 100644 index 000000000..478f60a51 --- /dev/null +++ b/go-ycsb/Makefile @@ -0,0 +1,41 @@ +FDB_CHECK := $(shell command -v fdbcli 2> /dev/null) +ROCKSDB_CHECK := $(shell echo "int main() { return 0; }" | gcc -lrocksdb -x c++ -o /dev/null - 2>/dev/null; echo $$?) +SQLITE_CHECK := $(shell echo "int main() { return 0; }" | gcc -lsqlite3 -x c++ -o /dev/null - 2>/dev/null; echo $$?) + +TAGS = + +ifdef FDB_CHECK + TAGS += foundationdb +endif + +ifneq ($(shell go env GOOS), $(shell go env GOHOSTOS)) + CROSS_COMPILE := 1 +endif +ifneq ($(shell go env GOARCH), $(shell go env GOHOSTARCH)) + CROSS_COMPILE := 1 +endif + +ifndef CROSS_COMPILE + +ifeq ($(SQLITE_CHECK), 0) + TAGS += libsqlite3 +endif + +ifeq ($(ROCKSDB_CHECK), 0) + TAGS += rocksdb + CGO_CXXFLAGS := "${CGO_CXXFLAGS} -std=c++11" + CGO_FLAGS += CGO_CXXFLAGS=$(CGO_CXXFLAGS) +endif + +endif + +default: build + +build: export GO111MODULE=on +build: + $(CGO_FLAGS) go build -o bin/go-ycsb cmd/go-ycsb/* + +check: + golint -set_exit_status db/... cmd/... pkg/... + + diff --git a/go-ycsb/README.md b/go-ycsb/README.md new file mode 100644 index 000000000..9b4c1f094 --- /dev/null +++ b/go-ycsb/README.md @@ -0,0 +1,341 @@ +# go-ycsb + +go-ycsb is a Go port of [YCSB](https://github.com/brianfrankcooper/YCSB). It fully supports all YCSB generators and the Core workload so we can do the basic CRUD benchmarks with Go. + +## Why another Go YCSB? + ++ We want to build a standard benchmark tool in Go. ++ We are not familiar with Java. + +## Getting Started + +### Download + +https://github.com/pingcap/go-ycsb/releases/latest + +**Linux** +``` +wget -c https://github.com/pingcap/go-ycsb/releases/latest/download/go-ycsb-linux-amd64.tar.gz -O - | tar -xz + +# give it a try +./go-ycsb --help +``` + +**OSX** +``` +wget -c https://github.com/pingcap/go-ycsb/releases/latest/download/go-ycsb-darwin-amd64.tar.gz -O - | tar -xz + +# give it a try +./go-ycsb --help +``` + +### Building from source + +```bash +git clone https://github.com/pingcap/go-ycsb.git +cd go-ycsb +make + +# give it a try +./bin/go-ycsb --help +``` + +Notice: + ++ Minimum supported go version is 1.16. ++ To use FoundationDB, you must install [client](https://www.foundationdb.org/download/) library at first, now the supported version is 6.2.11. ++ To use RocksDB, you must follow [INSTALL](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) to install RocksDB at first. + +## Usage + +Mostly, we can start from the official document [Running-a-Workload](https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload). + +### Shell + +```basic +./bin/go-ycsb shell basic +» help +YCSB shell command + +Usage: + shell [command] + +Available Commands: + delete Delete a record + help Help about any command + insert Insert a record + read Read a record + scan Scan starting at key + table Get or [set] the name of the table + update Update a record +``` + +### Load + +```bash +./bin/go-ycsb load basic -P workloads/workloada +``` + +### Run + +```bash +./bin/go-ycsb run basic -P workloads/workloada +``` + +## Supported Database + +- MySQL / TiDB +- TiKV +- FoundationDB +- Aerospike +- Badger +- Cassandra / ScyllaDB +- Pegasus +- PostgreSQL / CockroachDB / AlloyDB / Yugabyte +- RocksDB +- Spanner +- Sqlite +- MongoDB +- Redis and Redis Cluster +- BoltDB +- etcd +- DynamoDB + +## Output configuration + +|field|default value|description| +|-|-|-| +|measurementtype|"histogram"|The mechanism for recording measurements, one of `histogram`, `raw` or `csv`| +|measurement.output_file|""|File to write output to, default writes to stdout| + +## Database Configuration + +You can pass the database configurations through `-p field=value` in the command line directly. + +Common configurations: + +|field|default value|description| +|-|-|-| +|dropdata|false|Whether to remove all data before test| +|verbose|false|Output the execution query| +|debug.pprof|":6060"|Go debug profile address| + +### MySQL & TiDB + +|field|default value|description| +|-|-|-| +|mysql.host|"127.0.0.1"|MySQL Host| +|mysql.port|3306|MySQL Port| +|mysql.user|"root"|MySQL User| +|mysql.password||MySQL Password| +|mysql.db|"test"|MySQL Database| +|tidb.cluster_index|true|Whether to use cluster index, for TiDB only| +|tidb.instances|""|Comma-seperated address list of tidb instances (eg: `tidb-0:4000,tidb-1:4000`)| + + +### TiKV + +|field|default value|description| +|-|-|-| +|tikv.pd|"127.0.0.1:2379"|PD endpoints, seperated by comma| +|tikv.type|"raw"|TiKV mode, "raw", "txn", or "coprocessor"| +|tikv.conncount|128|gRPC connection count| +|tikv.batchsize|128|Request batch size| +|tikv.async_commit|true|Enalbe async commit or not| +|tikv.one_pc|true|Enable one phase or not| +|tikv.apiversion|"V1"|[api-version](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#api-version-new-in-v610) of tikv server, "V1" or "V2"| + +### FoundationDB + +|field|default value|description| +|-|-|-| +|fdb.cluster|""|The cluster file used for FoundationDB, if not set, will use the [default](https://apple.github.io/foundationdb/administration.html#default-cluster-file)| +|fdb.dbname|"DB"|The cluster database name| +|fdb.apiversion|510|API version, now only 5.1 is supported| + +### PostgreSQL & CockroachDB & AlloyDB & Yugabyte + +|field|default value|description| +|-|-|-| +|pg.host|"127.0.0.1"|PostgreSQL Host| +|pg.port|5432|PostgreSQL Port| +|pg.user|"root"|PostgreSQL User| +|pg.password||PostgreSQL Password| +|pg.db|"test"|PostgreSQL Database| +|pg.sslmode|"disable|PostgreSQL ssl mode| + +### Aerospike + +|field|default value|description| +|-|-|-| +|aerospike.host|"localhost"|The port of the Aerospike service| +|aerospike.port|3000|The port of the Aerospike service| +|aerospike.ns|"test"|The namespace to use| + +### Badger + +|field|default value|description| +|-|-|-| +|badger.dir|"/tmp/badger"|The directory to save data| +|badger.valuedir|"/tmp/badger"|The directory to save value, if not set, use badger.dir| +|badger.sync_writes|false|Sync all writes to disk| +|badger.num_versions_to_keep|1|How many versions to keep per key| +|badger.max_table_size|64MB|Each table (or file) is at most this size| +|badger.level_size_multiplier|10|Equals SizeOf(Li+1)/SizeOf(Li)| +|badger.max_levels|7|Maximum number of levels of compaction| +|badger.value_threshold|32|If value size >= this threshold, only store value offsets in tree| +|badger.num_memtables|5|Maximum number of tables to keep in memory, before stalling| +|badger.num_level0_tables|5|Maximum number of Level 0 tables before we start compacting| +|badger.num_level0_tables_stall|10|If we hit this number of Level 0 tables, we will stall until L0 is compacted away| +|badger.level_one_size|256MB|Maximum total size for L1| +|badger.value_log_file_size|1GB|Size of single value log file| +|badger.value_log_max_entries|1000000|Max number of entries a value log file can hold (approximately). A value log file would be determined by the smaller of its file size and max entries| +|badger.num_compactors|3|Number of compaction workers to run concurrently| +|badger.do_not_compact|false|Stops LSM tree from compactions| +|badger.table_loading_mode|options.LoadToRAM|How should LSM tree be accessed| +|badger.value_log_loading_mode|options.MemoryMap|How should value log be accessed| + +### RocksDB + +|field|default value|description| +|-|-|-| +|rocksdb.dir|"/tmp/rocksdb"|The directory to save data| +|rocksdb.allow_concurrent_memtable_writes|true|Sets whether to allow concurrent memtable writes| +|rocksdb.allow_mmap_reads|false|Enable/Disable mmap reads for reading sst tables| +|rocksdb.allow_mmap_writes|false|Enable/Disable mmap writes for writing sst tables| +|rocksdb.arena_block_size|0(write_buffer_size / 8)|Sets the size of one block in arena memory allocation| +|rocksdb.db_write_buffer_size|0(disable)|Sets the amount of data to build up in memtables across all column families before writing to disk| +|rocksdb.hard_pending_compaction_bytes_limit|256GB|Sets the bytes threshold at which all writes are stopped if estimated bytes needed to be compaction exceed this threshold| +|rocksdb.level0_file_num_compaction_trigger|4|Sets the number of files to trigger level-0 compaction| +|rocksdb.level0_slowdown_writes_trigger|20|Sets the soft limit on number of level-0 files| +|rocksdb.level0_stop_writes_trigger|36|Sets the maximum number of level-0 files. We stop writes at this point| +|rocksdb.max_bytes_for_level_base|256MB|Sets the maximum total data size for base level| +|rocksdb.max_bytes_for_level_multiplier|10|Sets the max Bytes for level multiplier| +|rocksdb.max_total_wal_size|0(\[sum of all write_buffer_size * max_write_buffer_number\] * 4)|Sets the maximum total wal size in bytes. Once write-ahead logs exceed this size, we will start forcing the flush of column families whose memtables are backed by the oldest live WAL file (i.e. the ones that are causing all the space amplification)| +|rocksdb.memtable_huge_page_size|0|Sets the page size for huge page for arena used by the memtable| +|rocksdb.num_levels|7|Sets the number of levels for this database| +|rocksdb.use_direct_reads|false|Enable/Disable direct I/O mode (O_DIRECT) for reads| +|rocksdb.use_fsync|false|Enable/Disable fsync| +|rocksdb.write_buffer_size|64MB|Sets the amount of data to build up in memory (backed by an unsorted log on disk) before converting to a sorted on-disk file| +|rocksdb.max_write_buffer_number|2|Sets the maximum number of write buffers that are built up in memory| +|rocksdb.max_background_jobs|2|Sets maximum number of concurrent background jobs (compactions and flushes)| +|rocksdb.block_size|4KB|Sets the approximate size of user data packed per block. Note that the block size specified here corresponds opts uncompressed data. The actual size of the unit read from disk may be smaller if compression is enabled| +|rocksdb.block_size_deviation|10|Sets the block size deviation. This is used opts close a block before it reaches the configured 'block_size'. If the percentage of free space in the current block is less than this specified number and adding a new record opts the block will exceed the configured block size, then this block will be closed and the new record will be written opts the next block| +|rocksdb.cache_index_and_filter_blocks|false|Indicating if we'd put index/filter blocks to the block cache. If not specified, each "table reader" object will pre-load index/filter block during table initialization| +|rocksdb.no_block_cache|false|Specify whether block cache should be used or not| +|rocksdb.pin_l0_filter_and_index_blocks_in_cache|false|Sets cache_index_and_filter_blocks. If is true and the below is true (hash_index_allow_collision), then filter and index blocks are stored in the cache, but a reference is held in the "table reader" object so the blocks are pinned and only evicted from cache when the table reader is freed| +|rocksdb.whole_key_filtering|true|Specify if whole keys in the filter (not just prefixes) should be placed. This must generally be true for gets opts be efficient| +|rocksdb.block_restart_interval|16|Sets the number of keys between restart points for delta encoding of keys. This parameter can be changed dynamically| +|rocksdb.filter_policy|nil|Sets the filter policy opts reduce disk reads. Many applications will benefit from passing the result of NewBloomFilterPolicy() here| +|rocksdb.index_type|kBinarySearch|Sets the index type used for this table. __kBinarySearch__: A space efficient index block that is optimized for binary-search-based index. __kHashSearch__: The hash index, if enabled, will do the hash lookup when `Options.prefix_extractor` is provided. __kTwoLevelIndexSearch__: A two-level index implementation. Both levels are binary search indexes| +|rocksdb.block_align|false|Enable/Disable align data blocks on lesser of page size and block size| + +### Spanner + +|field|default value|description| +|-|-|-| +|spanner.db|""|Spanner Database| +|spanner.credentials|"~/.spanner/credentials.json"|Google application credentials for Spanner| + +### Sqlite + +|field|default value|description| +|-|-|-| +|sqlite.db|"/tmp/sqlite.db"|Database path| +|sqlite.mode|"rwc"|Open Mode: ro, rc, rwc, memory| +|sqlite.journalmode|"DELETE"|Journal mode: DELETE, TRUNCSTE, PERSIST, MEMORY, WAL, OFF| +|sqlite.cache|"Shared"|Cache: shared, private| + +### Cassandra + +|field|default value|description| +|-|-|-| +|cassandra.cluster|"127.0.0.1:9042"|Cassandra cluster| +|cassandra.keyspace|"test"|Keyspace| +|cassandra.connections|2|Number of connections per host| +|cassandra.username|cassandra|Username| +|cassandra.password|cassandra|Password| + +### MongoDB + +|field|default value|description| +|-|-|-| +|mongodb.url|"mongodb://127.0.0.1:27017"|MongoDB URI| +|mongodb.tls_skip_verify|false|Enable/disable server ca certificate verification| +|mongodb.tls_ca_file|""|Path to mongodb server ca certificate file| +|mongodb.namespace|"ycsb.ycsb"|Namespace to use| +|mongodb.authdb|"admin"|Authentication database| +|mongodb.username|N/A|Username for authentication| +|mongodb.password|N/A|Password for authentication| + +### Redis +|field|default value|description| +|-|-|-| +|redis.datatype|hash|"hash", "string" or "json" ("json" requires [RedisJSON](https://redis.io/docs/stack/json/) available)| +|redis.mode|single|"single" or "cluster"| +|redis.network|tcp|"tcp" or "unix"| +|redis.addr||Redis server address(es) in "host:port" form, can be semi-colon `;` separated in cluster mode| +|redis.password||Redis server password| +|redis.db|0|Redis server target db| +|redis.max_redirects|8|The maximum number of retries before giving up (only for cluster mode)| +|redis.read_only|false|Enables read-only commands on slave nodes (only for cluster mode)| +|redis.route_by_latency|false|Allows routing read-only commands to the closest master or slave node (only for cluster mode)| +|redis.route_randomly|false|Allows routing read-only commands to the random master or slave node (only for cluster mode)| +|redis.max_retries||Max retries before giving up connection| +|redis.min_retry_backoff|8ms|Minimum backoff between each retry| +|redis.max_retry_backoff|512ms|Maximum backoff between each retry| +|redis.dial_timeout|5s|Dial timeout for establishing new connection| +|redis.read_timeout|3s|Timeout for socket reads| +|redis.write_timeout|3s|Timeout for socket writes| +|redis.pool_size|10|Maximum number of socket connections| +|redis.min_idle_conns|0|Minimum number of idle connections| +|redis.max_conn_age|0|Connection age at which client closes the connection| +|redis.pool_timeout|4s|Amount of time client waits for connections are busy before returning an error| +|redis.idle_timeout|5m|Amount of time after which client closes idle connections. Should be less than server timeout| +|redis.idle_check_frequency|1m|Frequency of idle checks made by idle connections reaper| +|redis.tls_ca||Path to CA file| +|redis.tls_cert||Path to cert file| +|redis.tls_key||Path to key file| +|redis.tls_insecure_skip_verify|false|Controls whether a client verifies the server's certificate chain and host name| + +### BoltDB + +|field|default value|description| +|-|-|-| +|bolt.path|"/tmp/boltdb"|The database file path. If the file does not exists then it will be created automatically| +|bolt.timeout|0|The amount of time to wait to obtain a file lock. When set to zero it will wait indefinitely. This option is only available on Darwin and Linux| +|bolt.no_grow_sync|false|Sets DB.NoGrowSync flag before memory mapping the file| +|bolt.read_only|false|Open the database in read-only mode| +|bolt.mmap_flags|0|Set the DB.MmapFlags flag before memory mapping the file| +|bolt.initial_mmap_size|0|The initial mmap size of the database in bytes. If <= 0, the initial map size is 0. If the size is smaller than the previous database, it takes no effect| + +### etcd + +|field|default value|description| +|-|-|-| +|etcd.endpoints|"localhost:2379"|The etcd endpoint(s), multiple endpoints can be passed separated by comma.| +|etcd.dial_timeout|"2s"|The dial timeout duration passed into the client config.| +|etcd.cert_file|""|When using secure etcd, this should point to the crt file.| +|etcd.key_file|""|When using secure etcd, this should point to the pem file.| +|etcd.cacert_file|""|When using secure etcd, this should point to the ca file.| + +### DynamoDB + +|field|default value|description| +|-|-|-| +|dynamodb.tablename|"ycsb"|The database tablename| +|dynamodb.primarykey|"_key"|The table primary key fieldname| +|dynamodb.rc.units|10|Read request units throughput| +|dynamodb.wc.units|10|Write request units throughput| +|dynamodb.ensure.clean.table|true|On load mode ensure that the table is clean at the begining. In case of true and if the table previously exists it will be deleted and recreated| +|dynamodb.endpoint|""|Used endpoint for connection. If empty will use the default loaded configs| +|dynamodb.region|""|Used region for connection ( should match endpoint ). If empty will use the default loaded configs| +|dynamodb.consistent.reads|false|Reads on DynamoDB provide an eventually consistent read by default. If your benchmark/use-case requires a strongly consistent read, set this option to true| +|dynamodb.delete.after.run.stage|false|Detele the database table after the run stage| + + + +## TODO + +- [ ] Support more measurement, like HdrHistogram +- [ ] Add tests for generators diff --git a/go-ycsb/YCSB-LICENSE b/go-ycsb/YCSB-LICENSE new file mode 100644 index 000000000..7a4a3ea24 --- /dev/null +++ b/go-ycsb/YCSB-LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/go-ycsb/cmd/go-ycsb/client.go b/go-ycsb/cmd/go-ycsb/client.go new file mode 100644 index 000000000..22abb5d36 --- /dev/null +++ b/go-ycsb/cmd/go-ycsb/client.go @@ -0,0 +1,111 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "strconv" + "time" + + "github.com/pingcap/go-ycsb/pkg/client" + "github.com/pingcap/go-ycsb/pkg/measurement" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/spf13/cobra" +) + +func runClientCommandFunc(cmd *cobra.Command, args []string, doTransactions bool, command string) { + dbName := args[0] + + initialGlobal(dbName, func() { + doTransFlag := "true" + if !doTransactions { + doTransFlag = "false" + } + globalProps.Set(prop.DoTransactions, doTransFlag) + globalProps.Set(prop.Command, command) + + if cmd.Flags().Changed("threads") { + // We set the threadArg via command line. + globalProps.Set(prop.ThreadCount, strconv.Itoa(threadsArg)) + } + + if cmd.Flags().Changed("target") { + globalProps.Set(prop.Target, strconv.Itoa(targetArg)) + } + + if cmd.Flags().Changed("interval") { + globalProps.Set(prop.LogInterval, strconv.Itoa(reportInterval)) + } + }) + + fmt.Println("***************** properties *****************") + for key, value := range globalProps.Map() { + fmt.Printf("\"%s\"=\"%s\"\n", key, value) + } + fmt.Println("**********************************************") + + c := client.NewClient(globalProps, globalWorkload, globalDB) + start := time.Now() + c.Run(globalContext) + + fmt.Printf("Run finished, takes %s\n", time.Now().Sub(start)) + measurement.Output() +} + +func runLoadCommandFunc(cmd *cobra.Command, args []string) { + runClientCommandFunc(cmd, args, false, "load") +} + +func runTransCommandFunc(cmd *cobra.Command, args []string) { + runClientCommandFunc(cmd, args, true, "run") +} + +var ( + threadsArg int + targetArg int + reportInterval int +) + +func initClientCommand(m *cobra.Command) { + m.Flags().StringSliceVarP(&propertyFiles, "property_file", "P", nil, "Spefify a property file") + m.Flags().StringArrayVarP(&propertyValues, "prop", "p", nil, "Specify a property value with name=value") + m.Flags().StringVar(&tableName, "table", "", "Use the table name instead of the default \""+prop.TableNameDefault+"\"") + m.Flags().IntVar(&threadsArg, "threads", 1, "Execute using n threads - can also be specified as the \"threadcount\" property") + m.Flags().IntVar(&targetArg, "target", 0, "Attempt to do n operations per second (default: unlimited) - can also be specified as the \"target\" property") + m.Flags().IntVar(&reportInterval, "interval", 10, "Interval of outputting measurements in seconds") +} + +func newLoadCommand() *cobra.Command { + m := &cobra.Command{ + Use: "load db", + Short: "YCSB load benchmark", + Args: cobra.MinimumNArgs(1), + Run: runLoadCommandFunc, + } + + initClientCommand(m) + return m +} + +func newRunCommand() *cobra.Command { + m := &cobra.Command{ + Use: "run db", + Short: "YCSB run benchmark", + Args: cobra.MinimumNArgs(1), + Run: runTransCommandFunc, + } + + initClientCommand(m) + return m +} diff --git a/go-ycsb/cmd/go-ycsb/main.go b/go-ycsb/cmd/go-ycsb/main.go new file mode 100644 index 000000000..1bc3aa275 --- /dev/null +++ b/go-ycsb/cmd/go-ycsb/main.go @@ -0,0 +1,200 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "log" + "net/http" + _ "net/http/pprof" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/magiconair/properties" + + // Register workload + + "github.com/spf13/cobra" + + "github.com/pingcap/go-ycsb/pkg/client" + "github.com/pingcap/go-ycsb/pkg/measurement" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + _ "github.com/pingcap/go-ycsb/pkg/workload" + "github.com/pingcap/go-ycsb/pkg/ycsb" + + // Register basic database + _ "github.com/pingcap/go-ycsb/db/basic" + // Register MySQL database + _ "github.com/pingcap/go-ycsb/db/mysql" + // Register TiKV database + //_ "github.com/pingcap/go-ycsb/db/tikv" + // Register PostgreSQL database + _ "github.com/pingcap/go-ycsb/db/pg" + // Register Aerospike database + _ "github.com/pingcap/go-ycsb/db/aerospike" + // Register Badger database + //_ "github.com/pingcap/go-ycsb/db/badger" + // Register FoundationDB database + _ "github.com/pingcap/go-ycsb/db/foundationdb" + // Register RocksDB database + _ "github.com/pingcap/go-ycsb/db/rocksdb" + // Register Spanner database + //_ "github.com/pingcap/go-ycsb/db/spanner" + // Register pegasus database + _ "github.com/pingcap/go-ycsb/db/pegasus" + // Register sqlite database + _ "github.com/pingcap/go-ycsb/db/sqlite" + // Register cassandra database + _ "github.com/pingcap/go-ycsb/db/cassandra" + // Register mongodb database + _ "github.com/pingcap/go-ycsb/db/mongodb" + // Register redis database + _ "github.com/pingcap/go-ycsb/db/redis" + // Register boltdb database + _ "github.com/pingcap/go-ycsb/db/boltdb" + // Register minio + _ "github.com/pingcap/go-ycsb/db/minio" + // Register elastic + _ "github.com/pingcap/go-ycsb/db/elasticsearch" + // Register etcd + //_ "github.com/pingcap/go-ycsb/db/etcd" + // Register tinydb + _ "github.com/pingcap/go-ycsb/db/tinydb" + // Register dynamodb + _ "github.com/pingcap/go-ycsb/db/dynamodb" +) + +var ( + propertyFiles []string + propertyValues []string + dbName string + tableName string + + globalContext context.Context + globalCancel context.CancelFunc + + globalDB ycsb.DB + globalWorkload ycsb.Workload + globalProps *properties.Properties +) + +func initialGlobal(dbName string, onProperties func()) { + globalProps = properties.NewProperties() + if len(propertyFiles) > 0 { + globalProps = properties.MustLoadFiles(propertyFiles, properties.UTF8, false) + } + + for _, prop := range propertyValues { + seps := strings.SplitN(prop, "=", 2) + if len(seps) != 2 { + log.Fatalf("bad property: `%s`, expected format `name=value`", prop) + } + globalProps.Set(seps[0], seps[1]) + } + + if onProperties != nil { + onProperties() + } + + addr := globalProps.GetString(prop.DebugPprof, prop.DebugPprofDefault) + go func() { + http.ListenAndServe(addr, nil) + }() + + measurement.InitMeasure(globalProps) + + if len(tableName) == 0 { + tableName = globalProps.GetString(prop.TableName, prop.TableNameDefault) + } + + workloadName := globalProps.GetString(prop.Workload, "core") + workloadCreator := ycsb.GetWorkloadCreator(workloadName) + + var err error + if globalWorkload, err = workloadCreator.Create(globalProps); err != nil { + util.Fatalf("create workload %s failed %v", workloadName, err) + } + + dbCreator := ycsb.GetDBCreator(dbName) + if dbCreator == nil { + util.Fatalf("%s is not registered", dbName) + } + if globalDB, err = dbCreator.Create(globalProps); err != nil { + util.Fatalf("create db %s failed %v", dbName, err) + } + globalDB = client.DbWrapper{globalDB} +} + +func main() { + globalContext, globalCancel = context.WithCancel(context.Background()) + + sc := make(chan os.Signal, 1) + signal.Notify(sc, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT) + + closeDone := make(chan struct{}, 1) + go func() { + sig := <-sc + fmt.Printf("\nGot signal [%v] to exit.\n", sig) + globalCancel() + + select { + case <-sc: + // send signal again, return directly + fmt.Printf("\nGot signal [%v] again to exit.\n", sig) + os.Exit(1) + case <-time.After(10 * time.Second): + fmt.Print("\nWait 10s for closed, force exit\n") + os.Exit(1) + case <-closeDone: + return + } + }() + + rootCmd := &cobra.Command{ + Use: "go-ycsb", + Short: "Go YCSB", + } + + rootCmd.AddCommand( + newShellCommand(), + newLoadCommand(), + newRunCommand(), + ) + + cobra.EnablePrefixMatching = true + + if err := rootCmd.Execute(); err != nil { + fmt.Println(rootCmd.UsageString()) + } + + globalCancel() + if globalDB != nil { + globalDB.Close() + } + + if globalWorkload != nil { + globalWorkload.Close() + } + + closeDone <- struct{}{} +} diff --git a/go-ycsb/cmd/go-ycsb/shell.go b/go-ycsb/cmd/go-ycsb/shell.go new file mode 100644 index 000000000..ef1fe2a50 --- /dev/null +++ b/go-ycsb/cmd/go-ycsb/shell.go @@ -0,0 +1,247 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io" + "strconv" + "strings" + + "github.com/chzyer/readline" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + + "github.com/spf13/cobra" +) + +func newShellCommand() *cobra.Command { + m := &cobra.Command{ + Use: "shell db", + Short: "YCSB Command Line Client", + Args: cobra.MinimumNArgs(1), + Run: runShellCommandFunc, + } + m.Flags().StringSliceVarP(&propertyFiles, "property_file", "P", nil, "Spefify a property file") + m.Flags().StringSliceVarP(&propertyValues, "prop", "p", nil, "Specify a property value with name=value") + m.Flags().StringVar(&tableName, "table", "", "Use the table name instead of the default \""+prop.TableNameDefault+"\"") + return m +} + +var shellContext context.Context + +func runShellCommandFunc(cmd *cobra.Command, args []string) { + dbName := args[0] + initialGlobal(dbName, nil) + + shellContext = globalWorkload.InitThread(globalContext, 0, 1) + shellContext = globalDB.InitThread(shellContext, 0, 1) + + shellLoop() + + globalDB.CleanupThread(shellContext) + globalWorkload.CleanupThread(shellContext) +} + +func runShellCommand(args []string) { + cmd := &cobra.Command{ + Use: "shell", + Short: "YCSB shell command", + } + + cmd.SetArgs(args) + cmd.ParseFlags(args) + + cmd.AddCommand( + &cobra.Command{ + Use: "read key [field0 field1 field2 ...]", + Short: "Read a record", + Args: cobra.MinimumNArgs(1), + Run: runShellReadCommand, + DisableFlagsInUseLine: true, + }, + &cobra.Command{ + Use: "scan key recordcount [field0 field1 field2 ...]", + Short: "Scan starting at key", + Args: cobra.MinimumNArgs(2), + Run: runShellScanCommand, + DisableFlagsInUseLine: true, + }, + &cobra.Command{ + Use: "insert key field0=value0 [field1=value1 ...]", + Short: "Insert a record", + Args: cobra.MinimumNArgs(2), + Run: runShellInsertCommand, + DisableFlagsInUseLine: true, + }, + &cobra.Command{ + Use: "update key field0=value0 [field1=value1 ...]", + Short: "Update a record", + Args: cobra.MinimumNArgs(2), + Run: runShellUpdateCommand, + DisableFlagsInUseLine: true, + }, + &cobra.Command{ + Use: "delete key", + Short: "Delete a record", + Args: cobra.MinimumNArgs(1), + Run: runShellDeleteCommand, + DisableFlagsInUseLine: true, + }, + &cobra.Command{ + Use: "table [tablename]", + Short: "Get or [set] the name of the table", + Args: cobra.MaximumNArgs(1), + Run: runShellTableCommand, + DisableFlagsInUseLine: true, + }, + ) + + if err := cmd.Execute(); err != nil { + fmt.Println(cmd.UsageString()) + } +} + +func runShellReadCommand(cmd *cobra.Command, args []string) { + key := args[0] + fields := args[1:] + row, err := globalDB.Read(shellContext, tableName, key, fields) + if err != nil { + fmt.Printf("Read %s failed %v\n", key, err) + return + } + + if row == nil { + fmt.Printf("Read empty for %s\n", key) + return + } + + fmt.Printf("Read %s ok\n", key) + for key, value := range row { + fmt.Printf("%s=%q\n", key, value) + } +} + +func runShellScanCommand(cmd *cobra.Command, args []string) { + key := args[0] + recordCount, err := strconv.Atoi(args[1]) + if err != nil { + fmt.Printf("invalid record count %s for scan\n", args[1]) + return + } + fields := args[2:] + + rows, err := globalDB.Scan(shellContext, tableName, key, recordCount, fields) + if err != nil { + fmt.Printf("Scan from %s with %d failed %v\n", key, recordCount, err) + return + } + + if len(rows) == 0 { + fmt.Println("0 records") + return + } + + fmt.Println("--------------------------------") + for i, row := range rows { + fmt.Printf("Record %d\n", i+1) + for key, value := range row { + fmt.Printf("%s=%q\n", key, value) + } + } + fmt.Println("--------------------------------") +} + +func runShellInsertCommand(cmd *cobra.Command, args []string) { + key := args[0] + values := make(map[string][]byte, len(args[1:])) + + for _, arg := range args[1:] { + sep := strings.SplitN(arg, "=", 2) + values[sep[0]] = []byte(sep[1]) + } + + if err := globalDB.Insert(shellContext, tableName, key, values); err != nil { + fmt.Printf("Insert %s failed %v\n", key, err) + return + } + + fmt.Printf("Insert %s ok\n", key) +} + +func runShellUpdateCommand(cmd *cobra.Command, args []string) { + key := args[0] + values := make(map[string][]byte, len(args[1:])) + + for _, arg := range args[1:] { + sep := strings.SplitN(arg, "=", 2) + values[sep[0]] = []byte(sep[1]) + } + + if err := globalDB.Update(shellContext, tableName, key, values); err != nil { + fmt.Printf("Update %s failed %v\n", key, err) + return + } + + fmt.Printf("Update %s ok\n", key) +} + +func runShellDeleteCommand(cmd *cobra.Command, args []string) { + key := args[0] + if err := globalDB.Delete(shellContext, tableName, key); err != nil { + fmt.Printf("Delete %s failed %v\n", key, err) + return + } + + fmt.Printf("Delete %s ok\n", key) +} + +func runShellTableCommand(cmd *cobra.Command, args []string) { + if len(args) == 1 { + tableName = args[0] + } + fmt.Printf("Using table %s\n", tableName) +} + +func shellLoop() { + l, err := readline.NewEx(&readline.Config{ + Prompt: "\033[31m»\033[0m ", + HistoryFile: "/tmp/readline.tmp", + InterruptPrompt: "^C", + EOFPrompt: "^D", + HistorySearchFold: true, + }) + if err != nil { + util.Fatal(err) + } + defer l.Close() + + for { + line, err := l.Readline() + if err != nil { + if err == readline.ErrInterrupt { + return + } else if err == io.EOF { + return + } + continue + } + if line == "exit" { + return + } + args := strings.Split(strings.TrimSpace(line), " ") + runShellCommand(args) + } +} diff --git a/go-ycsb/db/aerospike/db.go b/go-ycsb/db/aerospike/db.go new file mode 100644 index 000000000..cc8cd4e5f --- /dev/null +++ b/go-ycsb/db/aerospike/db.go @@ -0,0 +1,176 @@ +package aerospike + +import ( + "context" + "errors" + + as "github.com/aerospike/aerospike-client-go" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +const ( + asNs = "aerospike.ns" + asHost = "aerospike.host" + asPort = "aerospike.port" +) + +type aerospikedb struct { + client *as.Client + ns string +} + +// Close closes the database layer. +func (adb *aerospikedb) Close() error { + adb.client.Close() + return nil +} + +// InitThread initializes the state associated to the goroutine worker. +// The Returned context will be passed to the following usage. +func (adb *aerospikedb) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { + return ctx +} + +// CleanupThread cleans up the state when the worker finished. +func (adb *aerospikedb) CleanupThread(ctx context.Context) { +} + +// Read reads a record from the database and returns a map of each field/value pair. +// table: The name of the table. +// key: The record key of the record to read. +// fileds: The list of fields to read, nil|empty for reading all. +func (adb *aerospikedb) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + asKey, err := as.NewKey(adb.ns, table, key) + record, err := adb.client.Get(nil, asKey) + if err != nil { + return nil, err + } + if record == nil { + return map[string][]byte{}, nil + } + res := make(map[string][]byte, len(record.Bins)) + var ok bool + for k, v := range record.Bins { + res[k], ok = v.([]byte) + if !ok { + return nil, errors.New("couldn't convert to byte array") + } + } + return res, nil +} + +// Scan scans records from the database. +// table: The name of the table. +// startKey: The first record key to read. +// count: The number of records to read. +// fields: The list of fields to read, nil|empty for reading all. +func (adb *aerospikedb) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + policy := as.NewScanPolicy() + policy.ConcurrentNodes = true + recordset, err := adb.client.ScanAll(policy, adb.ns, table) + if err != nil { + return nil, err + } + filter := make(map[string]bool, len(fields)) + for _, field := range fields { + filter[field] = true + } + scanRes := make([]map[string][]byte, 0) + var ok bool + nRead := 0 + for res := range recordset.Results() { + if res.Err != nil { + recordset.Close() + return nil, res.Err + } + vals := make(map[string][]byte, len(res.Record.Bins)) + for k, v := range res.Record.Bins { + if !filter[k] { + continue + } + vals[k], ok = v.([]byte) + if !ok { + return nil, errors.New("couldn't convert to byte array") + } + } + scanRes = append(scanRes, vals) + nRead++ + if nRead == count { + break + } + } + return scanRes, nil +} + +// Update updates a record in the database. Any field/value pairs will be written into the +// database or overwritten the existing values with the same field name. +// table: The name of the table. +// key: The record key of the record to update. +// values: A map of field/value pairs to update in the record. +func (adb *aerospikedb) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + asKey, err := as.NewKey(adb.ns, table, key) + if err != nil { + return err + } + record, err := adb.client.Get(nil, asKey) + if err != nil { + return err + } + bins := as.BinMap{} + var policy *as.WritePolicy + if record != nil { + bins = record.Bins + policy := as.NewWritePolicy(record.Generation, 0) + policy.GenerationPolicy = as.EXPECT_GEN_EQUAL + } + for k, v := range values { + bins[k] = v + } + return adb.client.Put(policy, asKey, bins) +} + +// Insert inserts a record in the database. Any field/value pairs will be written into the +// database. +// table: The name of the table. +// key: The record key of the record to insert. +// values: A map of field/value pairs to insert in the record. +func (adb *aerospikedb) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + asKey, err := as.NewKey(adb.ns, table, key) + if err != nil { + return err + } + bins := make([]*as.Bin, len(values)) + i := 0 + for k, v := range values { + bins[i] = as.NewBin(k, v) + i++ + } + return adb.client.PutBins(nil, asKey, bins...) +} + +// Delete deletes a record from the database. +// table: The name of the table. +// key: The record key of the record to delete. +func (adb *aerospikedb) Delete(ctx context.Context, table string, key string) error { + asKey, err := as.NewKey(adb.ns, table, key) + if err != nil { + return err + } + _, err = adb.client.Delete(nil, asKey) + return err +} + +type aerospikeCreator struct{} + +func (a aerospikeCreator) Create(p *properties.Properties) (ycsb.DB, error) { + adb := &aerospikedb{} + adb.ns = p.GetString(asNs, "test") + var err error + adb.client, err = as.NewClient(p.GetString(asHost, "localhost"), p.GetInt(asPort, 3000)) + return adb, err +} + +func init() { + ycsb.RegisterDBCreator("aerospike", aerospikeCreator{}) +} diff --git a/go-ycsb/db/badger/db.go b/go-ycsb/db/badger/db.go new file mode 100644 index 000000000..6599141af --- /dev/null +++ b/go-ycsb/db/badger/db.go @@ -0,0 +1,263 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package badger + +import ( + "context" + "fmt" + "os" + + "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/options" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// properties +const ( + badgerDir = "badger.dir" + badgerValueDir = "badger.valuedir" + badgerSyncWrites = "badger.sync_writes" + badgerNumVersionsToKeep = "badger.num_versions_to_keep" + badgerMaxTableSize = "badger.max_table_size" + badgerLevelSizeMultiplier = "badger.level_size_multiplier" + badgerMaxLevels = "badger.max_levels" + badgerValueThreshold = "badger.value_threshold" + badgerNumMemtables = "badger.num_memtables" + badgerNumLevelZeroTables = "badger.num_level0_tables" + badgerNumLevelZeroTablesStall = "badger.num_level0_tables_stall" + badgerLevelOneSize = "badger.level_one_size" + badgerValueLogFileSize = "badger.value_log_file_size" + badgerValueLogMaxEntries = "badger.value_log_max_entries" + badgerNumCompactors = "badger.num_compactors" + badgerDoNotCompact = "badger.do_not_compact" + badgerTableLoadingMode = "badger.table_loading_mode" + badgerValueLogLoadingMode = "badger.value_log_loading_mode" + // TODO: add more configurations +) + +type badgerCreator struct { +} + +type badgerDB struct { + p *properties.Properties + + db *badger.DB + + r *util.RowCodec + bufPool *util.BufPool +} + +type contextKey string + +const stateKey = contextKey("badgerDB") + +type badgerState struct { +} + +func (c badgerCreator) Create(p *properties.Properties) (ycsb.DB, error) { + opts := getOptions(p) + + if p.GetBool(prop.DropData, prop.DropDataDefault) { + os.RemoveAll(opts.Dir) + os.RemoveAll(opts.ValueDir) + } + + db, err := badger.Open(opts) + if err != nil { + return nil, err + } + + return &badgerDB{ + p: p, + db: db, + r: util.NewRowCodec(p), + bufPool: util.NewBufPool(), + }, nil +} + +func getOptions(p *properties.Properties) badger.Options { + opts := badger.DefaultOptions + opts.Dir = p.GetString(badgerDir, "/tmp/badger") + opts.ValueDir = p.GetString(badgerValueDir, opts.Dir) + + opts.SyncWrites = p.GetBool(badgerSyncWrites, false) + opts.NumVersionsToKeep = p.GetInt(badgerNumVersionsToKeep, 1) + opts.MaxTableSize = p.GetInt64(badgerMaxTableSize, 64<<20) + opts.LevelSizeMultiplier = p.GetInt(badgerLevelSizeMultiplier, 10) + opts.MaxLevels = p.GetInt(badgerMaxLevels, 7) + opts.ValueThreshold = p.GetInt(badgerValueThreshold, 32) + opts.NumMemtables = p.GetInt(badgerNumMemtables, 5) + opts.NumLevelZeroTables = p.GetInt(badgerNumLevelZeroTables, 5) + opts.NumLevelZeroTablesStall = p.GetInt(badgerNumLevelZeroTablesStall, 10) + opts.LevelOneSize = p.GetInt64(badgerLevelOneSize, 256<<20) + opts.ValueLogFileSize = p.GetInt64(badgerValueLogFileSize, 1<<30) + opts.ValueLogMaxEntries = uint32(p.GetUint64(badgerValueLogMaxEntries, 1000000)) + opts.NumCompactors = p.GetInt(badgerNumCompactors, 3) + opts.DoNotCompact = p.GetBool(badgerDoNotCompact, false) + if b := p.GetString(badgerTableLoadingMode, "LoadToRAM"); len(b) > 0 { + if b == "FileIO" { + opts.TableLoadingMode = options.FileIO + } else if b == "LoadToRAM" { + opts.TableLoadingMode = options.LoadToRAM + } else if b == "MemoryMap" { + opts.TableLoadingMode = options.MemoryMap + } + } + if b := p.GetString(badgerValueLogLoadingMode, "MemoryMap"); len(b) > 0 { + if b == "FileIO" { + opts.ValueLogLoadingMode = options.FileIO + } else if b == "LoadToRAM" { + opts.ValueLogLoadingMode = options.LoadToRAM + } else if b == "MemoryMap" { + opts.ValueLogLoadingMode = options.MemoryMap + } + } + + return opts +} + +func (db *badgerDB) Close() error { + return db.db.Close() +} + +func (db *badgerDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (db *badgerDB) CleanupThread(_ context.Context) { +} + +func (db *badgerDB) getRowKey(table string, key string) []byte { + return util.Slice(fmt.Sprintf("%s:%s", table, key)) +} + +func (db *badgerDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + var m map[string][]byte + err := db.db.View(func(txn *badger.Txn) error { + rowKey := db.getRowKey(table, key) + item, err := txn.Get(rowKey) + if err != nil { + return err + } + row, err := item.Value() + if err != nil { + return err + } + + m, err = db.r.Decode(row, fields) + return err + }) + + return m, err +} + +func (db *badgerDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + res := make([]map[string][]byte, count) + err := db.db.View(func(txn *badger.Txn) error { + rowStartKey := db.getRowKey(table, startKey) + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + + i := 0 + for it.Seek(rowStartKey); it.Valid() && i < count; it.Next() { + item := it.Item() + value, err := item.ValueCopy(nil) + if err != nil { + return err + } + + m, err := db.r.Decode(value, fields) + if err != nil { + return err + } + + res[i] = m + i++ + } + + return nil + }) + + return res, err +} + +func (db *badgerDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + err := db.db.Update(func(txn *badger.Txn) error { + rowKey := db.getRowKey(table, key) + item, err := txn.Get(rowKey) + if err != nil { + return err + } + + value, err := item.Value() + if err != nil { + return err + } + + data, err := db.r.Decode(value, nil) + if err != nil { + return err + } + + for field, value := range values { + data[field] = value + } + + buf := db.bufPool.Get() + defer func() { + db.bufPool.Put(buf) + }() + + buf, err = db.r.Encode(buf, data) + if err != nil { + return err + } + return txn.Set(rowKey, buf) + }) + return err +} + +func (db *badgerDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + err := db.db.Update(func(txn *badger.Txn) error { + rowKey := db.getRowKey(table, key) + + buf := db.bufPool.Get() + defer func() { + db.bufPool.Put(buf) + }() + + buf, err := db.r.Encode(buf, values) + if err != nil { + return err + } + return txn.Set(rowKey, buf) + }) + + return err +} + +func (db *badgerDB) Delete(ctx context.Context, table string, key string) error { + err := db.db.Update(func(txn *badger.Txn) error { + return txn.Delete(db.getRowKey(table, key)) + }) + + return err +} + +func init() { + ycsb.RegisterDBCreator("badger", badgerCreator{}) +} diff --git a/go-ycsb/db/basic/db.go b/go-ycsb/db/basic/db.go new file mode 100644 index 000000000..9d1ec192f --- /dev/null +++ b/go-ycsb/db/basic/db.go @@ -0,0 +1,272 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package basic + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "time" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +const ( + simulateDelay = "basicdb.simulatedelay" + simulateDelayDefault = int64(0) + randomizeDelay = "basicdb.randomizedelay" + randomizeDelayDefault = true +) + +type contextKey string + +const stateKey = contextKey("basicDB") + +type basicState struct { + r *rand.Rand + + buf *bytes.Buffer +} + +// BasicDB just prints out the requested operations, instead of doing them against a database +type basicDB struct { + verbose bool + randomizeDelay bool + toDelay int64 +} + +func (db *basicDB) delay(ctx context.Context, state *basicState) { + if db.toDelay == 0 { + return + } + + r := state.r + delayTime := time.Duration(db.toDelay) * time.Millisecond + if db.randomizeDelay { + delayTime = time.Duration(r.Int63n(db.toDelay)) * time.Millisecond + if delayTime == 0 { + return + } + } + + select { + case <-time.After(delayTime): + case <-ctx.Done(): + } +} + +func (db *basicDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + state := new(basicState) + state.r = rand.New(rand.NewSource(time.Now().UnixNano())) + state.buf = new(bytes.Buffer) + + return context.WithValue(ctx, stateKey, state) +} + +func (db *basicDB) CleanupThread(_ context.Context) { + +} + +func (db *basicDB) Close() error { + return nil +} + +func (db *basicDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + state := ctx.Value(stateKey).(*basicState) + + db.delay(ctx, state) + + if !db.verbose { + return nil, nil + } + + buf := state.buf + s := fmt.Sprintf("READ %s %s [ ", table, key) + buf.WriteString(s) + + if len(fields) > 0 { + for _, f := range fields { + buf.WriteString(f) + buf.WriteByte(' ') + } + } else { + buf.WriteString(" ") + } + buf.WriteByte(']') + fmt.Println(buf.String()) + buf.Reset() + return nil, nil +} + +func (db *basicDB) BatchRead(ctx context.Context, table string, keys []string, fields []string) ([]map[string][]byte, error) { + panic("The basicDB has not implemented the batch operation") +} + +func (db *basicDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + state := ctx.Value(stateKey).(*basicState) + + db.delay(ctx, state) + + if !db.verbose { + return nil, nil + } + + buf := state.buf + s := fmt.Sprintf("SCAN %s %s %d [ ", table, startKey, count) + buf.WriteString(s) + + if len(fields) > 0 { + for _, f := range fields { + buf.WriteString(f) + buf.WriteByte(' ') + } + } else { + buf.WriteString(" ") + } + buf.WriteByte(']') + fmt.Println(buf.String()) + buf.Reset() + return nil, nil +} + +func (db *basicDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + state := ctx.Value(stateKey).(*basicState) + + db.delay(ctx, state) + + if !db.verbose { + return nil + } + + buf := state.buf + s := fmt.Sprintf("UPDATE %s %s [ ", table, key) + buf.WriteString(s) + + for key, value := range values { + buf.WriteString(key) + buf.WriteByte('=') + buf.Write(value) + buf.WriteByte(' ') + } + + buf.WriteByte(']') + fmt.Println(buf.String()) + buf.Reset() + return nil +} + +func (db *basicDB) BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) error { + panic("The basicDB has not implemented the batch operation") +} + +func (db *basicDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + state := ctx.Value(stateKey).(*basicState) + + db.delay(ctx, state) + + if !db.verbose { + return nil + } + + buf := state.buf + insertRecord(buf, table, key, values) + return nil +} + +func (db *basicDB) BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) error { + state := ctx.Value(stateKey).(*basicState) + + db.delay(ctx, state) + + if !db.verbose { + return nil + } + buf := state.buf + for i, key := range keys { + insertRecord(buf, table, key, values[i]) + } + return nil +} + +func insertRecord(buf *bytes.Buffer, table string, key string, values map[string][]byte) { + s := fmt.Sprintf("INSERT %s %s [ ", table, key) + buf.WriteString(s) + for valueKey, value := range values { + buf.WriteString(valueKey) + buf.WriteByte('=') + buf.Write(value) + buf.WriteByte(' ') + } + buf.WriteByte(']') + fmt.Println(buf.String()) + buf.Reset() +} + +func (db *basicDB) Delete(ctx context.Context, table string, key string) error { + state := ctx.Value(stateKey).(*basicState) + + db.delay(ctx, state) + if !db.verbose { + return nil + } + + buf := state.buf + s := fmt.Sprintf("DELETE %s %s", table, key) + buf.WriteString(s) + + fmt.Println(buf.String()) + buf.Reset() + return nil +} + +func (db *basicDB) BatchDelete(ctx context.Context, table string, keys []string) error { + panic("The basicDB has not implemented the batch operation") +} + +type basicDBCreator struct{} + +func (basicDBCreator) Create(p *properties.Properties) (ycsb.DB, error) { + db := new(basicDB) + + db.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) + db.randomizeDelay = p.GetBool(randomizeDelay, randomizeDelayDefault) + db.toDelay = p.GetInt64(simulateDelay, simulateDelayDefault) + + return db, nil +} + +func init() { + fmt.Println("basic ok") + ycsb.RegisterDBCreator("basic", basicDBCreator{}) +} diff --git a/go-ycsb/db/boltdb/db.go b/go-ycsb/db/boltdb/db.go new file mode 100644 index 000000000..278261226 --- /dev/null +++ b/go-ycsb/db/boltdb/db.go @@ -0,0 +1,244 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package boltdb + +import ( + "context" + "fmt" + "os" + + "github.com/boltdb/bolt" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// properties +const ( + boltPath = "bolt.path" + boltTimeout = "bolt.timeout" + boltNoGrowSync = "bolt.no_grow_sync" + boltReadOnly = "bolt.read_only" + boltMmapFlags = "bolt.mmap_flags" + boltInitialMmapSize = "bolt.initial_mmap_size" +) + +type boltCreator struct { +} + +type boltOptions struct { + Path string + FileMode os.FileMode + DBOptions *bolt.Options +} + +type boltDB struct { + p *properties.Properties + + db *bolt.DB + + r *util.RowCodec + bufPool *util.BufPool +} + +func (c boltCreator) Create(p *properties.Properties) (ycsb.DB, error) { + opts := getOptions(p) + + if p.GetBool(prop.DropData, prop.DropDataDefault) { + os.RemoveAll(opts.Path) + } + + db, err := bolt.Open(opts.Path, opts.FileMode, opts.DBOptions) + if err != nil { + return nil, err + } + + return &boltDB{ + p: p, + db: db, + r: util.NewRowCodec(p), + bufPool: util.NewBufPool(), + }, nil +} + +func getOptions(p *properties.Properties) boltOptions { + path := p.GetString(boltPath, "/tmp/boltdb") + + opts := bolt.DefaultOptions + opts.Timeout = p.GetDuration(boltTimeout, 0) + opts.NoGrowSync = p.GetBool(boltNoGrowSync, false) + opts.ReadOnly = p.GetBool(boltReadOnly, false) + opts.MmapFlags = p.GetInt(boltMmapFlags, 0) + opts.InitialMmapSize = p.GetInt(boltInitialMmapSize, 0) + + return boltOptions{ + Path: path, + FileMode: 0600, + DBOptions: opts, + } +} + +func (db *boltDB) Close() error { + return db.db.Close() +} + +func (db *boltDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (db *boltDB) CleanupThread(_ context.Context) { +} + +func (db *boltDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + var m map[string][]byte + err := db.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(table)) + if bucket == nil { + return fmt.Errorf("table not found: %s", table) + } + + row := bucket.Get([]byte(key)) + if row == nil { + return fmt.Errorf("key not found: %s.%s", table, key) + } + + var err error + m, err = db.r.Decode(row, fields) + return err + }) + return m, err +} + +func (db *boltDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + res := make([]map[string][]byte, count) + err := db.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(table)) + if bucket == nil { + return fmt.Errorf("table not found: %s", table) + } + + cursor := bucket.Cursor() + key, value := cursor.Seek([]byte(startKey)) + for i := 0; key != nil && i < count; i++ { + m, err := db.r.Decode(value, fields) + if err != nil { + return err + } + + res[i] = m + key, value = cursor.Next() + } + + return nil + }) + return res, err +} + +func (db *boltDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + err := db.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(table)) + if bucket == nil { + return fmt.Errorf("table not found: %s", table) + } + + value := bucket.Get([]byte(key)) + if value == nil { + return fmt.Errorf("key not found: %s.%s", table, key) + } + + data, err := db.r.Decode(value, nil) + if err != nil { + return err + } + + for field, value := range values { + data[field] = value + } + + buf := db.bufPool.Get() + defer func() { + db.bufPool.Put(buf) + }() + + buf, err = db.r.Encode(buf, data) + if err != nil { + return err + } + + return bucket.Put([]byte(key), buf) + }) + return err +} + +func (db *boltDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + err := db.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(table)) + if err != nil { + return err + } + + buf := db.bufPool.Get() + defer func() { + db.bufPool.Put(buf) + }() + + buf, err = db.r.Encode(buf, values) + if err != nil { + return err + } + + return bucket.Put([]byte(key), buf) + }) + return err +} + +func (db *boltDB) Delete(ctx context.Context, table string, key string) error { + err := db.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(table)) + if bucket == nil { + return nil + } + + err := bucket.Delete([]byte(key)) + if err != nil { + return err + } + + if bucket.Stats().KeyN == 0 { + _ = tx.DeleteBucket([]byte(table)) + } + return nil + }) + return err +} + +func init() { + ycsb.RegisterDBCreator("boltdb", boltCreator{}) +} diff --git a/go-ycsb/db/cassandra/db.go b/go-ycsb/db/cassandra/db.go new file mode 100644 index 000000000..cc9cd6352 --- /dev/null +++ b/go-ycsb/db/cassandra/db.go @@ -0,0 +1,268 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cassandra + +import ( + "bytes" + "context" + "fmt" + "strings" + "time" + + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + + "github.com/gocql/gocql" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// cassandra properties +const ( + cassandraCluster = "cassandra.cluster" + cassandraKeyspace = "cassandra.keyspace" + cassandraConnections = "cassandra.connections" + cassandraUsername = "cassandra.username" + cassandraPassword = "cassandra.password" + + cassandraUsernameDefault = "cassandra" + cassandraPasswordDefault = "cassandra" + cassandraClusterDefault = "127.0.0.1:9042" + cassandraKeyspaceDefault = "test" + cassandraConnectionsDefault = 2 // refer to https://github.com/gocql/gocql/blob/master/cluster.go#L52 +) + +type cassandraCreator struct { +} + +type cassandraDB struct { + p *properties.Properties + session *gocql.Session + verbose bool + + bufPool *util.BufPool + keySpace string + + fieldNames []string +} + +type contextKey string + +const stateKey = contextKey("cassandraDB") + +type cassandraState struct { +} + +func (c cassandraCreator) Create(p *properties.Properties) (ycsb.DB, error) { + d := new(cassandraDB) + d.p = p + + hosts := strings.Split(p.GetString(cassandraCluster, cassandraClusterDefault), ",") + + cluster := gocql.NewCluster(hosts...) + cluster.Keyspace = p.GetString(cassandraKeyspace, cassandraKeyspaceDefault) + d.keySpace = cluster.Keyspace + + cluster.NumConns = p.GetInt(cassandraConnections, cassandraConnectionsDefault) + cluster.Timeout = 30 * time.Second + cluster.Consistency = gocql.Quorum + + username := p.GetString(cassandraUsername, cassandraUsernameDefault) + password := p.GetString(cassandraPassword, cassandraPasswordDefault) + cluster.Authenticator = gocql.PasswordAuthenticator{Username: username, Password: password} + + session, err := cluster.CreateSession() + if err != nil { + return nil, err + } + + d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) + d.session = session + + d.bufPool = util.NewBufPool() + + if err := d.createTable(); err != nil { + return nil, err + } + + return d, nil +} + +func (db *cassandraDB) createTable() error { + tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) + + if db.p.GetBool(prop.DropData, prop.DropDataDefault) { + if err := db.session.Query(fmt.Sprintf("DROP TABLE IF EXISTS %s.%s", db.keySpace, tableName)).Exec(); err != nil { + return err + } + } + + fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) + + db.fieldNames = make([]string, fieldCount) + for i := int64(0); i < fieldCount; i++ { + db.fieldNames[i] = fmt.Sprintf("field%d", i) + } + + buf := new(bytes.Buffer) + s := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s.%s (YCSB_KEY VARCHAR PRIMARY KEY", db.keySpace, tableName) + buf.WriteString(s) + + for i := int64(0); i < fieldCount; i++ { + buf.WriteString(fmt.Sprintf(", FIELD%d VARCHAR", i)) + } + + buf.WriteString(");") + + if db.verbose { + fmt.Println(buf.String()) + } + + err := db.session.Query(buf.String()).Exec() + return err +} + +func (db *cassandraDB) Close() error { + if db.session == nil { + return nil + } + + db.session.Close() + return nil +} + +func (db *cassandraDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (db *cassandraDB) CleanupThread(_ctx context.Context) { + +} + +func (db *cassandraDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + var query string + if len(fields) == 0 { + fields = db.fieldNames + } + + query = fmt.Sprintf(`SELECT %s FROM %s.%s WHERE YCSB_KEY = ?`, strings.Join(fields, ","), db.keySpace, table) + + if db.verbose { + fmt.Printf("%s\n", query) + } + + m := make(map[string][]byte, len(fields)) + dest := make([]interface{}, len(fields)) + for i := 0; i < len(fields); i++ { + v := new([]byte) + dest[i] = v + } + + err := db.session.Query(query, key).WithContext(ctx).Scan(dest...) + if err == gocql.ErrNotFound { + return nil, nil + } else if err != nil { + return nil, err + } + + for i, v := range dest { + m[fields[i]] = *v.(*[]byte) + } + + return m, nil +} + +func (db *cassandraDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + return nil, fmt.Errorf("scan is not supported") +} + +func (db *cassandraDB) execQuery(ctx context.Context, query string, args ...interface{}) error { + if db.verbose { + fmt.Printf("%s %v\n", query, args) + } + + err := db.session.Query(query, args...).WithContext(ctx).Exec() + return err +} + +func (db *cassandraDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + buf := bytes.NewBuffer(db.bufPool.Get()) + defer func() { + db.bufPool.Put(buf.Bytes()) + }() + + buf.WriteString("UPDATE ") + buf.WriteString(fmt.Sprintf("%s.%s", db.keySpace, table)) + buf.WriteString(" SET ") + firstField := true + pairs := util.NewFieldPairs(values) + args := make([]interface{}, 0, len(values)+1) + for _, p := range pairs { + if firstField { + firstField = false + } else { + buf.WriteString(", ") + } + + buf.WriteString(p.Field) + buf.WriteString(`= ?`) + args = append(args, p.Value) + } + buf.WriteString(" WHERE YCSB_KEY = ?") + + args = append(args, key) + + return db.execQuery(ctx, buf.String(), args...) +} + +func (db *cassandraDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + args := make([]interface{}, 0, 1+len(values)) + args = append(args, key) + + buf := bytes.NewBuffer(db.bufPool.Get()) + defer func() { + db.bufPool.Put(buf.Bytes()) + }() + + buf.WriteString("INSERT INTO ") + buf.WriteString(fmt.Sprintf("%s.%s", db.keySpace, table)) + buf.WriteString(" (YCSB_KEY") + + pairs := util.NewFieldPairs(values) + for _, p := range pairs { + args = append(args, p.Value) + buf.WriteString(" ,") + buf.WriteString(p.Field) + } + buf.WriteString(") VALUES (?") + + for i := 0; i < len(pairs); i++ { + buf.WriteString(" ,?") + } + + buf.WriteByte(')') + + return db.execQuery(ctx, buf.String(), args...) +} + +func (db *cassandraDB) Delete(ctx context.Context, table string, key string) error { + query := fmt.Sprintf(`DELETE FROM %s.%s WHERE YCSB_KEY = ?`, db.keySpace, table) + + return db.execQuery(ctx, query, key) +} + +func init() { + ycsb.RegisterDBCreator("cassandra", cassandraCreator{}) + ycsb.RegisterDBCreator("scylla", cassandraCreator{}) +} diff --git a/go-ycsb/db/dynamodb/db.go b/go-ycsb/db/dynamodb/db.go new file mode 100644 index 000000000..16e604bc2 --- /dev/null +++ b/go-ycsb/db/dynamodb/db.go @@ -0,0 +1,295 @@ +package dynamodb + +import ( + "context" + "errors" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/ycsb" + "log" + "strings" + "time" +) + +type dynamodbWrapper struct { + client *dynamodb.Client + tablename *string + primarykey string + primarykeyPtr *string + readCapacityUnits int64 + writeCapacityUnits int64 + consistentRead bool + deleteAfterRun bool + command string +} + +func (r *dynamodbWrapper) Close() error { + var err error = nil + if strings.Compare("run", r.command) == 0 { + log.Printf("Ensuring that the table is deleted after the run stage...\n") + if r.deleteAfterRun { + err = r.deleteTable() + if err != nil { + log.Printf("Couldn't delete table after run. Here's why: %v\n", err) + } + } + } + return err +} + +func (r *dynamodbWrapper) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (r *dynamodbWrapper) CleanupThread(_ context.Context) { +} + +func (r *dynamodbWrapper) Read(ctx context.Context, table string, key string, fields []string) (data map[string][]byte, err error) { + data = make(map[string][]byte, len(fields)) + + response, err := r.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + Key: r.GetKey(key), + TableName: r.tablename, + ConsistentRead: aws.Bool(r.consistentRead), + }) + if err != nil { + log.Printf("Couldn't get info about %v. Here's why: %v\n", key, err) + } else { + err = attributevalue.UnmarshalMap(response.Item, &data) + if err != nil { + log.Printf("Couldn't unmarshal response. Here's why: %v\n", err) + } + } + return + +} + +// GetKey returns the composite primary key of the document in a format that can be +// sent to DynamoDB. +func (r *dynamodbWrapper) GetKey(key string) map[string]types.AttributeValue { + return map[string]types.AttributeValue{ + r.primarykey: &types.AttributeValueMemberB{Value: []byte(key)}, + } +} + +func (r *dynamodbWrapper) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + return nil, fmt.Errorf("scan is not supported") +} + +func (r *dynamodbWrapper) Update(ctx context.Context, table string, key string, values map[string][]byte) (err error) { + var upd = expression.UpdateBuilder{} + for name, value := range values { + upd = upd.Set(expression.Name(name), expression.Value(&types.AttributeValueMemberB{Value: value})) + } + expr, err := expression.NewBuilder().WithUpdate(upd).Build() + + _, err = r.client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + Key: r.GetKey(key), + TableName: r.tablename, + UpdateExpression: expr.Update(), + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + }) + if err != nil { + log.Printf("Couldn't update item to table. Here's why: %v\nUpdateExpression:%s\nExpressionAttributeNames:%s\n", err, *expr.Update(), expr.Names()) + } + return +} + +func (r *dynamodbWrapper) Insert(ctx context.Context, table string, key string, values map[string][]byte) (err error) { + values[r.primarykey] = []byte(key) + item, err := attributevalue.MarshalMap(values) + if err != nil { + panic(err) + } + _, err = r.client.PutItem(context.TODO(), + &dynamodb.PutItemInput{ + TableName: r.tablename, Item: item, + }) + if err != nil { + log.Printf("Couldn't add item to table. Here's why: %v\n", err) + } + return +} + +func (r *dynamodbWrapper) Delete(ctx context.Context, table string, key string) error { + _, err := r.client.DeleteItem(context.TODO(), &dynamodb.DeleteItemInput{ + TableName: r.tablename, + Key: r.GetKey(key), + }) + return err +} + +type dynamoDbCreator struct{} + +// TableExists determines whether a DynamoDB table exists. +func (r *dynamodbWrapper) tableExists() (bool, error) { + exists := true + _, err := r.client.DescribeTable( + context.TODO(), &dynamodb.DescribeTableInput{TableName: r.tablename}, + ) + if err != nil { + var notFoundEx *types.ResourceNotFoundException + if errors.As(err, ¬FoundEx) { + log.Printf("Table %v does not exist.\n", *r.tablename) + err = nil + } else { + log.Printf("Couldn't determine existence of table %v. Here's why: %v\n", *r.tablename, err) + } + exists = false + } + return exists, err +} + +// This function uses NewTableExistsWaiter to wait for the table to be created by +// DynamoDB before it returns. +func (r *dynamodbWrapper) createTable() (*types.TableDescription, error) { + var tableDesc *types.TableDescription + table, err := r.client.CreateTable(context.TODO(), &dynamodb.CreateTableInput{ + AttributeDefinitions: []types.AttributeDefinition{{ + AttributeName: r.primarykeyPtr, + AttributeType: types.ScalarAttributeTypeB, + }}, + KeySchema: []types.KeySchemaElement{ + { + AttributeName: r.primarykeyPtr, + KeyType: types.KeyTypeHash, + }, + }, + TableName: r.tablename, + ProvisionedThroughput: &types.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(r.readCapacityUnits), + WriteCapacityUnits: aws.Int64(r.writeCapacityUnits), + }, + }) + if err != nil { + log.Printf("Couldn't create table %v. Here's why: %v\n", *r.tablename, err) + } else { + log.Printf("Waiting for table to be available.\n") + waiter := dynamodb.NewTableExistsWaiter(r.client) + err = waiter.Wait(context.TODO(), &dynamodb.DescribeTableInput{ + TableName: r.tablename}, 5*time.Minute) + if err != nil { + log.Printf("Wait for table exists failed. Here's why: %v\n", err) + } + tableDesc = table.TableDescription + } + return tableDesc, err +} + +func (r dynamoDbCreator) Create(p *properties.Properties) (ycsb.DB, error) { + rds := &dynamodbWrapper{} + + rds.tablename = aws.String(p.GetString(tablename, tablenameDefault)) + // other than the primary key, you do not need to define + // any extra attributes or data types when you create a table. + rds.primarykey = p.GetString(primaryKeyFieldName, primaryKeyFieldNameDefault) + rds.primarykeyPtr = aws.String(rds.primarykey) + rds.readCapacityUnits = p.GetInt64(readCapacityUnitsFieldName, readCapacityUnitsFieldNameDefault) + rds.writeCapacityUnits = p.GetInt64(writeCapacityUnitsFieldName, writeCapacityUnitsFieldNameDefault) + rds.consistentRead = p.GetBool(consistentReadFieldName, consistentReadFieldNameDefault) + rds.deleteAfterRun = p.GetBool(deleteTableAfterRunFieldName, deleteTableAfterRunFieldNameDefault) + endpoint := p.GetString(endpointField, endpointFieldDefault) + region := p.GetString(regionField, regionFieldDefault) + rds.command, _ = p.Get(prop.Command) + var err error = nil + var cfg aws.Config + if strings.Contains(endpoint, "localhost") && strings.Compare(region, "localhost") != 0 { + log.Printf("given you're using dynamodb local endpoint you need to specify -p %s='localhost'. Ignoring %s and enforcing -p %s='localhost'\n", regionField, region, regionField) + region = "localhost" + } + if strings.Compare(endpoint, endpointFieldDefault) == 0 { + if strings.Compare(region, regionFieldDefault) != 0 { + // if endpoint is default but we have region + cfg, err = config.LoadDefaultConfig(context.TODO(), config.WithRegion(region)) + } else { + // if both endpoint and region are default + cfg, err = config.LoadDefaultConfig(context.TODO()) + } + } else { + cfg, err = config.LoadDefaultConfig(context.TODO(), + config.WithRegion(region), + config.WithEndpointResolver(aws.EndpointResolverFunc( + func(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{URL: endpoint, SigningRegion: region}, nil + })), + ) + } + if err != nil { + log.Fatalf("unable to load SDK config, %v", err) + } + // Create DynamoDB client + rds.client = dynamodb.NewFromConfig(cfg) + exists, err := rds.tableExists() + + if strings.Compare("load", rds.command) == 0 { + if !exists { + _, err = rds.createTable() + } else { + ensureCleanTable := p.GetBool(ensureCleanTableFieldName, ensureCleanTableFieldNameDefault) + if ensureCleanTable { + log.Printf("dynamo table named %s already existed. Deleting it...\n", *rds.tablename) + _ = rds.deleteTable() + _, err = rds.createTable() + } else { + log.Printf("dynamo table named %s already existed. Skipping table creation.\n", *rds.tablename) + } + } + } else { + if !exists { + log.Fatalf("dynamo table named %s does not exist. You need to run the load stage previous than '%s'...\n", *rds.tablename, "run") + } + } + return rds, err +} + +func (rds *dynamodbWrapper) deleteTable() error { + _, err := rds.client.DeleteTable(context.TODO(), &dynamodb.DeleteTableInput{ + TableName: rds.tablename, + }) + if err != nil { + log.Fatalf("Unable to delete table, %v", err) + } + waiter := dynamodb.NewTableNotExistsWaiter(rds.client) + err = waiter.Wait(context.TODO(), &dynamodb.DescribeTableInput{ + TableName: rds.tablename}, 5*time.Minute) + if err != nil { + log.Fatalf("Wait for table deletion failed. Here's why: %v", err) + } + return err +} + +const ( + tablename = "dynamodb.tablename" + tablenameDefault = "ycsb" + primaryKeyFieldName = "dynamodb.primarykey" + primaryKeyFieldNameDefault = "_key" + readCapacityUnitsFieldName = "dynamodb.rc.units" + readCapacityUnitsFieldNameDefault = 10 + writeCapacityUnitsFieldName = "dynamodb.wc.units" + writeCapacityUnitsFieldNameDefault = 10 + ensureCleanTableFieldName = "dynamodb.ensure.clean.table" + ensureCleanTableFieldNameDefault = true + endpointField = "dynamodb.endpoint" + endpointFieldDefault = "" + regionField = "dynamodb.region" + regionFieldDefault = "" + // GetItem provides an eventually consistent read by default. + // If your application requires a strongly consistent read, set ConsistentRead to true. + // Although a strongly consistent read might take more time than an eventually consistent read, it always returns the last updated value. + consistentReadFieldName = "dynamodb.consistent.reads" + consistentReadFieldNameDefault = false + deleteTableAfterRunFieldName = "dynamodb.delete.after.run.stage" + deleteTableAfterRunFieldNameDefault = false +) + +func init() { + ycsb.RegisterDBCreator("dynamodb", dynamoDbCreator{}) +} diff --git a/go-ycsb/db/elasticsearch/db.go b/go-ycsb/db/elasticsearch/db.go new file mode 100644 index 000000000..a896b8494 --- /dev/null +++ b/go-ycsb/db/elasticsearch/db.go @@ -0,0 +1,346 @@ +package elastic + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "github.com/cenkalti/backoff/v4" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "github.com/elastic/go-elasticsearch/v8/esutil" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/ycsb" + "net" + "net/http" + "runtime" + "strings" + "time" +) + +const ( + elasticUrl = "es.hosts.list" + elasticUrlDefault = "http://127.0.0.1:9200" + elasticInsecureSSLProp = "es.insecure.ssl" + elasticInsecureSSLPropDefault = false + elasticExitOnIndexCreateFailureProp = "es.exit.on.index.create.fail" + elasticExitOnIndexCreateFailurePropDefault = false + elasticShardCountProp = "es.number_of_shards" + elasticShardCountPropDefault = 1 + elasticReplicaCountProp = "es.number_of_replicas" + elasticReplicaCountPropDefault = 0 + elasticUsername = "es.username" + elasticUsernameDefault = "elastic" + elasticPassword = "es.password" + elasticPasswordPropDefault = "" + elasticFlushInterval = "es.flush_interval" + bulkIndexerNumberOfWorkers = "es.bulk.num_workers" + elasticMaxRetriesProp = "es.max_retires" + elasticMaxRetriesPropDefault = 10 + bulkIndexerFlushBytesProp = "es.bulk.flush_bytes" + bulkIndexerFlushBytesDefault = 5e+6 + bulkIndexerFlushIntervalSecondsProp = "es.bulk.flush_interval_secs" + bulkIndexerFlushIntervalSecondsPropDefault = 30 + elasticIndexNameDefault = "ycsb" + elasticIndexName = "es.index" +) + +type elastic struct { + cli *elasticsearch.Client + bi esutil.BulkIndexer + indexName string + verbose bool +} + +func (m *elastic) Close() error { + return nil +} + +func (m *elastic) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { + return ctx +} + +func (m *elastic) CleanupThread(ctx context.Context) { + m.bi.Close(context.Background()) +} + +// Read a document. +func (m *elastic) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + res, err := m.cli.Get(m.indexName, key) + if err != nil { + if m.verbose { + fmt.Println("Cannot read document %d: %s", key, err) + } + return nil, err + } + var r map[string][]byte + json.NewDecoder(res.Body).Decode(&r) + return r, nil +} + +// Scan documents. +func (m *elastic) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + return nil, fmt.Errorf("scan is not supported") + +} + +// Insert a document. +func (m *elastic) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + data, err := json.Marshal(values) + if err != nil { + if m.verbose { + fmt.Println("Cannot encode document %d: %s", key, err) + } + return err + } + // Add an item to the BulkIndexer + err = m.bi.Add( + context.Background(), + esutil.BulkIndexerItem{ + // Action field configures the operation to perform (index, create, delete, update) + Action: "index", + + // DocumentID is the (optional) document ID + DocumentID: key, + + // Body is an `io.Reader` with the payload + Body: bytes.NewReader(data), + + // OnSuccess is called for each successful operation + OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) { + }, + // OnFailure is called for each failed operation + OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { + if err != nil { + fmt.Printf("ERROR BULK INSERT: %s", err) + } else { + fmt.Printf("ERROR BULK INSERT: %s: %s", res.Error.Type, res.Error.Reason) + } + }, + }, + ) + if err != nil { + if m.verbose { + fmt.Println("Unexpected error while bulk inserting: %s", err) + } + return err + } + return nil +} + +// Update a document. +func (m *elastic) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + data, err := json.Marshal(values) + if err != nil { + if m.verbose { + fmt.Println("Cannot encode document %d: %s", key, err) + } + return err + } + // Add an item to the BulkIndexer + err = m.bi.Add( + context.Background(), + esutil.BulkIndexerItem{ + // Action field configures the operation to perform (index, create, delete, update) + Action: "update", + + // DocumentID is the (optional) document ID + DocumentID: key, + + // Body is an `io.Reader` with the payload + Body: bytes.NewReader(data), + + // OnSuccess is called for each successful operation + OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) { + }, + // OnFailure is called for each failed operation + OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { + if err != nil { + fmt.Printf("ERROR BULK UPDATING: %s", err) + } else { + fmt.Printf("ERROR BULK UPDATING: %s: %s", res.Error.Type, res.Error.Reason) + } + }, + }, + ) + if err != nil { + if m.verbose { + fmt.Println("Unexpected error while bulk updating: %s", err) + } + return err + } + return nil +} + +// Delete a document. +func (m *elastic) Delete(ctx context.Context, table string, key string) error { + // Add an delete to the BulkIndexer + err := m.bi.Add( + context.Background(), + esutil.BulkIndexerItem{ + // Action field configures the operation to perform (index, create, delete, update) + Action: "delete", + + // DocumentID is the (optional) document ID + DocumentID: key, + + // OnSuccess is called for each successful operation + OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) { + }, + // OnFailure is called for each failed operation + OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { + if err != nil { + fmt.Printf("ERROR BULK UPDATING: %s", err) + } else { + fmt.Printf("ERROR BULK UPDATING: %s: %s", res.Error.Type, res.Error.Reason) + } + }, + }, + ) + if err != nil { + if m.verbose { + fmt.Println("Unexpected error while bulk deleting: %s", err) + } + return err + } + return nil +} + +type elasticCreator struct { +} + +func (c elasticCreator) Create(p *properties.Properties) (ycsb.DB, error) { + defaultNumCpus := runtime.NumCPU() + bulkIndexerRefresh := "false" + batchSize := p.GetInt(prop.BatchSize, prop.DefaultBatchSize) + if batchSize <= 1 { + bulkIndexerRefresh = "wait_for" + fmt.Printf("Bulk API is disable given the property `%s`=1. For optimal indexing speed please increase this value property\n", prop.BatchSize) + } + bulkIndexerNumCpus := p.GetInt(bulkIndexerNumberOfWorkers, defaultNumCpus) + elasticMaxRetries := p.GetInt(elasticMaxRetriesProp, elasticMaxRetriesPropDefault) + bulkIndexerFlushBytes := p.GetInt(bulkIndexerFlushBytesProp, bulkIndexerFlushBytesDefault) + flushIntervalSeconds := p.GetInt(bulkIndexerFlushIntervalSecondsProp, bulkIndexerFlushIntervalSecondsPropDefault) + elasticReplicaCount := p.GetInt(elasticReplicaCountProp, elasticReplicaCountPropDefault) + elasticShardCount := p.GetInt(elasticShardCountProp, elasticShardCountPropDefault) + + addressesS := p.GetString(elasticUrl, elasticUrlDefault) + insecureSSL := p.GetBool(elasticInsecureSSLProp, elasticInsecureSSLPropDefault) + failOnCreate := p.GetBool(elasticExitOnIndexCreateFailureProp, elasticExitOnIndexCreateFailurePropDefault) + esUser := p.GetString(elasticUsername, elasticUsernameDefault) + esPass := p.GetString(elasticPassword, elasticPasswordPropDefault) + command, _ := p.Get(prop.Command) + verbose := p.GetBool(prop.Verbose, prop.VerboseDefault) + iname := p.GetString(elasticIndexName, elasticIndexNameDefault) + addresses := strings.Split(addressesS, ",") + + retryBackoff := backoff.NewExponentialBackOff() + // + //// Get the SystemCertPool, continue with an empty pool on error + //rootCAs, _ := x509.SystemCertPool() + //if rootCAs == nil { + // rootCAs = x509.NewCertPool() + //} + + cfg := elasticsearch.Config{ + Addresses: addresses, + // Retry on 429 TooManyRequests statuses + RetryOnStatus: []int{502, 503, 504, 429}, + + // Configure the backoff function + RetryBackoff: func(i int) time.Duration { + if i == 1 { + retryBackoff.Reset() + } + return retryBackoff.NextBackOff() + }, + MaxRetries: elasticMaxRetries, + Username: esUser, + Password: esPass, + // Transport / SSL + Transport: &http.Transport{ + MaxIdleConnsPerHost: 10, + ResponseHeaderTimeout: time.Second, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: insecureSSL, + }, + }, + } + es, err := elasticsearch.NewClient(cfg) + if err != nil { + fmt.Println(fmt.Sprintf("Error creating the elastic client: %s", err)) + return nil, err + } + fmt.Println("Connected to elastic!") + if verbose { + fmt.Println(es.Info()) + } + // Create the BulkIndexer + var flushIntervalTime = flushIntervalSeconds * int(time.Second) + + bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ + Index: iname, // The default index name + Client: es, // The Elasticsearch client + NumWorkers: bulkIndexerNumCpus, // The number of worker goroutines + FlushBytes: bulkIndexerFlushBytes, // The flush threshold in bytes + FlushInterval: time.Duration(flushIntervalTime), // The periodic flush interval + // If true, Elasticsearch refreshes the affected + // shards to make this operation visible to search + // if wait_for then wait for a refresh to make this operation visible to search, + // if false do nothing with refreshes. Valid values: true, false, wait_for. Default: false. + Refresh: bulkIndexerRefresh, + }) + if err != nil { + fmt.Println("Error creating the elastic indexer: %s", err) + return nil, err + } + + if strings.Compare("load", command) == 0 { + fmt.Println("Ensuring that if the index exists we recreat it") + // Re-create the index + var res *esapi.Response + if res, err = es.Indices.Delete([]string{iname}, es.Indices.Delete.WithIgnoreUnavailable(true)); err != nil || res.IsError() { + fmt.Println(fmt.Sprintf("Cannot delete index: %s", err)) + return nil, err + } + res.Body.Close() + + // Define index mapping. + mapping := map[string]interface{}{"settings": map[string]interface{}{"index": map[string]interface{}{"number_of_shards": elasticShardCount, "number_of_replicas": elasticReplicaCount}}} + data, err := json.Marshal(mapping) + if err != nil { + if verbose { + fmt.Println(fmt.Sprintf("Cannot encode index mapping %v: %s", mapping, err)) + } + return nil, err + } + res, err = es.Indices.Create(iname, es.Indices.Create.WithBody(strings.NewReader(string(data)))) + if err != nil && failOnCreate { + fmt.Println(fmt.Sprintf("Cannot create index: %s", err)) + return nil, err + } + if res.IsError() && failOnCreate { + fmt.Println(fmt.Sprintf("Cannot create index: %s", res)) + return nil, fmt.Errorf("Cannot create index: %s", res) + } + res.Body.Close() + } + + m := &elastic{ + cli: es, + bi: bi, + indexName: iname, + verbose: verbose, + } + return m, nil +} + +func init() { + ycsb.RegisterDBCreator("elastic", elasticCreator{}) +} diff --git a/go-ycsb/db/etcd/db.go b/go-ycsb/db/etcd/db.go new file mode 100644 index 000000000..10580a849 --- /dev/null +++ b/go-ycsb/db/etcd/db.go @@ -0,0 +1,163 @@ +package etcd + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "strings" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" + + "github.com/magiconair/properties" + "go.etcd.io/etcd/client/pkg/v3/transport" + + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// properties +const ( + etcdEndpoints = "etcd.endpoints" + etcdDialTimeout = "etcd.dial_timeout" + etcdCertFile = "etcd.cert_file" + etcdKeyFile = "etcd.key_file" + etcdCaFile = "etcd.cacert_file" +) + +type etcdCreator struct{} + +type etcdDB struct { + p *properties.Properties + client *clientv3.Client +} + +func init() { + ycsb.RegisterDBCreator("etcd", etcdCreator{}) +} + +func (c etcdCreator) Create(p *properties.Properties) (ycsb.DB, error) { + cfg, err := getClientConfig(p) + if err != nil { + return nil, err + } + + client, err := clientv3.New(*cfg) + if err != nil { + return nil, err + } + + return &etcdDB{ + p: p, + client: client, + }, nil +} + +func getClientConfig(p *properties.Properties) (*clientv3.Config, error) { + endpoints := p.GetString(etcdEndpoints, "localhost:2379") + dialTimeout := p.GetDuration(etcdDialTimeout, 2*time.Second) + + var tlsConfig *tls.Config + if strings.Contains(endpoints, "https") { + tlsInfo := transport.TLSInfo{ + CertFile: p.MustGetString(etcdCertFile), + KeyFile: p.MustGetString(etcdKeyFile), + TrustedCAFile: p.MustGetString(etcdCaFile), + } + c, err := tlsInfo.ClientConfig() + if err != nil { + return nil, err + } + tlsConfig = c + } + + return &clientv3.Config{ + Endpoints: strings.Split(endpoints, ","), + DialTimeout: dialTimeout, + TLS: tlsConfig, + }, nil +} + +func (db *etcdDB) Close() error { + return db.client.Close() +} + +func (db *etcdDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (db *etcdDB) CleanupThread(_ context.Context) { +} + +func getRowKey(table string, key string) string { + return fmt.Sprintf("%s:%s", table, key) +} + +func (db *etcdDB) Read(ctx context.Context, table string, key string, _ []string) (map[string][]byte, error) { + rkey := getRowKey(table, key) + value, err := db.client.Get(ctx, rkey) + if err != nil { + return nil, err + } + + if value.Count == 0 { + return nil, fmt.Errorf("could not find value for key [%s]", rkey) + } + + var r map[string][]byte + err = json.NewDecoder(bytes.NewReader(value.Kvs[0].Value)).Decode(&r) + if err != nil { + return nil, err + } + return r, nil +} + +func (db *etcdDB) Scan(ctx context.Context, table string, startKey string, count int, _ []string) ([]map[string][]byte, error) { + res := make([]map[string][]byte, count) + rkey := getRowKey(table, startKey) + values, err := db.client.Get(ctx, rkey, clientv3.WithFromKey(), clientv3.WithLimit(int64(count))) + if err != nil { + return nil, err + } + + if values.Count != int64(count) { + return nil, fmt.Errorf("unexpected number of result for key [%s], expected %d but was %d", rkey, count, values.Count) + } + + for _, v := range values.Kvs { + var r map[string][]byte + err = json.NewDecoder(bytes.NewReader(v.Value)).Decode(&r) + if err != nil { + return nil, err + } + res = append(res, r) + } + return res, nil +} + +func (db *etcdDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + rkey := getRowKey(table, key) + data, err := json.Marshal(values) + if err != nil { + return err + } + _, err = db.client.Put(ctx, rkey, string(data)) + if err != nil { + return err + } + + return nil +} + +func (db *etcdDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + return db.Update(ctx, table, key, values) +} + +func (db *etcdDB) Delete(ctx context.Context, table string, key string) error { + _, err := db.client.Delete(ctx, getRowKey(table, key)) + if err != nil { + return err + } + return nil +} diff --git a/go-ycsb/db/etcd/doc.go b/go-ycsb/db/etcd/doc.go new file mode 100644 index 000000000..3ae594e7d --- /dev/null +++ b/go-ycsb/db/etcd/doc.go @@ -0,0 +1,3 @@ +package etcd + +// If you want to use etcd, please follow the [Getting Started](https://github.com/etcd-io/etcd#getting-etcd) guide to install it. diff --git a/go-ycsb/db/foundationdb/db.go b/go-ycsb/db/foundationdb/db.go new file mode 100644 index 000000000..962f81d21 --- /dev/null +++ b/go-ycsb/db/foundationdb/db.go @@ -0,0 +1,204 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build foundationdb + +package foundationdb + +import ( + "context" + "fmt" + + "github.com/apple/foundationdb/bindings/go/src/fdb" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/util" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +const ( + fdbClusterFile = "fdb.cluster" + fdbDatabase = "fdb.dbname" + fdbAPIVersion = "fdb.apiversion" +) + +type fDB struct { + db fdb.Database + r *util.RowCodec + bufPool *util.BufPool +} + +func createDB(p *properties.Properties) (ycsb.DB, error) { + clusterFile := p.GetString(fdbClusterFile, "") + database := p.GetString(fdbDatabase, "DB") + apiVersion := p.GetInt(fdbAPIVersion, 510) + + fdb.MustAPIVersion(apiVersion) + + db, err := fdb.Open(clusterFile, []byte(database)) + if err != nil { + return nil, err + } + + bufPool := util.NewBufPool() + + return &fDB{ + db: db, + r: util.NewRowCodec(p), + bufPool: bufPool, + }, nil +} + +func (db *fDB) Close() error { + return nil +} + +func (db *fDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (db *fDB) CleanupThread(ctx context.Context) { +} + +func (db *fDB) getRowKey(table string, key string) []byte { + return util.Slice(fmt.Sprintf("%s:%s", table, key)) +} + +func (db *fDB) getEndRowKey(table string) []byte { + // ';' is ':' + 1 in the ASCII + return util.Slice(fmt.Sprintf("%s;", table)) +} + +func (db *fDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + rowKey := db.getRowKey(table, key) + row, err := db.db.Transact(func(tr fdb.Transaction) (interface{}, error) { + f := tr.Get(fdb.Key(rowKey)) + return f.Get() + }) + + if err != nil { + return nil, err + } else if row == nil { + return nil, nil + } + + return db.r.Decode(row.([]byte), fields) +} + +func (db *fDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + rowKey := db.getRowKey(table, startKey) + res, err := db.db.Transact(func(tr fdb.Transaction) (interface{}, error) { + r := fdb.KeyRange{ + Begin: fdb.Key(rowKey), + End: fdb.Key(db.getEndRowKey(table)), + } + ri := tr.GetRange(r, fdb.RangeOptions{Limit: count}).Iterator() + res := make([]map[string][]byte, 0, count) + for ri.Advance() { + kv, err := ri.Get() + if err != nil { + return nil, err + } + + if kv.Value == nil { + res = append(res, nil) + } else { + v, err := db.r.Decode(kv.Value, fields) + if err != nil { + return nil, err + } + res = append(res, v) + } + + } + + return res, nil + }) + if err != nil { + return nil, err + } + return res.([]map[string][]byte), nil +} + +func (db *fDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + rowKey := db.getRowKey(table, key) + _, err := db.db.Transact(func(tr fdb.Transaction) (ret interface{}, e error) { + f := tr.Get(fdb.Key(rowKey)) + row, err := f.Get() + if err != nil { + return nil, err + } else if row == nil { + return nil, nil + } + + data, err := db.r.Decode(row, nil) + if err != nil { + return nil, err + } + + for field, value := range values { + data[field] = value + } + + buf := db.bufPool.Get() + defer db.bufPool.Put(buf) + + buf, err = db.r.Encode(buf, data) + if err != nil { + return nil, err + } + + tr.Set(fdb.Key(rowKey), buf) + return + }) + + return err +} + +func (db *fDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + // Simulate TiDB data + buf := db.bufPool.Get() + defer db.bufPool.Put(buf) + + buf, err := db.r.Encode(buf, values) + if err != nil { + return err + } + + rowKey := db.getRowKey(table, key) + _, err = db.db.Transact(func(tr fdb.Transaction) (ret interface{}, e error) { + tr.Set(fdb.Key(rowKey), buf) + return + }) + return err +} + +func (db *fDB) Delete(ctx context.Context, table string, key string) error { + rowKey := db.getRowKey(table, key) + _, err := db.db.Transact(func(tr fdb.Transaction) (ret interface{}, e error) { + tr.Clear(fdb.Key(rowKey)) + return + }) + return err +} + +type fdbCreator struct { +} + +func (c fdbCreator) Create(p *properties.Properties) (ycsb.DB, error) { + return createDB(p) +} + +func init() { + ycsb.RegisterDBCreator("fdb", fdbCreator{}) + ycsb.RegisterDBCreator("foundationdb", fdbCreator{}) +} diff --git a/go-ycsb/db/foundationdb/doc.go b/go-ycsb/db/foundationdb/doc.go new file mode 100644 index 000000000..42dbd4777 --- /dev/null +++ b/go-ycsb/db/foundationdb/doc.go @@ -0,0 +1,3 @@ +package foundationdb + +// If you want to use FoundationDB, you must install [client](https://www.foundationdb.org/download/) libraray. diff --git a/go-ycsb/db/minio/db.go b/go-ycsb/db/minio/db.go new file mode 100644 index 000000000..d946e52cf --- /dev/null +++ b/go-ycsb/db/minio/db.go @@ -0,0 +1,128 @@ +package minio + +import ( + "bytes" + "context" + "io/ioutil" + + "github.com/magiconair/properties" + "github.com/minio/minio-go" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +const ( + minioAccessKey = "minio.access-key" + minioSecretKey = "minio.secret-key" + minioEndpoint = "minio.endpoint" + minioSecure = "minio.secure" +) + +type minioCreator struct{} + +func (c minioCreator) Create(p *properties.Properties) (ycsb.DB, error) { + accessKeyID := p.GetString(minioAccessKey, "minio") + secretAccessKey := p.GetString(minioSecretKey, "myminio") + endpoint := p.GetString(minioEndpoint, "http://127.0.0.1:9000") + secure := p.GetBool(minioSecure, false) + client, err := minio.New(endpoint, accessKeyID, secretAccessKey, secure) + if err != nil { + return nil, err + } + return &minioDB{ + db: client, + }, nil +} + +type minioDB struct { + db *minio.Client +} + +// Close closes the database layer. +func (db *minioDB) Close() error { + return nil +} + +// InitThread initializes the state associated to the goroutine worker. +// The Returned context will be passed to the following usage. +func (db *minioDB) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { + return ctx +} + +// CleanupThread cleans up the state when the worker finished. +func (db *minioDB) CleanupThread(ctx context.Context) { +} + +// Read reads a record from the database and returns a map of each field/value pair. +// table: The name of the table. +// key: The record key of the record to read. +// fields: The list of fields to read, nil|empty for reading all. +func (db *minioDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + obj, err := db.db.GetObjectWithContext(ctx, table, key, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + bs, err := ioutil.ReadAll(obj) + if err != nil { + return nil, err + } + return map[string][]byte{"field0": bs}, nil +} + +// Scan scans records from the database. +// table: The name of the table. +// startKey: The first record key to read. +// count: The number of records to read. +// fields: The list of fields to read, nil|empty for reading all. +func (db *minioDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + res := make([]map[string][]byte, count) + done := make(chan struct{}) + defer close(done) + ch := db.db.ListObjectsV2(table, startKey, true, done) + + for i := 0; i < count; i++ { + obj, ok := <-ch + if !ok { + break + } + res[i] = map[string][]byte{obj.Key: nil} + } + return res, nil +} + +// Update updates a record in the database. Any field/value pairs will be written into the +// database or overwritten the existing values with the same field name. +// table: The name of the table. +// key: The record key of the record to update. +// values: A map of field/value pairs to update in the record. +func (db *minioDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + var bs []byte + for _, v := range values { + bs = v + break + } + reader := bytes.NewBuffer(bs) + size := int64(len(bs)) + _, err := db.db.PutObjectWithContext(ctx, table, key, reader, size, minio.PutObjectOptions{}) + return err +} + +// Insert inserts a record in the database. Any field/value pairs will be written into the +// database. +// table: The name of the table. +// key: The record key of the record to insert. +// values: A map of field/value pairs to insert in the record. +func (db *minioDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + return db.Update(ctx, table, key, values) +} + +// Delete deletes a record from the database. +// table: The name of the table. +// key: The record key of the record to delete. +func (db *minioDB) Delete(ctx context.Context, table string, key string) error { + return db.db.RemoveObject(table, key) +} + +func init() { + ycsb.RegisterDBCreator("minio", minioCreator{}) +} diff --git a/go-ycsb/db/mongodb/db.go b/go-ycsb/db/mongodb/db.go new file mode 100644 index 000000000..913c6bbf7 --- /dev/null +++ b/go-ycsb/db/mongodb/db.go @@ -0,0 +1,203 @@ +// Copyright (c) 2020 Daimler TSS GmbH TLS support + +package mongodb + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "io/ioutil" + "log" + "strings" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/ycsb" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/x/mongo/driver/connstring" +) + +const ( + mongodbUrl = "mongodb.url" + mongodbAuthdb = "mongodb.authdb" + mongodbUsername = "mongodb.username" + mongodbPassword = "mongodb.password" + + // see https://github.com/brianfrankcooper/YCSB/tree/master/mongodb#mongodb-configuration-parameters + mongodbUrlDefault = "mongodb://127.0.0.1:27017/ycsb?w=1" + mongodbDatabaseDefault = "ycsb" + mongodbAuthdbDefault = "admin" + mongodbTLSSkipVerify = "mongodb.tls_skip_verify" + mongodbTLSCAFile = "mongodb.tls_ca_file" +) + +type mongoDB struct { + cli *mongo.Client + db *mongo.Database +} + +func (m *mongoDB) Close() error { + return m.cli.Disconnect(context.Background()) +} + +func (m *mongoDB) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { + return ctx +} + +func (m *mongoDB) CleanupThread(ctx context.Context) { +} + +// Read a document. +func (m *mongoDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + projection := map[string]bool{"_id": false} + for _, field := range fields { + projection[field] = true + } + opt := &options.FindOneOptions{Projection: projection} + var doc map[string][]byte + if err := m.db.Collection(table).FindOne(ctx, bson.M{"_id": key}, opt).Decode(&doc); err != nil { + return nil, fmt.Errorf("Read error: %s", err.Error()) + } + return doc, nil +} + +// Scan documents. +func (m *mongoDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + projection := map[string]bool{"_id": false} + for _, field := range fields { + projection[field] = true + } + limit := int64(count) + opt := &options.FindOptions{Projection: projection, Sort: bson.M{"_id": 1}, Limit: &limit} + cursor, err := m.db.Collection(table).Find(ctx, bson.M{"_id": bson.M{"$gte": startKey}}, opt) + if err != nil { + return nil, fmt.Errorf("Scan error: %s", err.Error()) + } + defer cursor.Close(ctx) + var docs []map[string][]byte + for cursor.Next(ctx) { + var doc map[string][]byte + if err := cursor.Decode(&doc); err != nil { + return docs, fmt.Errorf("Scan error: %s", err.Error()) + } + docs = append(docs, doc) + } + return docs, nil +} + +// Insert a document. +func (m *mongoDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + doc := bson.M{"_id": key} + for k, v := range values { + doc[k] = v + } + if _, err := m.db.Collection(table).InsertOne(ctx, doc); err != nil { + fmt.Println(err) + return fmt.Errorf("Insert error: %s", err.Error()) + } + return nil +} + +// Update a document. +func (m *mongoDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + res, err := m.db.Collection(table).UpdateOne(ctx, bson.M{"_id": key}, bson.M{"$set": values}) + if err != nil { + return fmt.Errorf("Update error: %s", err.Error()) + } + if res.MatchedCount != 1 { + return fmt.Errorf("Update error: %s not found", key) + } + return nil +} + +// Delete a document. +func (m *mongoDB) Delete(ctx context.Context, table string, key string) error { + res, err := m.db.Collection(table).DeleteOne(ctx, bson.M{"_id": key}) + if err != nil { + return fmt.Errorf("Delete error: %s", err.Error()) + } + if res.DeletedCount != 1 { + return fmt.Errorf("Delete error: %s not found", key) + } + return nil +} + +type mongodbCreator struct{} + +func (c mongodbCreator) Create(p *properties.Properties) (ycsb.DB, error) { + uri := p.GetString(mongodbUrl, mongodbUrlDefault) + authdb := p.GetString(mongodbAuthdb, mongodbAuthdbDefault) + tlsSkipVerify := p.GetBool(mongodbTLSSkipVerify, false) + caFile := p.GetString(mongodbTLSCAFile, "") + + connString, err := connstring.Parse(uri) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cliOpts := options.Client().ApplyURI(uri) + if cliOpts.TLSConfig != nil { + if len(connString.Hosts) > 0 { + servername := strings.Split(connString.Hosts[0], ":")[0] + log.Printf("using server name for tls: %s\n", servername) + cliOpts.TLSConfig.ServerName = servername + } + if tlsSkipVerify { + log.Println("skipping tls cert validation") + cliOpts.TLSConfig.InsecureSkipVerify = true + } + + if caFile != "" { + // Load CA cert + caCert, err := ioutil.ReadFile(caFile) + if err != nil { + log.Fatal(err) + } + caCertPool := x509.NewCertPool() + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + log.Fatalf("certifacte %s could not be parsed", caFile) + } + + cliOpts.TLSConfig.RootCAs = caCertPool + } + } + + username, usrExist := p.Get(mongodbUsername) + password, pwdExist := p.Get(mongodbPassword) + if usrExist && pwdExist { + cliOpts.SetAuth(options.Credential{AuthSource: authdb, Username: username, Password: password}) + } else if usrExist { + return nil, errors.New("mongodb.username is set, but mongodb.password is missing") + } else if pwdExist { + return nil, errors.New("mongodb.password is set, but mongodb.username is missing") + } + + cli, err := mongo.Connect(ctx, cliOpts) + if err != nil { + return nil, err + } + if err := cli.Ping(ctx, nil); err != nil { + return nil, err + } + // check if auth passed + if _, err := cli.ListDatabaseNames(ctx, map[string]string{}); err != nil { + return nil, errors.New("auth failed") + } + + fmt.Println("Connected to MongoDB!") + + m := &mongoDB{ + cli: cli, + db: cli.Database(mongodbDatabaseDefault), + } + return m, nil +} + +func init() { + ycsb.RegisterDBCreator("mongodb", mongodbCreator{}) +} diff --git a/go-ycsb/db/mysql/db.go b/go-ycsb/db/mysql/db.go new file mode 100644 index 000000000..4c4887662 --- /dev/null +++ b/go-ycsb/db/mysql/db.go @@ -0,0 +1,512 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "bytes" + "context" + "crypto/sha1" + "database/sql" + "database/sql/driver" + "encoding/hex" + "fmt" + "strings" + "sync/atomic" + + "github.com/go-sql-driver/mysql" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// mysql properties +const ( + mysqlHost = "mysql.host" + mysqlPort = "mysql.port" + mysqlUser = "mysql.user" + mysqlPassword = "mysql.password" + mysqlDBName = "mysql.db" + mysqlForceIndex = "mysql.force_index" + // TODO: support batch and auto commit + + tidbClusterIndex = "tidb.cluster_index" + tidbInstances = "tidb.instances" +) + +type muxDriver struct { + cursor uint64 + instances []string + internal driver.Driver +} + +func (drv *muxDriver) Open(name string) (driver.Conn, error) { + k := atomic.AddUint64(&drv.cursor, 1) + return drv.internal.Open(drv.instances[int(k)%len(drv.instances)]) +} + +func openTiDBInstances(addrs []string, user string, pass string, db string) (*sql.DB, error) { + instances := make([]string, len(addrs)) + hash := sha1.New() + for i, addr := range addrs { + hash.Write([]byte("+" + addr)) + instances[i] = fmt.Sprintf("%s:%s@tcp(%s)/%s", user, pass, addr, db) + } + digest := hash.Sum(nil) + driver := "tidb:" + hex.EncodeToString(digest[:]) + for _, n := range sql.Drivers() { + if n == driver { + return sql.Open(driver, "") + } + } + sql.Register(driver, &muxDriver{instances: instances, internal: &mysql.MySQLDriver{}}) + return sql.Open(driver, "") +} + +type mysqlCreator struct { + name string +} + +type mysqlDB struct { + p *properties.Properties + db *sql.DB + verbose bool + forceIndexKeyword string + + bufPool *util.BufPool +} + +type contextKey string + +const stateKey = contextKey("mysqlDB") + +type mysqlState struct { + // Do we need a LRU cache here? + stmtCache map[string]*sql.Stmt + + conn *sql.Conn +} + +func (c mysqlCreator) Create(p *properties.Properties) (ycsb.DB, error) { + d := new(mysqlDB) + d.p = p + + host := p.GetString(mysqlHost, "127.0.0.1") + port := p.GetInt(mysqlPort, 3306) + user := p.GetString(mysqlUser, "root") + password := p.GetString(mysqlPassword, "") + dbName := p.GetString(mysqlDBName, "test") + tidbList := p.GetString(tidbInstances, "") + + var ( + db *sql.DB + err error + tidbs []string + ) + for _, tidb := range strings.Split(tidbList, ",") { + tidb = strings.TrimSpace(tidb) + if len(tidb) > 0 { + tidbs = append(tidbs, tidb) + } + } + if len(tidbs) > 0 { + db, err = openTiDBInstances(tidbs, user, password, dbName) + } else { + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", user, password, host, port, dbName) + db, err = sql.Open("mysql", dsn) + } + if err != nil { + return nil, err + } + + threadCount := int(p.GetInt64(prop.ThreadCount, prop.ThreadCountDefault)) + db.SetMaxIdleConns(threadCount + 1) + db.SetMaxOpenConns(threadCount * 2) + + d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) + if p.GetBool(mysqlForceIndex, true) { + d.forceIndexKeyword = "FORCE INDEX(`PRIMARY`)" + } + d.db = db + + d.bufPool = util.NewBufPool() + + if err := d.createTable(c.name); err != nil { + return nil, err + } + + return d, nil +} + +func (db *mysqlDB) createTable(driverName string) error { + tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) + if db.p.GetBool(prop.DropData, prop.DropDataDefault) && + !db.p.GetBool(prop.DoTransactions, true) { + if _, err := db.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)); err != nil { + return err + } + } + + fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) + fieldLength := db.p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) + + buf := new(bytes.Buffer) + s := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (YCSB_KEY VARCHAR(64) PRIMARY KEY", tableName) + buf.WriteString(s) + + if (driverName == "tidb" || driverName == "mysql") && db.p.GetBool(tidbClusterIndex, true) { + buf.WriteString(" /*T![clustered_index] CLUSTERED */") + } + + for i := int64(0); i < fieldCount; i++ { + buf.WriteString(fmt.Sprintf(", FIELD%d VARCHAR(%d)", i, fieldLength)) + } + + buf.WriteString(");") + + _, err := db.db.Exec(buf.String()) + return err +} + +func (db *mysqlDB) Close() error { + if db.db == nil { + return nil + } + + return db.db.Close() +} + +func (db *mysqlDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + conn, err := db.db.Conn(ctx) + if err != nil { + panic(fmt.Sprintf("failed to create db conn %v", err)) + } + + state := &mysqlState{ + stmtCache: make(map[string]*sql.Stmt), + conn: conn, + } + + return context.WithValue(ctx, stateKey, state) +} + +func (db *mysqlDB) CleanupThread(ctx context.Context) { + state := ctx.Value(stateKey).(*mysqlState) + + for _, stmt := range state.stmtCache { + stmt.Close() + } + state.conn.Close() +} + +func (db *mysqlDB) getAndCacheStmt(ctx context.Context, query string) (*sql.Stmt, error) { + state := ctx.Value(stateKey).(*mysqlState) + + if stmt, ok := state.stmtCache[query]; ok { + return stmt, nil + } + + stmt, err := state.conn.PrepareContext(ctx, query) + if err == sql.ErrConnDone { + // Try build the connection and prepare again + if state.conn, err = db.db.Conn(ctx); err == nil { + stmt, err = state.conn.PrepareContext(ctx, query) + } + } + + if err != nil { + return nil, err + } + + state.stmtCache[query] = stmt + return stmt, nil +} + +func (db *mysqlDB) clearCacheIfFailed(ctx context.Context, query string, err error) { + if err == nil { + return + } + + state := ctx.Value(stateKey).(*mysqlState) + if stmt, ok := state.stmtCache[query]; ok { + stmt.Close() + } + delete(state.stmtCache, query) +} + +func (db *mysqlDB) queryRows(ctx context.Context, query string, count int, args ...interface{}) ([]map[string][]byte, error) { + if db.verbose { + fmt.Printf("%s %v\n", query, args) + } + + stmt, err := db.getAndCacheStmt(ctx, query) + if err != nil { + return nil, err + } + rows, err := stmt.QueryContext(ctx, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + return nil, err + } + + vs := make([]map[string][]byte, 0, count) + for rows.Next() { + m := make(map[string][]byte, len(cols)) + dest := make([]interface{}, len(cols)) + for i := 0; i < len(cols); i++ { + v := new([]byte) + dest[i] = v + } + if err = rows.Scan(dest...); err != nil { + return nil, err + } + + for i, v := range dest { + m[cols[i]] = *v.(*[]byte) + } + + vs = append(vs, m) + } + + return vs, rows.Err() +} + +func (db *mysqlDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + var query string + if len(fields) == 0 { + query = fmt.Sprintf(`SELECT * FROM %s %s WHERE YCSB_KEY = ?`, table, db.forceIndexKeyword) + } else { + query = fmt.Sprintf(`SELECT %s FROM %s %s WHERE YCSB_KEY = ?`, strings.Join(fields, ","), table, db.forceIndexKeyword) + } + + rows, err := db.queryRows(ctx, query, 1, key) + db.clearCacheIfFailed(ctx, query, err) + + if err != nil { + return nil, err + } else if len(rows) == 0 { + return nil, nil + } + + return rows[0], nil +} + +func (db *mysqlDB) BatchRead(ctx context.Context, table string, keys []string, fields []string) ([]map[string][]byte, error) { + args := make([]interface{}, 0, len(keys)) + buf := db.bufPool.Get() + defer db.bufPool.Put(buf) + if len(fields) == 0 { + buf = append(buf, fmt.Sprintf(`SELECT * FROM %s %s WHERE YCSB_KEY IN (`, table, db.forceIndexKeyword)...) + } else { + buf = append(buf, fmt.Sprintf(`SELECT %s FROM %s %s WHERE YCSB_KEY IN (`, strings.Join(fields, ","), table, db.forceIndexKeyword)...) + } + for i, key := range keys { + buf = append(buf, '?') + if i < len(keys)-1 { + buf = append(buf, ',') + } + args = append(args, key) + } + buf = append(buf, ')') + + query := string(buf[:]) + rows, err := db.queryRows(ctx, query, len(keys), args...) + db.clearCacheIfFailed(ctx, query, err) + + if err != nil { + return nil, err + } else if len(rows) == 0 { + return nil, nil + } + + return rows, nil +} + +func (db *mysqlDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + var query string + if len(fields) == 0 { + query = fmt.Sprintf(`SELECT * FROM %s %s WHERE YCSB_KEY >= ? LIMIT ?`, table, db.forceIndexKeyword) + } else { + query = fmt.Sprintf(`SELECT %s FROM %s %s WHERE YCSB_KEY >= ? LIMIT ?`, strings.Join(fields, ","), table, db.forceIndexKeyword) + } + + rows, err := db.queryRows(ctx, query, count, startKey, count) + db.clearCacheIfFailed(ctx, query, err) + + return rows, err +} + +func (db *mysqlDB) execQuery(ctx context.Context, query string, args ...interface{}) error { + if db.verbose { + fmt.Printf("%s %v\n", query, args) + } + + stmt, err := db.getAndCacheStmt(ctx, query) + if err != nil { + return err + } + + _, err = stmt.ExecContext(ctx, args...) + db.clearCacheIfFailed(ctx, query, err) + return err +} + +func (db *mysqlDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + buf := bytes.NewBuffer(db.bufPool.Get()) + defer func() { + db.bufPool.Put(buf.Bytes()) + }() + + buf.WriteString("UPDATE ") + buf.WriteString(table) + buf.WriteString(" SET ") + firstField := true + pairs := util.NewFieldPairs(values) + args := make([]interface{}, 0, len(values)+1) + for _, p := range pairs { + if firstField { + firstField = false + } else { + buf.WriteString(", ") + } + + buf.WriteString(p.Field) + buf.WriteString(`= ?`) + args = append(args, p.Value) + } + buf.WriteString(" WHERE YCSB_KEY = ?") + + args = append(args, key) + + return db.execQuery(ctx, buf.String(), args...) +} + +func (db *mysqlDB) BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) error { + // mysql does not support BatchUpdate, fallback to Update like dbwrapper.go + for i := range keys { + err := db.Update(ctx, table, keys[i], values[i]) + if err != nil { + return err + } + } + return nil +} + +func (db *mysqlDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + args := make([]interface{}, 0, 1+len(values)) + args = append(args, key) + + buf := bytes.NewBuffer(db.bufPool.Get()) + defer func() { + db.bufPool.Put(buf.Bytes()) + }() + + buf.WriteString("INSERT IGNORE INTO ") + buf.WriteString(table) + buf.WriteString(" (YCSB_KEY") + + pairs := util.NewFieldPairs(values) + for _, p := range pairs { + args = append(args, p.Value) + buf.WriteString(" ,") + buf.WriteString(p.Field) + } + buf.WriteString(") VALUES (?") + + for i := 0; i < len(pairs); i++ { + buf.WriteString(" ,?") + } + + buf.WriteByte(')') + + return db.execQuery(ctx, buf.String(), args...) +} + +func (db *mysqlDB) BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) error { + args := make([]interface{}, 0, (1+len(values))*len(keys)) + buf := db.bufPool.Get() + defer db.bufPool.Put(buf) + buf = append(buf, "INSERT IGNORE INTO "...) + buf = append(buf, table...) + buf = append(buf, " (YCSB_KEY"...) + + valueString := strings.Builder{} + valueString.WriteString("(?") + pairs := util.NewFieldPairs(values[0]) + for _, p := range pairs { + buf = append(buf, " ,"...) + buf = append(buf, p.Field...) + + valueString.WriteString(" ,?") + } + // Example: INSERT IGNORE INTO table ([columns]) VALUES + buf = append(buf, ") VALUES "...) + // Example: (?, ?, ?, ....) + valueString.WriteByte(')') + valueStrings := make([]string, 0, len(keys)) + for range keys { + valueStrings = append(valueStrings, valueString.String()) + } + // Example: INSERT IGNORE INTO table ([columns]) VALUES (?, ?, ?...), (?, ?, ?), ... + buf = append(buf, strings.Join(valueStrings, ",")...) + + for i, key := range keys { + args = append(args, key) + pairs := util.NewFieldPairs(values[i]) + for _, p := range pairs { + args = append(args, p.Value) + } + } + + return db.execQuery(ctx, string(buf[:]), args...) +} + +func (db *mysqlDB) Delete(ctx context.Context, table string, key string) error { + query := fmt.Sprintf(`DELETE FROM %s WHERE YCSB_KEY = ?`, table) + + return db.execQuery(ctx, query, key) +} + +func (db *mysqlDB) BatchDelete(ctx context.Context, table string, keys []string) error { + args := make([]interface{}, 0, len(keys)) + buf := db.bufPool.Get() + defer db.bufPool.Put(buf) + buf = append(buf, fmt.Sprintf("DELETE FROM %s WHERE YCSB_KEY IN (", table)...) + for i, key := range keys { + buf = append(buf, '?') + if i < len(keys)-1 { + buf = append(buf, ',') + } + args = append(args, key) + } + buf = append(buf, ')') + + return db.execQuery(ctx, string(buf[:]), args...) +} + +func (db *mysqlDB) Analyze(ctx context.Context, table string) error { + _, err := db.db.Exec(fmt.Sprintf(`ANALYZE TABLE %s`, table)) + return err +} + +func init() { + ycsb.RegisterDBCreator("mysql", mysqlCreator{name: "mysql"}) + ycsb.RegisterDBCreator("tidb", mysqlCreator{name: "tidb"}) + ycsb.RegisterDBCreator("mariadb", mysqlCreator{name: "mariadb"}) +} diff --git a/go-ycsb/db/pegasus/db.go b/go-ycsb/db/pegasus/db.go new file mode 100644 index 000000000..033dabfea --- /dev/null +++ b/go-ycsb/db/pegasus/db.go @@ -0,0 +1,176 @@ +// Copyright (c) 2017, Xiaomi, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package pegasus + +import ( + "context" + "encoding/json" + _ "net/http/pprof" + "strings" + "time" + + "github.com/XiaoMi/pegasus-go-client/pegalog" + "github.com/XiaoMi/pegasus-go-client/pegasus" + "github.com/XiaoMi/pegasus-go-client/pegasus2" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +var ( + RequestTimeout = 3 * time.Second +) + +type pegasusDB struct { + client *pegasus2.Client + sessions []pegasus.TableConnector +} + +func (db *pegasusDB) InitThread(ctx context.Context, threadId int, _ int) context.Context { + return context.WithValue(ctx, "tid", threadId) +} + +func (db *pegasusDB) CleanupThread(_ context.Context) { +} + +func (db *pegasusDB) Close() error { + for _, s := range db.sessions { + if err := s.Close(); err != nil { + return err + } + } + return db.client.Close() +} + +func (db *pegasusDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + timeoutCtx, _ := context.WithTimeout(ctx, RequestTimeout) + s := db.sessions[ctx.Value("tid").(int)] + + rawValue, err := s.Get(timeoutCtx, []byte(key), []byte("")) + if err == nil { + var value map[string][]byte + json.Unmarshal(rawValue, value) + + result := make(map[string][]byte) + for _, field := range fields { + if v, ok := value[field]; ok { + result[field] = v + } + } + + return result, nil + } else { + pegalog.GetLogger().Println(err) + return nil, err + } +} + +func (db *pegasusDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + return nil, nil +} + +func (db *pegasusDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + timeoutCtx, _ := context.WithTimeout(ctx, RequestTimeout) + s := db.sessions[ctx.Value("tid").(int)] + + value, _ := json.Marshal(values) + err := s.Set(timeoutCtx, []byte(key), nil, value) + if err != nil { + pegalog.GetLogger().Println(err) + } + return err +} + +func (db *pegasusDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + timeoutCtx, _ := context.WithTimeout(ctx, RequestTimeout) + s := db.sessions[ctx.Value("tid").(int)] + + value, _ := json.Marshal(values) + err := s.Set(timeoutCtx, []byte(key), []byte(""), value) + if err != nil { + pegalog.GetLogger().Println(err) + } + return err +} + +func (db *pegasusDB) Delete(ctx context.Context, table string, key string) error { + timeoutCtx, _ := context.WithTimeout(ctx, RequestTimeout) + s := db.sessions[ctx.Value("tid").(int)] + + err := s.Del(timeoutCtx, []byte(key), []byte("")) + if err != nil { + pegalog.GetLogger().Println(err) + } + return err +} + +type pegasusCreator struct{} + +func (pegasusCreator) Create(p *properties.Properties) (ycsb.DB, error) { + conf := p.MustGetString("meta_servers") + metaServers := strings.Split(conf, ",") + tbName := p.MustGetString("table") + threadCount := p.MustGetInt("threadcount") + + cfg := pegasus.Config{MetaServers: metaServers} + db := &pegasusDB{} + db.sessions = make([]pegasus.TableConnector, threadCount) + c := pegasus2.NewClient(cfg) + for i := 0; i < threadCount; i++ { + + var err error + timeoutCtx, _ := context.WithTimeout(context.Background(), RequestTimeout) + tb, err := c.OpenTable(timeoutCtx, tbName) + if err != nil { + pegalog.GetLogger().Println("failed to open table: ", err) + return nil, err + } + db.sessions[i] = tb + } + db.client = c + return db, nil +} + +func init() { + ycsb.RegisterDBCreator("pegasus", pegasusCreator{}) +} diff --git a/go-ycsb/db/pg/db.go b/go-ycsb/db/pg/db.go new file mode 100644 index 000000000..81a4a0bda --- /dev/null +++ b/go-ycsb/db/pg/db.go @@ -0,0 +1,361 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package pg + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "strings" + + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + + // pg package + _ "github.com/lib/pq" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// pg properties +const ( + pgHost = "pg.host" + pgPort = "pg.port" + pgUser = "pg.user" + pgPassword = "pg.password" + pgDBName = "pg.db" + pdSSLMode = "pg.sslmode" + // TODO: support batch and auto commit +) + +type pgCreator struct { +} + +type pgDB struct { + p *properties.Properties + db *sql.DB + verbose bool + + bufPool *util.BufPool + + dbName string +} + +type contextKey string + +const stateKey = contextKey("pgDB") + +type pgState struct { + // Do we need a LRU cache here? + stmtCache map[string]*sql.Stmt + + conn *sql.Conn +} + +func (c pgCreator) Create(p *properties.Properties) (ycsb.DB, error) { + d := new(pgDB) + d.p = p + + host := p.GetString(pgHost, "127.0.0.1") + port := p.GetInt(pgPort, 5432) + user := p.GetString(pgUser, "root") + password := p.GetString(pgPassword, "") + dbName := p.GetString(pgDBName, "test") + sslMode := p.GetString(pdSSLMode, "disable") + + dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s", user, password, host, port, dbName, sslMode) + var err error + db, err := sql.Open("postgres", dsn) + if err != nil { + fmt.Printf("open pg failed %v", err) + return nil, err + } + + threadCount := int(p.GetInt64(prop.ThreadCount, prop.ThreadCountDefault)) + db.SetMaxIdleConns(threadCount + 1) + db.SetMaxOpenConns(threadCount * 2) + + d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) + d.db = db + d.dbName = dbName + + d.bufPool = util.NewBufPool() + + if err := d.createTable(); err != nil { + return nil, err + } + + return d, nil +} + +func (db *pgDB) createTable() error { + tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) + + if db.p.GetBool(prop.DropData, prop.DropDataDefault) { + if _, err := db.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)); err != nil { + return err + } + } + + fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) + fieldLength := db.p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) + + buf := new(bytes.Buffer) + s := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (YCSB_KEY VARCHAR(64) PRIMARY KEY", tableName) + buf.WriteString(s) + + for i := int64(0); i < fieldCount; i++ { + buf.WriteString(fmt.Sprintf(", FIELD%d VARCHAR(%d)", i, fieldLength)) + } + + buf.WriteString(");") + + if db.verbose { + fmt.Println(buf.String()) + } + + _, err := db.db.Exec(buf.String()) + return err +} + +func (db *pgDB) Close() error { + if db.db == nil { + return nil + } + + return db.db.Close() +} + +func (db *pgDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + conn, err := db.db.Conn(ctx) + if err != nil { + panic(fmt.Sprintf("failed to create db conn %v", err)) + } + + state := &pgState{ + stmtCache: make(map[string]*sql.Stmt), + conn: conn, + } + + return context.WithValue(ctx, stateKey, state) +} + +func (db *pgDB) CleanupThread(ctx context.Context) { + state := ctx.Value(stateKey).(*pgState) + + for _, stmt := range state.stmtCache { + stmt.Close() + } + state.conn.Close() +} + +func (db *pgDB) getAndCacheStmt(ctx context.Context, query string) (*sql.Stmt, error) { + state := ctx.Value(stateKey).(*pgState) + + if stmt, ok := state.stmtCache[query]; ok { + return stmt, nil + } + + stmt, err := state.conn.PrepareContext(ctx, query) + if err == sql.ErrConnDone { + // Try build the connection and prepare again + if state.conn, err = db.db.Conn(ctx); err == nil { + stmt, err = state.conn.PrepareContext(ctx, query) + } + } + + if err != nil { + return nil, err + } + + state.stmtCache[query] = stmt + return stmt, nil +} + +func (db *pgDB) clearCacheIfFailed(ctx context.Context, query string, err error) { + if err == nil { + return + } + + state := ctx.Value(stateKey).(*pgState) + if stmt, ok := state.stmtCache[query]; ok { + stmt.Close() + } + delete(state.stmtCache, query) +} + +func (db *pgDB) queryRows(ctx context.Context, query string, count int, args ...interface{}) ([]map[string][]byte, error) { + if db.verbose { + fmt.Printf("%s %v\n", query, args) + } + + stmt, err := db.getAndCacheStmt(ctx, query) + if err != nil { + return nil, err + } + rows, err := stmt.QueryContext(ctx, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + return nil, err + } + + vs := make([]map[string][]byte, 0, count) + for rows.Next() { + m := make(map[string][]byte, len(cols)) + dest := make([]interface{}, len(cols)) + for i := 0; i < len(cols); i++ { + v := new([]byte) + dest[i] = v + } + if err = rows.Scan(dest...); err != nil { + return nil, err + } + + for i, v := range dest { + m[cols[i]] = *v.(*[]byte) + } + + vs = append(vs, m) + } + + return vs, rows.Err() +} + +func (db *pgDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + var query string + if len(fields) == 0 { + query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY = $1`, table) + } else { + query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY = $1`, strings.Join(fields, ","), table) + } + + rows, err := db.queryRows(ctx, query, 1, key) + db.clearCacheIfFailed(ctx, query, err) + + if err != nil { + return nil, err + } else if len(rows) == 0 { + return nil, nil + } + + return rows[0], nil +} + +func (db *pgDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + var query string + if len(fields) == 0 { + query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY >= $1 LIMIT $2`, table) + } else { + query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY >= $1 LIMIT $2`, strings.Join(fields, ","), table) + } + + rows, err := db.queryRows(ctx, query, count, startKey, count) + db.clearCacheIfFailed(ctx, query, err) + + return rows, err +} + +func (db *pgDB) execQuery(ctx context.Context, query string, args ...interface{}) error { + if db.verbose { + fmt.Printf("%s %v\n", query, args) + } + + stmt, err := db.getAndCacheStmt(ctx, query) + if err != nil { + return err + } + + _, err = stmt.ExecContext(ctx, args...) + db.clearCacheIfFailed(ctx, query, err) + return err +} + +func (db *pgDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + buf := bytes.NewBuffer(db.bufPool.Get()) + defer func() { + db.bufPool.Put(buf.Bytes()) + }() + + buf.WriteString("UPDATE ") + buf.WriteString(table) + buf.WriteString(" SET ") + firstField := true + args := make([]interface{}, 0, len(values)+1) + placeHolderIndex := 1 + pairs := util.NewFieldPairs(values) + for _, p := range pairs { + if firstField { + firstField = false + } else { + buf.WriteString(", ") + } + + buf.WriteString(fmt.Sprintf("%s = $%d", p.Field, placeHolderIndex)) + args = append(args, p.Value) + placeHolderIndex++ + } + buf.WriteString(fmt.Sprintf(" WHERE YCSB_KEY = $%d", placeHolderIndex)) + + args = append(args, key) + + return db.execQuery(ctx, buf.String(), args...) +} + +func (db *pgDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + args := make([]interface{}, 0, 1+len(values)) + args = append(args, key) + + buf := bytes.NewBuffer(db.bufPool.Get()) + defer func() { + db.bufPool.Put(buf.Bytes()) + }() + + buf.WriteString("INSERT INTO ") + buf.WriteString(table) + buf.WriteString(" (YCSB_KEY") + pairs := util.NewFieldPairs(values) + for _, p := range pairs { + args = append(args, p.Value) + buf.WriteString(" ,") + buf.WriteString(p.Field) + } + buf.WriteString(") VALUES ($1") + + for i := 0; i < len(pairs); i++ { + buf.WriteString(fmt.Sprintf(" ,$%d", i+2)) + } + + buf.WriteString(") ON CONFLICT DO NOTHING") + + return db.execQuery(ctx, buf.String(), args...) +} + +func (db *pgDB) Delete(ctx context.Context, table string, key string) error { + query := fmt.Sprintf(`DELETE FROM %s WHERE YCSB_KEY = $1`, table) + + return db.execQuery(ctx, query, key) +} + +func init() { + ycsb.RegisterDBCreator("pg", pgCreator{}) + ycsb.RegisterDBCreator("postgresql", pgCreator{}) + ycsb.RegisterDBCreator("cockroach", pgCreator{}) + ycsb.RegisterDBCreator("cdb", pgCreator{}) +} diff --git a/go-ycsb/db/redis/db.go b/go-ycsb/db/redis/db.go new file mode 100644 index 000000000..8032dd2b8 --- /dev/null +++ b/go-ycsb/db/redis/db.go @@ -0,0 +1,358 @@ +package redis + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "strings" + "time" + + goredis "github.com/go-redis/redis/v9" + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +const HASH_DATATYPE string = "hash" +const STRING_DATATYPE string = "string" +const JSON_DATATYPE string = "json" +const JSON_SET string = "JSON.SET" +const JSON_GET string = "JSON.GET" +const HSET string = "HSET" +const HMGET string = "HMGET" + +type redisClient interface { + Get(ctx context.Context, key string) *goredis.StringCmd + Do(ctx context.Context, args ...interface{}) *goredis.Cmd + Pipeline() goredis.Pipeliner + Scan(ctx context.Context, cursor uint64, match string, count int64) *goredis.ScanCmd + Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *goredis.StatusCmd + Del(ctx context.Context, keys ...string) *goredis.IntCmd + FlushDB(ctx context.Context) *goredis.StatusCmd + Close() error +} + +type redis struct { + client redisClient + mode string + datatype string +} + +func (r *redis) Close() error { + return r.client.Close() +} + +func (r *redis) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (r *redis) CleanupThread(_ context.Context) { +} + +func (r *redis) Read(ctx context.Context, table string, key string, fields []string) (data map[string][]byte, err error) { + data = make(map[string][]byte, len(fields)) + err = nil + switch r.datatype { + case JSON_DATATYPE: + cmds := make([]*goredis.Cmd, len(fields)) + pipe := r.client.Pipeline() + for pos, fieldName := range fields { + cmds[pos] = pipe.Do(ctx, JSON_GET, getKeyName(table, key), getFieldJsonPath(fieldName)) + } + _, err = pipe.Exec(ctx) + if err != nil { + return + } + var s string = "" + for pos, fieldName := range fields { + s, err = cmds[pos].Text() + if err != nil { + return + } + data[fieldName] = []byte(s) + } + case HASH_DATATYPE: + args := make([]interface{}, 0, len(fields)+2) + args = append(args, HMGET, getKeyName(table, key)) + for _, fieldName := range fields { + args = append(args, fieldName) + } + sliceReply, errI := r.client.Do(ctx, args...).StringSlice() + if errI != nil { + return + } + for pos, slicePos := range sliceReply { + data[fields[pos]] = []byte(slicePos) + } + case STRING_DATATYPE: + fallthrough + default: + { + var res string = "" + res, err = r.client.Get(ctx, getKeyName(table, key)).Result() + if err != nil { + return + } + err = json.Unmarshal([]byte(res), &data) + return + } + } + return + +} + +func (r *redis) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + return nil, fmt.Errorf("scan is not supported") +} + +func (r *redis) Update(ctx context.Context, table string, key string, values map[string][]byte) (err error) { + err = nil + switch r.datatype { + case JSON_DATATYPE: + cmds := make([]*goredis.Cmd, 0, len(values)) + pipe := r.client.Pipeline() + for fieldName, bytes := range values { + cmd := pipe.Do(ctx, JSON_SET, getKeyName(table, key), getFieldJsonPath(fieldName), jsonEscape(bytes)) + cmds = append(cmds, cmd) + } + _, err = pipe.Exec(ctx) + if err != nil { + return + } + for _, cmd := range cmds { + err = cmd.Err() + if err != nil { + return + } + } + case HASH_DATATYPE: + args := make([]interface{}, 0, 2*len(values)+2) + args = append(args, HSET, getKeyName(table, key)) + for fieldName, bytes := range values { + args = append(args, fieldName, string(bytes)) + } + err = r.client.Do(ctx, args...).Err() + case STRING_DATATYPE: + fallthrough + default: + { + var initialEncodedJson string = "" + initialEncodedJson, err = r.client.Get(ctx, getKeyName(table, key)).Result() + if err != nil { + return + } + var encodedJson = make([]byte, 0) + err, encodedJson = mergeEncodedJsonWithMap(initialEncodedJson, values) + if err != nil { + return + } + return r.client.Set(ctx, getKeyName(table, key), string(encodedJson), 0).Err() + } + } + return +} + +func mergeEncodedJsonWithMap(stringReply string, values map[string][]byte) (err error, data []byte) { + curVal := map[string][]byte{} + err = json.Unmarshal([]byte(stringReply), &curVal) + if err != nil { + return + } + for k, v := range values { + curVal[k] = v + } + data, err = json.Marshal(curVal) + return +} + +func jsonEscape(bytes []byte) string { + return fmt.Sprintf("\"%s\"", string(bytes)) +} + +func getFieldJsonPath(fieldName string) string { + return fmt.Sprintf("$.%s", fieldName) +} + +func getKeyName(table string, key string) string { + return table + "/" + key +} + +func (r *redis) Insert(ctx context.Context, table string, key string, values map[string][]byte) (err error) { + data, err := json.Marshal(values) + if err != nil { + return err + } + switch r.datatype { + case JSON_DATATYPE: + err = r.client.Do(ctx, JSON_SET, getKeyName(table, key), ".", string(data)).Err() + case HASH_DATATYPE: + args := make([]interface{}, 0, 2*len(values)+2) + args = append(args, HSET, getKeyName(table, key)) + for fieldName, bytes := range values { + args = append(args, fieldName, string(bytes)) + } + err = r.client.Do(ctx, args...).Err() + case STRING_DATATYPE: + fallthrough + default: + err = r.client.Set(ctx, getKeyName(table, key), string(data), 0).Err() + } + return +} + +func (r *redis) Delete(ctx context.Context, table string, key string) error { + return r.client.Del(ctx, getKeyName(table, key)).Err() +} + +type redisCreator struct{} + +func (r redisCreator) Create(p *properties.Properties) (ycsb.DB, error) { + rds := &redis{} + + mode := p.GetString(redisMode, redisModeDefault) + switch mode { + case "cluster": + rds.client = goredis.NewClusterClient(getOptionsCluster(p)) + + if p.GetBool(prop.DropData, prop.DropDataDefault) { + err := rds.client.FlushDB(context.Background()).Err() + if err != nil { + return nil, err + } + } + case "single": + fallthrough + default: + mode = "single" + rds.client = goredis.NewClient(getOptionsSingle(p)) + + if p.GetBool(prop.DropData, prop.DropDataDefault) { + err := rds.client.FlushDB(context.Background()).Err() + if err != nil { + return nil, err + } + } + } + rds.mode = mode + rds.datatype = p.GetString(redisDatatype, redisDatatypeDefault) + fmt.Println(fmt.Sprintf("Using the redis datatype: %s", rds.datatype)) + + return rds, nil +} + +const ( + redisMode = "redis.mode" + redisModeDefault = "single" + redisDatatype = "redis.datatype" + redisDatatypeDefault = "hash" + redisNetwork = "redis.network" + redisNetworkDefault = "tcp" + redisAddr = "redis.addr" + redisAddrDefault = "localhost:6379" + redisPassword = "redis.password" + redisDB = "redis.db" + redisMaxRedirects = "redis.max_redirects" + redisReadOnly = "redis.read_only" + redisRouteByLatency = "redis.route_by_latency" + redisRouteRandomly = "redis.route_randomly" + redisMaxRetries = "redis.max_retries" + redisMinRetryBackoff = "redis.min_retry_backoff" + redisMaxRetryBackoff = "redis.max_retry_backoff" + redisDialTimeout = "redis.dial_timeout" + redisReadTimeout = "redis.read_timeout" + redisWriteTimeout = "redis.write_timeout" + redisPoolSize = "redis.pool_size" + redisPoolSizeDefault = 0 + redisMinIdleConns = "redis.min_idle_conns" + redisMaxConnAge = "redis.max_conn_age" + redisPoolTimeout = "redis.pool_timeout" + redisIdleTimeout = "redis.idle_timeout" + redisIdleCheckFreq = "redis.idle_check_frequency" + redisTLSCA = "redis.tls_ca" + redisTLSCert = "redis.tls_cert" + redisTLSKey = "redis.tls_key" + redisTLSInsecureSkipVerify = "redis.tls_insecure_skip_verify" +) + +func parseTLS(p *properties.Properties) *tls.Config { + caPath, _ := p.Get(redisTLSCA) + certPath, _ := p.Get(redisTLSCert) + keyPath, _ := p.Get(redisTLSKey) + insecureSkipVerify := p.GetBool(redisTLSInsecureSkipVerify, false) + if certPath != "" && keyPath != "" { + config, err := util.CreateTLSConfig(caPath, certPath, keyPath, insecureSkipVerify) + if err == nil { + return config + } + } + + return nil +} + +func getOptionsSingle(p *properties.Properties) *goredis.Options { + opts := &goredis.Options{} + + opts.Addr = p.GetString(redisAddr, redisAddrDefault) + opts.DB = p.GetInt(redisDB, 0) + opts.Network = p.GetString(redisNetwork, redisNetworkDefault) + opts.Password, _ = p.Get(redisPassword) + opts.MaxRetries = p.GetInt(redisMaxRetries, 0) + opts.MinRetryBackoff = p.GetDuration(redisMinRetryBackoff, time.Millisecond*8) + opts.MaxRetryBackoff = p.GetDuration(redisMaxRetryBackoff, time.Millisecond*512) + opts.DialTimeout = p.GetDuration(redisDialTimeout, time.Second*5) + opts.ReadTimeout = p.GetDuration(redisReadTimeout, time.Second*3) + opts.WriteTimeout = p.GetDuration(redisWriteTimeout, opts.ReadTimeout) + opts.PoolSize = p.GetInt(redisPoolSize, redisPoolSizeDefault) + threadCount := p.MustGetInt("threadcount") + if opts.PoolSize == 0 { + opts.PoolSize = threadCount + fmt.Println(fmt.Sprintf("Setting %s=%d (from ) given you haven't specified a value.", redisPoolSize, opts.PoolSize)) + } + opts.MinIdleConns = p.GetInt(redisMinIdleConns, 0) + opts.MaxConnAge = p.GetDuration(redisMaxConnAge, 0) + opts.PoolTimeout = p.GetDuration(redisPoolTimeout, time.Second+opts.ReadTimeout) + opts.IdleTimeout = p.GetDuration(redisIdleTimeout, time.Minute*5) + opts.IdleCheckFrequency = p.GetDuration(redisIdleCheckFreq, time.Minute) + opts.TLSConfig = parseTLS(p) + + return opts +} + +func getOptionsCluster(p *properties.Properties) *goredis.ClusterOptions { + opts := &goredis.ClusterOptions{} + + addresses, _ := p.Get(redisAddr) + opts.Addrs = strings.Split(addresses, ";") + opts.MaxRedirects = p.GetInt(redisMaxRedirects, 0) + opts.ReadOnly = p.GetBool(redisReadOnly, false) + opts.RouteByLatency = p.GetBool(redisRouteByLatency, false) + opts.RouteRandomly = p.GetBool(redisRouteRandomly, false) + opts.Password, _ = p.Get(redisPassword) + opts.MaxRetries = p.GetInt(redisMaxRetries, 0) + opts.MinRetryBackoff = p.GetDuration(redisMinRetryBackoff, time.Millisecond*8) + opts.MaxRetryBackoff = p.GetDuration(redisMaxRetryBackoff, time.Millisecond*512) + opts.DialTimeout = p.GetDuration(redisDialTimeout, time.Second*5) + opts.ReadTimeout = p.GetDuration(redisReadTimeout, time.Second*3) + opts.WriteTimeout = p.GetDuration(redisWriteTimeout, opts.ReadTimeout) + opts.PoolSize = p.GetInt(redisPoolSize, redisPoolSizeDefault) + threadCount := p.MustGetInt("threadcount") + if opts.PoolSize == 0 { + opts.PoolSize = threadCount + fmt.Println(fmt.Sprintf("Setting %s=%d (from ) given you haven't specified a value.", redisPoolSize, opts.PoolSize)) + } + opts.MinIdleConns = p.GetInt(redisMinIdleConns, 0) + opts.MaxConnAge = p.GetDuration(redisMaxConnAge, 0) + opts.PoolTimeout = p.GetDuration(redisPoolTimeout, time.Second+opts.ReadTimeout) + opts.IdleTimeout = p.GetDuration(redisIdleTimeout, time.Minute*5) + opts.IdleCheckFrequency = p.GetDuration(redisIdleCheckFreq, time.Minute) + + opts.TLSConfig = parseTLS(p) + + return opts +} + +func init() { + ycsb.RegisterDBCreator("redis", redisCreator{}) +} diff --git a/go-ycsb/db/rocksdb/db.go b/go-ycsb/db/rocksdb/db.go new file mode 100644 index 000000000..52ac77e80 --- /dev/null +++ b/go-ycsb/db/rocksdb/db.go @@ -0,0 +1,265 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build rocksdb + +package rocksdb + +import ( + "context" + "fmt" + "os" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + "github.com/pingcap/go-ycsb/pkg/ycsb" + "github.com/tecbot/gorocksdb" +) + +// properties +const ( + rocksdbDir = "rocksdb.dir" + // DBOptions + rocksdbAllowConcurrentMemtableWrites = "rocksdb.allow_concurrent_memtable_writes" + rocsdbAllowMmapReads = "rocksdb.allow_mmap_reads" + rocksdbAllowMmapWrites = "rocksdb.allow_mmap_writes" + rocksdbArenaBlockSize = "rocksdb.arena_block_size" + rocksdbDBWriteBufferSize = "rocksdb.db_write_buffer_size" + rocksdbHardPendingCompactionBytesLimit = "rocksdb.hard_pending_compaction_bytes_limit" + rocksdbLevel0FileNumCompactionTrigger = "rocksdb.level0_file_num_compaction_trigger" + rocksdbLevel0SlowdownWritesTrigger = "rocksdb.level0_slowdown_writes_trigger" + rocksdbLevel0StopWritesTrigger = "rocksdb.level0_stop_writes_trigger" + rocksdbMaxBytesForLevelBase = "rocksdb.max_bytes_for_level_base" + rocksdbMaxBytesForLevelMultiplier = "rocksdb.max_bytes_for_level_multiplier" + rocksdbMaxTotalWalSize = "rocksdb.max_total_wal_size" + rocksdbMemtableHugePageSize = "rocksdb.memtable_huge_page_size" + rocksdbNumLevels = "rocksdb.num_levels" + rocksdbUseDirectReads = "rocksdb.use_direct_reads" + rocksdbUseFsync = "rocksdb.use_fsync" + rocksdbWriteBufferSize = "rocksdb.write_buffer_size" + rocksdbMaxWriteBufferNumber = "rocksdb.max_write_buffer_number" + // TableOptions/BlockBasedTable + rocksdbBlockSize = "rocksdb.block_size" + rocksdbBlockSizeDeviation = "rocksdb.block_size_deviation" + rocksdbCacheIndexAndFilterBlocks = "rocksdb.cache_index_and_filter_blocks" + rocksdbNoBlockCache = "rocksdb.no_block_cache" + rocksdbPinL0FilterAndIndexBlocksInCache = "rocksdb.pin_l0_filter_and_index_blocks_in_cache" + rocksdbWholeKeyFiltering = "rocksdb.whole_key_filtering" + rocksdbBlockRestartInterval = "rocksdb.block_restart_interval" + rocksdbFilterPolicy = "rocksdb.filter_policy" + rocksdbIndexType = "rocksdb.index_type" + rocksdbWALDir = "rocksdb.wal_dir" + // TODO: add more configurations +) + +type rocksDBCreator struct{} + +type rocksDB struct { + p *properties.Properties + + db *gorocksdb.DB + + r *util.RowCodec + bufPool *util.BufPool + + readOpts *gorocksdb.ReadOptions + writeOpts *gorocksdb.WriteOptions +} + +type contextKey string + +func (c rocksDBCreator) Create(p *properties.Properties) (ycsb.DB, error) { + dir := p.GetString(rocksdbDir, "/tmp/rocksdb") + + if p.GetBool(prop.DropData, prop.DropDataDefault) { + os.RemoveAll(dir) + } + + opts := getOptions(p) + + db, err := gorocksdb.OpenDb(opts, dir) + if err != nil { + return nil, err + } + + return &rocksDB{ + p: p, + db: db, + r: util.NewRowCodec(p), + bufPool: util.NewBufPool(), + readOpts: gorocksdb.NewDefaultReadOptions(), + writeOpts: gorocksdb.NewDefaultWriteOptions(), + }, nil +} + +func getTableOptions(p *properties.Properties) *gorocksdb.BlockBasedTableOptions { + tblOpts := gorocksdb.NewDefaultBlockBasedTableOptions() + + tblOpts.SetBlockSize(p.GetInt(rocksdbBlockSize, 4<<10)) + tblOpts.SetBlockSizeDeviation(p.GetInt(rocksdbBlockSizeDeviation, 10)) + tblOpts.SetCacheIndexAndFilterBlocks(p.GetBool(rocksdbCacheIndexAndFilterBlocks, false)) + tblOpts.SetNoBlockCache(p.GetBool(rocksdbNoBlockCache, false)) + tblOpts.SetPinL0FilterAndIndexBlocksInCache(p.GetBool(rocksdbPinL0FilterAndIndexBlocksInCache, false)) + tblOpts.SetWholeKeyFiltering(p.GetBool(rocksdbWholeKeyFiltering, true)) + tblOpts.SetBlockRestartInterval(p.GetInt(rocksdbBlockRestartInterval, 16)) + + if b := p.GetString(rocksdbFilterPolicy, ""); len(b) > 0 { + if b == "rocksdb.BuiltinBloomFilter" { + const defaultBitsPerKey = 10 + tblOpts.SetFilterPolicy(gorocksdb.NewBloomFilter(defaultBitsPerKey)) + } + } + + indexType := p.GetString(rocksdbIndexType, "kBinarySearch") + if indexType == "kBinarySearch" { + tblOpts.SetIndexType(gorocksdb.KBinarySearchIndexType) + } else if indexType == "kHashSearch" { + tblOpts.SetIndexType(gorocksdb.KHashSearchIndexType) + } else if indexType == "kTwoLevelIndexSearch" { + tblOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType) + } + + return tblOpts +} + +func getOptions(p *properties.Properties) *gorocksdb.Options { + opts := gorocksdb.NewDefaultOptions() + opts.SetCreateIfMissing(true) + + opts.SetAllowConcurrentMemtableWrites(p.GetBool(rocksdbAllowConcurrentMemtableWrites, true)) + opts.SetAllowMmapReads(p.GetBool(rocsdbAllowMmapReads, false)) + opts.SetAllowMmapWrites(p.GetBool(rocksdbAllowMmapWrites, false)) + opts.SetArenaBlockSize(p.GetInt(rocksdbArenaBlockSize, 0)) + opts.SetDbWriteBufferSize(p.GetInt(rocksdbDBWriteBufferSize, 0)) + opts.SetHardPendingCompactionBytesLimit(p.GetUint64(rocksdbHardPendingCompactionBytesLimit, 256<<30)) + opts.SetLevel0FileNumCompactionTrigger(p.GetInt(rocksdbLevel0FileNumCompactionTrigger, 4)) + opts.SetLevel0SlowdownWritesTrigger(p.GetInt(rocksdbLevel0SlowdownWritesTrigger, 20)) + opts.SetLevel0StopWritesTrigger(p.GetInt(rocksdbLevel0StopWritesTrigger, 36)) + opts.SetMaxBytesForLevelBase(p.GetUint64(rocksdbMaxBytesForLevelBase, 256<<20)) + opts.SetMaxBytesForLevelMultiplier(p.GetFloat64(rocksdbMaxBytesForLevelMultiplier, 10)) + opts.SetMaxTotalWalSize(p.GetUint64(rocksdbMaxTotalWalSize, 0)) + opts.SetMemtableHugePageSize(p.GetInt(rocksdbMemtableHugePageSize, 0)) + opts.SetNumLevels(p.GetInt(rocksdbNumLevels, 7)) + opts.SetUseDirectReads(p.GetBool(rocksdbUseDirectReads, false)) + opts.SetUseFsync(p.GetBool(rocksdbUseFsync, false)) + opts.SetWriteBufferSize(p.GetInt(rocksdbWriteBufferSize, 64<<20)) + opts.SetMaxWriteBufferNumber(p.GetInt(rocksdbMaxWriteBufferNumber, 2)) + opts.SetWalDir(p.GetString(rocksdbWALDir, "")) + + opts.SetBlockBasedTableFactory(getTableOptions(p)) + + return opts +} + +func (db *rocksDB) Close() error { + db.db.Close() + return nil +} + +func (db *rocksDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (db *rocksDB) CleanupThread(_ context.Context) { +} + +func (db *rocksDB) getRowKey(table string, key string) []byte { + return util.Slice(fmt.Sprintf("%s:%s", table, key)) +} + +func cloneValue(v *gorocksdb.Slice) []byte { + return append([]byte(nil), v.Data()...) +} + +func (db *rocksDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + value, err := db.db.Get(db.readOpts, db.getRowKey(table, key)) + if err != nil { + return nil, err + } + defer value.Free() + + return db.r.Decode(cloneValue(value), fields) +} + +func (db *rocksDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + res := make([]map[string][]byte, count) + it := db.db.NewIterator(db.readOpts) + defer it.Close() + + rowStartKey := db.getRowKey(table, startKey) + + it.Seek(rowStartKey) + i := 0 + for it = it; it.Valid() && i < count; it.Next() { + value := it.Value() + m, err := db.r.Decode(cloneValue(value), fields) + if err != nil { + return nil, err + } + res[i] = m + i++ + } + + if err := it.Err(); err != nil { + return nil, err + } + + return res, nil +} + +func (db *rocksDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + m, err := db.Read(ctx, table, key, nil) + if err != nil { + return err + } + + for field, value := range values { + m[field] = value + } + + buf := db.bufPool.Get() + defer db.bufPool.Put(buf) + + buf, err = db.r.Encode(buf, m) + if err != nil { + return err + } + + rowKey := db.getRowKey(table, key) + + return db.db.Put(db.writeOpts, rowKey, buf) +} + +func (db *rocksDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + rowKey := db.getRowKey(table, key) + + buf := db.bufPool.Get() + defer db.bufPool.Put(buf) + + buf, err := db.r.Encode(buf, values) + if err != nil { + return err + } + return db.db.Put(db.writeOpts, rowKey, buf) +} + +func (db *rocksDB) Delete(ctx context.Context, table string, key string) error { + rowKey := db.getRowKey(table, key) + + return db.db.Delete(db.writeOpts, rowKey) +} + +func init() { + ycsb.RegisterDBCreator("rocksdb", rocksDBCreator{}) +} diff --git a/go-ycsb/db/rocksdb/doc.go b/go-ycsb/db/rocksdb/doc.go new file mode 100644 index 000000000..bf741cd89 --- /dev/null +++ b/go-ycsb/db/rocksdb/doc.go @@ -0,0 +1,3 @@ +package rocksdb + +// If you want to use RocksDB, please follow [INSTALL](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) to install it. diff --git a/go-ycsb/db/spanner/db.go b/go-ycsb/db/spanner/db.go new file mode 100644 index 000000000..a5483e262 --- /dev/null +++ b/go-ycsb/db/spanner/db.go @@ -0,0 +1,358 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spanner + +import ( + "bytes" + "context" + "fmt" + "os" + "os/user" + "path" + "regexp" + "strings" + + "cloud.google.com/go/spanner" + database "cloud.google.com/go/spanner/admin/database/apiv1" + "google.golang.org/api/iterator" + + adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" + + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +const ( + spannerDBName = "spanner.db" + spannerCredentials = "spanner.credentials" +) + +type spannerCreator struct { +} + +type spannerDB struct { + p *properties.Properties + client *spanner.Client + verbose bool +} + +type contextKey string + +const stateKey = contextKey("spannerDB") + +type spannerState struct { +} + +func (c spannerCreator) Create(p *properties.Properties) (ycsb.DB, error) { + d := new(spannerDB) + d.p = p + + credentials := p.GetString(spannerCredentials, "") + if len(credentials) == 0 { + // no credentials provided, try using ~/.spanner/credentials.json" + usr, err := user.Current() + if err != nil { + return nil, err + } + credentials = path.Join(usr.HomeDir, ".spanner/credentials.json") + } + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credentials) + + ctx := context.Background() + + adminClient, err := database.NewDatabaseAdminClient(ctx) + if err != nil { + return nil, err + } + defer adminClient.Close() + + dbName := p.GetString(spannerDBName, "") + if len(dbName) == 0 { + return nil, fmt.Errorf("must provide a database like projects/xxxx/instances/xxxx/databases/xxx") + } + + d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) + + _, err = d.createDatabase(ctx, adminClient, dbName) + if err != nil { + return nil, err + } + + client, err := spanner.NewClient(ctx, dbName) + if err != nil { + return nil, err + } + d.client = client + + if err = d.createTable(ctx, adminClient, dbName); err != nil { + return nil, err + } + + return d, nil +} + +func (db *spannerDB) createDatabase(ctx context.Context, adminClient *database.DatabaseAdminClient, dbName string) (string, error) { + matches := regexp.MustCompile("^(.*)/databases/(.*)$").FindStringSubmatch(dbName) + if matches == nil || len(matches) != 3 { + return "", fmt.Errorf("Invalid database id %s", dbName) + } + + database, err := adminClient.GetDatabase(ctx, &adminpb.GetDatabaseRequest{ + Name: dbName, + }) + if err != nil { + return "", err + } + + if database.State == adminpb.Database_STATE_UNSPECIFIED { + op, err := adminClient.CreateDatabase(ctx, &adminpb.CreateDatabaseRequest{ + Parent: matches[1], + CreateStatement: "CREATE DATABASE `" + matches[2] + "`", + ExtraStatements: []string{}, + }) + if err != nil { + return "", err + } + if _, err := op.Wait(ctx); err != nil { + return "", err + } + } + + return matches[2], nil +} + +func (db *spannerDB) tableExisted(ctx context.Context, table string) (bool, error) { + stmt := spanner.NewStatement(`SELECT t.table_name FROM information_schema.tables AS t + WHERE t.table_catalog = '' AND t.table_schema = '' AND t.table_name = @name`) + stmt.Params["name"] = table + iter := db.client.Single().Query(ctx, stmt) + defer iter.Stop() + + found := false + for { + _, err := iter.Next() + if err == iterator.Done { + break + } + + if err != nil { + return false, err + } + found = true + break + } + + return found, nil +} + +func (db *spannerDB) createTable(ctx context.Context, adminClient *database.DatabaseAdminClient, dbName string) error { + tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) + fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) + fieldLength := db.p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) + + existed, err := db.tableExisted(ctx, tableName) + if err != nil { + return err + } + + if db.p.GetBool(prop.DropData, prop.DropDataDefault) && existed { + op, err := adminClient.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ + Database: dbName, + Statements: []string{ + fmt.Sprintf("DROP TABLE %s", tableName), + }, + }) + if err != nil { + return err + } + + if err = op.Wait(ctx); err != nil { + return err + } + existed = false + } + + if existed { + return nil + } + + buf := new(bytes.Buffer) + s := fmt.Sprintf("CREATE TABLE %s (YCSB_KEY STRING(%d)", tableName, fieldLength) + buf.WriteString(s) + + for i := int64(0); i < fieldCount; i++ { + buf.WriteString(fmt.Sprintf(", FIELD%d STRING(%d)", i, fieldLength)) + } + + buf.WriteString(") PRIMARY KEY (YCSB_KEY)") + + op, err := adminClient.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ + Database: dbName, + Statements: []string{ + buf.String(), + }, + }) + if err != nil { + return err + } + + if err := op.Wait(ctx); err != nil { + return err + } + + return nil +} + +func (db *spannerDB) Close() error { + if db.client == nil { + return nil + } + + db.client.Close() + return nil +} + +func (db *spannerDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + state := &spannerState{} + + return context.WithValue(ctx, stateKey, state) +} + +func (db *spannerDB) CleanupThread(ctx context.Context) { + // state := ctx.Value(stateKey).(*spanner) +} + +func (db *spannerDB) queryRows(ctx context.Context, stmt spanner.Statement, count int) ([]map[string][]byte, error) { + if db.verbose { + fmt.Printf("%s %v\n", stmt.SQL, stmt.Params) + } + + iter := db.client.Single().Query(ctx, stmt) + defer iter.Stop() + + vs := make([]map[string][]byte, 0, count) + for { + row, err := iter.Next() + if err == iterator.Done { + break + } + + if err != nil { + return nil, err + } + + rowSize := row.Size() + m := make(map[string][]byte, rowSize) + dest := make([]interface{}, rowSize) + for i := 0; i < rowSize; i++ { + v := new(spanner.NullString) + dest[i] = v + } + + if err := row.Columns(dest...); err != nil { + return nil, err + } + + for i := 0; i < rowSize; i++ { + v := dest[i].(*spanner.NullString) + if v.Valid { + m[row.ColumnName(i)] = util.Slice(v.StringVal) + } + } + + vs = append(vs, m) + } + + return vs, nil +} + +func (db *spannerDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + var query string + if len(fields) == 0 { + query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY = @key`, table) + } else { + query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY = @key`, strings.Join(fields, ","), table) + } + + stmt := spanner.NewStatement(query) + stmt.Params["key"] = key + + rows, err := db.queryRows(ctx, stmt, 1) + + if err != nil { + return nil, err + } else if len(rows) == 0 { + return nil, nil + } + + return rows[0], nil +} + +func (db *spannerDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + var query string + if len(fields) == 0 { + query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY >= @key LIMIT @limit`, table) + } else { + query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY >= @key LIMIT @limit`, strings.Join(fields, ","), table) + } + + stmt := spanner.NewStatement(query) + stmt.Params["key"] = startKey + stmt.Params["limit"] = count + + rows, err := db.queryRows(ctx, stmt, count) + + return rows, err +} + +func createMutations(key string, mutations map[string][]byte) ([]string, []interface{}) { + keys := make([]string, 0, 1+len(mutations)) + values := make([]interface{}, 0, 1+len(mutations)) + keys = append(keys, "YCSB_KEY") + values = append(values, key) + + for key, value := range mutations { + keys = append(keys, key) + values = append(values, util.String(value)) + } + + return keys, values +} + +func (db *spannerDB) Update(ctx context.Context, table string, key string, mutations map[string][]byte) error { + keys, values := createMutations(key, mutations) + m := spanner.Update(table, keys, values) + _, err := db.client.Apply(ctx, []*spanner.Mutation{m}) + return err +} + +func (db *spannerDB) Insert(ctx context.Context, table string, key string, mutations map[string][]byte) error { + keys, values := createMutations(key, mutations) + m := spanner.InsertOrUpdate(table, keys, values) + _, err := db.client.Apply(ctx, []*spanner.Mutation{m}) + return err +} + +func (db *spannerDB) Delete(ctx context.Context, table string, key string) error { + m := spanner.Delete(table, spanner.Key{key}) + _, err := db.client.Apply(ctx, []*spanner.Mutation{m}) + return err +} + +func init() { + ycsb.RegisterDBCreator("spanner", spannerCreator{}) +} diff --git a/go-ycsb/db/sqlite/db.go b/go-ycsb/db/sqlite/db.go new file mode 100644 index 000000000..0e06ab4e5 --- /dev/null +++ b/go-ycsb/db/sqlite/db.go @@ -0,0 +1,400 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build libsqlite3 + +package sqlite + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "net/url" + "os" + "strings" + "time" + + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + + "github.com/magiconair/properties" + // sqlite package + "github.com/mattn/go-sqlite3" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// Sqlite properties +const ( + sqliteDBPath = "sqlite.db" + sqliteMode = "sqlite.mode" + sqliteJournalMode = "sqlite.journalmode" + sqliteCache = "sqlite.cache" + sqliteMaxOpenConns = "sqlite.maxopenconns" + sqliteMaxIdleConns = "sqlite.maxidleconns" + sqliteOptimistic = "sqlite.optimistic" + sqliteOptimisticBackoffMs = "sqlite.optimistic_backoff_ms" +) + +type sqliteCreator struct { +} + +type sqliteDB struct { + p *properties.Properties + db *sql.DB + verbose bool + optimistic bool + backoffMs int + + bufPool *util.BufPool +} + +func (c sqliteCreator) Create(p *properties.Properties) (ycsb.DB, error) { + d := new(sqliteDB) + d.p = p + + dbPath := p.GetString(sqliteDBPath, "/tmp/sqlite.db") + + if p.GetBool(prop.DropData, prop.DropDataDefault) { + os.RemoveAll(dbPath) + } + + mode := p.GetString(sqliteMode, "rwc") + journalMode := p.GetString(sqliteJournalMode, "WAL") + cache := p.GetString(sqliteCache, "shared") + maxOpenConns := p.GetInt(sqliteMaxOpenConns, 1) + maxIdleConns := p.GetInt(sqliteMaxIdleConns, 2) + + v := url.Values{} + v.Set("cache", cache) + v.Set("mode", mode) + v.Set("_journal_mode", journalMode) + dsn := fmt.Sprintf("file:%s?%s", dbPath, v.Encode()) + var err error + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, err + } + + db.SetMaxOpenConns(maxOpenConns) + db.SetMaxIdleConns(maxIdleConns) + + d.optimistic = p.GetBool(sqliteOptimistic, false) + d.backoffMs = p.GetInt(sqliteOptimisticBackoffMs, 5) + d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) + d.db = db + + d.bufPool = util.NewBufPool() + + if err := d.createTable(); err != nil { + return nil, err + } + + return d, nil +} + +func (db *sqliteDB) createTable() error { + tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) + + fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) + fieldLength := db.p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) + + buf := new(bytes.Buffer) + s := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (YCSB_KEY VARCHAR(64) PRIMARY KEY", tableName) + buf.WriteString(s) + + for i := int64(0); i < fieldCount; i++ { + buf.WriteString(fmt.Sprintf(", FIELD%d VARCHAR(%d)", i, fieldLength)) + } + + buf.WriteString(");") + + if db.verbose { + fmt.Println(buf.String()) + } + + _, err := db.db.Exec(buf.String()) + return err +} + +func (db *sqliteDB) Close() error { + if db.db == nil { + return nil + } + + return db.db.Close() +} + +func (db *sqliteDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (db *sqliteDB) CleanupThread(ctx context.Context) { + +} + +func (db *sqliteDB) optimisticTx(ctx context.Context, f func(tx *sql.Tx) error) error { + for { + tx, err := db.db.BeginTx(ctx, nil) + if err != nil { + return err + } + + if err = f(tx); err != nil { + tx.Rollback() + return err + } + + err = tx.Commit() + if err != nil && db.optimistic { + if err, ok := err.(sqlite3.Error); ok && (err.Code == sqlite3.ErrBusy || + err.ExtendedCode == sqlite3.ErrIoErrUnlock) { + time.Sleep(time.Duration(db.backoffMs) * time.Millisecond) + continue + } + } + return err + } +} + +func (db *sqliteDB) doQueryRows(ctx context.Context, tx *sql.Tx, query string, count int, args ...interface{}) ([]map[string][]byte, error) { + if db.verbose { + fmt.Printf("%s %v\n", query, args) + } + + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + return nil, err + } + + vs := make([]map[string][]byte, 0, count) + for rows.Next() { + m := make(map[string][]byte, len(cols)) + dest := make([]interface{}, len(cols)) + for i := 0; i < len(cols); i++ { + v := new([]byte) + dest[i] = v + } + if err = rows.Scan(dest...); err != nil { + return nil, err + } + + for i, v := range dest { + m[cols[i]] = *v.(*[]byte) + } + + vs = append(vs, m) + } + + return vs, rows.Err() +} + +func (db *sqliteDB) doRead(ctx context.Context, tx *sql.Tx, table string, key string, fields []string) (map[string][]byte, error) { + var query string + if len(fields) == 0 { + query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY = ?`, table) + } else { + query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY = ?`, strings.Join(fields, ","), table) + } + + rows, err := db.doQueryRows(ctx, tx, query, 1, key) + + if err != nil { + return nil, err + } else if len(rows) == 0 { + return nil, nil + } + + return rows[0], nil +} + +func (db *sqliteDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + var output map[string][]byte + err := db.optimisticTx(ctx, func(tx *sql.Tx) error { + res, err := db.doRead(ctx, tx, table, key, fields) + output = res + return err + }) + return output, err +} + +func (db *sqliteDB) doScan(ctx context.Context, tx *sql.Tx, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + var query string + if len(fields) == 0 { + query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY >= ? LIMIT ?`, table) + } else { + query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY >= ? LIMIT ?`, strings.Join(fields, ","), table) + } + + rows, err := db.doQueryRows(ctx, tx, query, count, startKey, count) + + return rows, err +} + +func (db *sqliteDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + var output []map[string][]byte + err := db.optimisticTx(ctx, func(tx *sql.Tx) error { + res, err := db.doScan(ctx, tx, table, startKey, count, fields) + output = res + return err + }) + return output, err +} + +func (db *sqliteDB) doUpdate(ctx context.Context, tx *sql.Tx, table string, key string, values map[string][]byte) error { + buf := bytes.NewBuffer(db.bufPool.Get()) + defer func() { + db.bufPool.Put(buf.Bytes()) + }() + + buf.WriteString("UPDATE ") + buf.WriteString(table) + buf.WriteString(" SET ") + firstField := true + pairs := util.NewFieldPairs(values) + args := make([]interface{}, 0, len(values)+1) + for _, p := range pairs { + if firstField { + firstField = false + } else { + buf.WriteString(", ") + } + + buf.WriteString(p.Field) + buf.WriteString(`= ?`) + args = append(args, p.Value) + } + buf.WriteString(" WHERE YCSB_KEY = ?") + + args = append(args, key) + + _, err := tx.ExecContext(ctx, buf.String(), args...) + return err +} + +func (db *sqliteDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + return db.optimisticTx(ctx, func(tx *sql.Tx) error { + return db.doUpdate(ctx, tx, table, key, values) + }) +} + +func (db *sqliteDB) doInsert(ctx context.Context, tx *sql.Tx, table string, key string, values map[string][]byte) error { + args := make([]interface{}, 0, 1+len(values)) + args = append(args, key) + + buf := bytes.NewBuffer(db.bufPool.Get()) + defer func() { + db.bufPool.Put(buf.Bytes()) + }() + + buf.WriteString("INSERT OR IGNORE INTO ") + buf.WriteString(table) + buf.WriteString(" (YCSB_KEY") + + pairs := util.NewFieldPairs(values) + for _, p := range pairs { + args = append(args, p.Value) + buf.WriteString(" ,") + buf.WriteString(p.Field) + } + buf.WriteString(") VALUES (?") + + for i := 0; i < len(pairs); i++ { + buf.WriteString(" ,?") + } + + buf.WriteByte(')') + + _, err := tx.ExecContext(ctx, buf.String(), args...) + if err != nil && db.verbose { + fmt.Printf("error(doInsert): %s: %+v\n", buf.String(), err) + } + return err +} + +func (db *sqliteDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + return db.optimisticTx(ctx, func(tx *sql.Tx) error { return db.doInsert(ctx, tx, table, key, values) }) +} + +func (db *sqliteDB) doDelete(ctx context.Context, tx *sql.Tx, table string, key string) error { + query := fmt.Sprintf(`DELETE FROM %s WHERE YCSB_KEY = ?`, table) + _, err := tx.ExecContext(ctx, query, key) + return err +} + +func (db *sqliteDB) Delete(ctx context.Context, table string, key string) error { + return db.optimisticTx(ctx, func(tx *sql.Tx) error { return db.doDelete(ctx, tx, table, key) }) +} + +func (db *sqliteDB) BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) error { + return db.optimisticTx(ctx, func(tx *sql.Tx) error { + for i := 0; i < len(keys); i++ { + err := db.doInsert(ctx, tx, table, keys[i], values[i]) + if err != nil { + return err + } + } + return nil + }) +} + +func (db *sqliteDB) BatchRead(ctx context.Context, table string, keys []string, fields []string) ([]map[string][]byte, error) { + var output []map[string][]byte + err := db.optimisticTx(ctx, func(tx *sql.Tx) error { + for i := 0; i < len(keys); i++ { + res, err := db.doRead(ctx, tx, table, keys[i], fields) + if err != nil { + return err + } + output = append(output, res) + } + return nil + }) + return output, err +} + +func (db *sqliteDB) BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) error { + return db.optimisticTx(ctx, func(tx *sql.Tx) error { + for i := 0; i < len(keys); i++ { + err := db.doUpdate(ctx, tx, table, keys[i], values[i]) + if err != nil { + return err + } + } + return nil + }) +} + +func (db *sqliteDB) BatchDelete(ctx context.Context, table string, keys []string) error { + return db.optimisticTx(ctx, func(tx *sql.Tx) error { + for i := 0; i < len(keys); i++ { + err := db.doDelete(ctx, tx, table, keys[i]) + if err != nil { + return err + } + } + return nil + }) +} + +func init() { + ycsb.RegisterDBCreator("sqlite", sqliteCreator{}) +} + +var _ ycsb.BatchDB = (*sqliteDB)(nil) diff --git a/go-ycsb/db/sqlite/doc.go b/go-ycsb/db/sqlite/doc.go new file mode 100644 index 000000000..a55792fe6 --- /dev/null +++ b/go-ycsb/db/sqlite/doc.go @@ -0,0 +1,3 @@ +package sqlite + +// If you want to use sqlite, you must install [client](https://www.sqlite.org/download.html) libraray. diff --git a/go-ycsb/db/tinydb/db.go b/go-ycsb/db/tinydb/db.go new file mode 100644 index 000000000..25f20b80c --- /dev/null +++ b/go-ycsb/db/tinydb/db.go @@ -0,0 +1,160 @@ +package tinydb + +import ( + "context" + "fmt" + + "github.com/magiconair/properties" + "github.com/pingcap-incubator/tinykv/kv/config" + "github.com/pingcap-incubator/tinykv/kv/server" + standalone_storage "github.com/pingcap-incubator/tinykv/kv/storage/standalone_storage" + "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" + "github.com/pingcap/go-ycsb/pkg/util" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +type tinydbCreator struct { +} + +type tinyDB struct { + p *properties.Properties + s *standalone_storage.StandAloneStorage + db *server.Server + + r *util.RowCodec + bufPool *util.BufPool +} + +func (c tinydbCreator) Create(p *properties.Properties) (ycsb.DB, error) { + fmt.Print("1\n") + conf := config.NewTestConfig() + s := standalone_storage.NewStandAloneStorage(conf) + s.Start() + server := server.NewServer(s) + + return &tinyDB{ + p: p, + s: s, + db: server, + r: util.NewRowCodec(p), + bufPool: util.NewBufPool(), + }, nil +} + +func (db *tinyDB) Close() error { + return db.s.Stop() +} + +func (db *tinyDB) InitThread(ctx context.Context, _ int, _ int) context.Context { + return ctx +} + +func (db *tinyDB) CleanupThread(_ context.Context) { +} + +func (db *tinyDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { + req := &kvrpcpb.RawGetRequest{ + Key: []byte(key), + Cf: table, + } + resp, _ := db.db.RawGet(nil, req) + value := resp.Value + return db.r.Decode(value, fields) +} + +func (db *tinyDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { + res := make([]map[string][]byte, count) + req := &kvrpcpb.RawScanRequest{ + StartKey: []byte(startKey), + Limit: uint32(count), + Cf: table, + } + resp, _ := db.db.RawScan(nil, req) + kvs := resp.Kvs + for i := 0; i < count; i++ { + value := kvs[i].Value + m, err := db.r.Decode(value, fields) + if err != nil { + return nil, err + } + res[i] = m + } + return res, nil +} + +func (db *tinyDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { + getReq := &kvrpcpb.RawGetRequest{ + Key: []byte(key), + Cf: table, + } + resp, err := db.db.RawGet(nil, getReq) + if err != nil { + return err + } + value := resp.Value + data, err := db.r.Decode(value, nil) + if err != nil { + return err + } + for field, value := range values { + data[field] = value + } + buf := db.bufPool.Get() + defer func() { + db.bufPool.Put(buf) + }() + + buf, err = db.r.Encode(buf, data) + if err != nil { + return err + } + putReq := &kvrpcpb.RawPutRequest{ + Key: []byte(key), + Value: buf, + Cf: table, + } + + _, err = db.db.RawPut(nil, putReq) + if err != nil { + return err + } + return nil +} + +func (db *tinyDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { + buf := db.bufPool.Get() + defer func() { + db.bufPool.Put(buf) + }() + + buf, err := db.r.Encode(buf, values) + if err != nil { + return err + } + putReq := &kvrpcpb.RawPutRequest{ + Key: []byte(key), + Value: buf, + Cf: table, + } + + _, err = db.db.RawPut(nil, putReq) + if err != nil { + return err + } + return nil +} + +func (db *tinyDB) Delete(ctx context.Context, table string, key string) error { + req := &kvrpcpb.RawDeleteRequest{ + Key: []byte(key), + Cf: table, + } + _, err := db.db.RawDelete(nil, req) + + return err + +} + +func init() { + ycsb.RegisterDBCreator("tinydb", tinydbCreator{}) +} diff --git a/go-ycsb/fdb-dockerfile b/go-ycsb/fdb-dockerfile new file mode 100644 index 000000000..8022dfce9 --- /dev/null +++ b/go-ycsb/fdb-dockerfile @@ -0,0 +1,37 @@ +FROM golang:1.13.6-stretch + +ENV GOPATH /go + +RUN apt-get update \ + && apt-get install -y \ + wget \ + dpkg \ + python \ + git \ + net-tools + +RUN cd / \ + && wget https://www.foundationdb.org/downloads/6.2.11/ubuntu/installers/foundationdb-clients_6.2.11-1_amd64.deb \ + && dpkg -i foundationdb-clients_6.2.11-1_amd64.deb + +ADD . /go/src/github.com/pingcap/go-ycsb + +WORKDIR /go/src/github.com/pingcap/go-ycsb + +RUN GO111MODULE=on go build -tags "foundationdb" -o /go-ycsb ./cmd/* + +FROM ubuntu:18.04 + +RUN apt-get update \ + && apt-get install -y dpkg + +COPY --from=0 /foundationdb-clients_6.2.11-1_amd64.deb /foundationdb-clients_6.2.11-1_amd64.deb +RUN dpkg -i foundationdb-clients_6.2.11-1_amd64.deb + +COPY --from=0 /go-ycsb /go-ycsb + +ADD workloads /workloads + +EXPOSE 6060 + +ENTRYPOINT [ "/go-ycsb" ] diff --git a/go-ycsb/go.mod b/go-ycsb/go.mod new file mode 100644 index 000000000..55b4866c9 --- /dev/null +++ b/go-ycsb/go.mod @@ -0,0 +1,176 @@ +module github.com/pingcap/go-ycsb + +require ( + cloud.google.com/go/spanner v1.1.0 + github.com/HdrHistogram/hdrhistogram-go v1.1.2 + github.com/XiaoMi/pegasus-go-client v0.0.0-20181029071519-9400942c5d1c + github.com/aerospike/aerospike-client-go v1.35.2 + github.com/apple/foundationdb/bindings/go v0.0.0-20200112054404-407dc0907f4f + github.com/boltdb/bolt v1.3.1 + github.com/cenkalti/backoff/v4 v4.1.3 + github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e + github.com/dgraph-io/badger v1.6.0 + github.com/elastic/go-elasticsearch/v8 v8.3.0 + github.com/fortytw2/leaktest v1.3.0 // indirect + github.com/go-ini/ini v1.49.0 // indirect + github.com/go-redis/redis/v9 v9.0.0-beta.1 + github.com/go-sql-driver/mysql v1.6.0 + github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b + github.com/lib/pq v1.1.1 + github.com/magiconair/properties v1.8.0 + github.com/mattn/go-sqlite3 v2.0.1+incompatible + github.com/minio/minio-go v6.0.14+incompatible + github.com/olekukonko/tablewriter v0.0.5 + github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/cobra v1.0.0 + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c + go.mongodb.org/mongo-driver v1.5.1 + google.golang.org/api v0.87.0 + google.golang.org/genproto v0.0.0-20220718134204-073382fd740c +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.16.16 + github.com/aws/aws-sdk-go-v2/config v1.17.8 + github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 + github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.26 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1 + github.com/pingcap-incubator/tinykv v0.0.0-20200514052412-e01d729bd45c + go.etcd.io/etcd/client/pkg/v3 v3.5.6 + go.etcd.io/etcd/client/v3 v3.5.6 +) + +// require github.com/pingcap/pd v1.1.0-beta.0.20200106144140-f5a7aa985497 // indirect + +require ( + cloud.google.com/go v0.103.0 // indirect + cloud.google.com/go/compute v1.7.0 // indirect + github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/Connor1996/badger v1.5.1-0.20220222053432-2d2cbf472c77 // indirect + github.com/DataDog/zstd v1.4.1 // indirect + github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d // indirect + github.com/aws/aws-sdk-go v1.34.28 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.21 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 // indirect + github.com/aws/smithy-go v1.13.3 // indirect + github.com/benbjohnson/clock v1.1.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 // indirect + github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect + github.com/dgraph-io/ristretto v0.0.1 // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-stack/stack v1.8.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/google/go-cmp v0.5.8 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect + github.com/googleapis/gax-go/v2 v2.4.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jmhodges/levigo v1.0.0 // indirect + github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect + github.com/klauspost/compress v1.9.5 // indirect + github.com/klauspost/cpuid v1.2.1 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/ncw/directio v1.0.4 // indirect + github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac // indirect + github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99 // indirect + github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 // indirect + github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect + github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect + github.com/pingcap/tidb v1.1.0-beta.0.20200309111804-d8264d47f760 // indirect + github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60 // indirect + github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sirupsen/logrus v1.6.0 // indirect + github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/pflag v1.0.3 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.0.2 // indirect + github.com/xdg-go/stringprep v1.0.2 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 // indirect + go.etcd.io/etcd/api/v3 v3.5.6 // indirect + go.opencensus.io v0.23.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/goleak v1.1.12 // indirect + go.uber.org/multierr v1.7.0 // indirect + go.uber.org/zap v1.20.0 // indirect + golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect + golang.org/x/exp v0.0.0-20220428152302-39d4317da171 // indirect + golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect + golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 // indirect + golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect + golang.org/x/sys v0.2.0 // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect + golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/grpc v1.48.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.42.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect +) + +replace github.com/apache/thrift => github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d + +replace google.golang.org/grpc => google.golang.org/grpc v1.25.1 + +replace github.com/pingcap-incubator/tinykv => ../ + +replace github.com/jmhodges/levigo => ../levigo + +replace github.com/tecbot/gorocksdb => ../gorocksdb + +replace github.com/pingcap/tidb => github.com/pingcap-incubator/tinysql v0.0.0-20200518090433-a7d00f9e6aa7 + +go 1.18 diff --git a/go-ycsb/go.sum b/go-ycsb/go.sum new file mode 100644 index 000000000..371d2dde1 --- /dev/null +++ b/go-ycsb/go.sum @@ -0,0 +1,1234 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.103.0 h1:YXtxp9ymmZjlGzxV7VrYQ8aaQuAgcqxSy6YhDX4I458= +cloud.google.com/go v0.103.0/go.mod h1:vwLx1nqLrzLX/fpwSMOXmFIqBOyHsvHbnAdbGSJ+mKk= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/spanner v1.1.0 h1:hIjiz2Pf6Hy3BWz+Oaw7XUqP+EzWDkj0/DtTkKazxzk= +cloud.google.com/go/spanner v1.1.0/go.mod h1:TzTaF9l2ZY2CIetNvVpUu6ZQy8YEOtzB6ICa5EwYjL0= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Connor1996/badger v1.5.1-0.20220222053432-2d2cbf472c77 h1:9wys07NWIZESExLQK8DFuaOsky842KZlQAM/mCNhAp0= +github.com/Connor1996/badger v1.5.1-0.20220222053432-2d2cbf472c77/go.mod h1:GnSYrIcuBMzgVM5wYQrQOh5OH6nFHVAAUfUmfqTetzU= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/XiaoMi/pegasus-go-client v0.0.0-20181029071519-9400942c5d1c h1:3fAhdHMhoSG57DjJ/dqLFfgD+FoooPbQH6szINbrr3k= +github.com/XiaoMi/pegasus-go-client v0.0.0-20181029071519-9400942c5d1c/go.mod h1:KcL6D/4RZ8RAYzQ5gKI0odcdWUmCVlbQTOlWrhP71CY= +github.com/aerospike/aerospike-client-go v1.35.2 h1:TWV2Bn59Ig7SM4Zue84fFsPGlfFJX/6xbuGHyYFS/ag= +github.com/aerospike/aerospike-client-go v1.35.2/go.mod h1:zj8LBEnWBDOVEIJt8LvaRvDG5ARAoa5dBeHaB472NRc= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d h1:b/FqDLjWXDQI6XBYvWDVgEKv3xOTs68qRkuqyU37lBc= +github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apple/foundationdb/bindings/go v0.0.0-20200112054404-407dc0907f4f h1:HkQOU77BCH+BZPpzwNxm3zjUAt+N7mJWRAGxyCwAGZw= +github.com/apple/foundationdb/bindings/go v0.0.0-20200112054404-407dc0907f4f/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= +github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= +github.com/aws/aws-sdk-go-v2/config v1.17.8 h1:b9LGqNnOdg9vR4Q43tBTVWk4J6F+W774MSchvKJsqnE= +github.com/aws/aws-sdk-go-v2/config v1.17.8/go.mod h1:UkCI3kb0sCdvtjiXYiU4Zx5h07BOpgBTtkPu/49r+kA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.21 h1:4tjlyCD0hRGNQivh5dN8hbP30qQhMLBE/FgQR1vHHWM= +github.com/aws/aws-sdk-go-v2/credentials v1.12.21/go.mod h1:O+4XyAt4e+oBAoIwNUYkRg3CVMscaIJdmZBOcPgJ8D8= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 h1:bKbdstt7+PzIRSIXZ11Yo8Qh8t0AHn6jEYUfsbVcLjE= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0/go.mod h1:+CBJZMhsb1pTUcB/NTdS505bDX10xS4xnPMqDZj2Ptw= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.26 h1:4FF+S7M7T/f1ORJSwgUJfywZRjB25W8NdGeosQVy+fE= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.26/go.mod h1:RHt+Uh6nvd2kccFcEzgmtsDrGG8AoFCPzvznfUDSg6c= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1 h1:1QpTkQIAaZpR387it1L+erjB5bStGFCJRvmXsodpPEU= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1/go.mod h1:BZhn/C3z13ULTSstVi2Kymc62bgjFh/JwLO9Tm2OFYI= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 h1:V9q4A0qnUfDsfivspY1LQRQTOG3Y9FLHvXIaTbcU7XM= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20/go.mod h1:7qWU48SMzlrfOlNhHpazW3psFWlOIWrq4SmOr2/ESmk= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17 h1:o0Ia3nb56m8+8NvhbCDiSBiZRNUwIknVWobx5vks0Vk= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17/go.mod h1:WJD9FbkwzM2a1bZ36ntH6+5Jc+x41Q4K2AcLeHDLAS8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 h1:pwvCchFUEnlceKIgPUouBJwK81aCkQ8UDMORfeFtW10= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 h1:OwhhKc1P9ElfWbMKPIbMMZBV6hzJlL2JKD76wNNVzgQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc6lu6OQ97U7hmdesg= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= +github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= +github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= +github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= +github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 h1:NnLfQ77q0G4k2Of2c1ceQ0ec6MkLQyDp+IGdVM0D8XM= +github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2/go.mod h1:7qG7YFnOALvsx6tKTNmQot8d7cGFXM9TidzvRFLWYwM= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= +github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= +github.com/cznic/parser v0.0.0-20181122101858-d773202d5b1f/go.mod h1:2B43mz36vGZNZEwkWi8ayRSSUXLfjL8OkbzwW4NcPMM= +github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGda1NeOBQwqlv7nUXpm+rIVHGxZZ4= +github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= +github.com/cznic/y v0.0.0-20181122101901-b05e8c2e8d7b/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs= +github.com/dgraph-io/ristretto v0.0.1 h1:cJwdnj42uV8Jg4+KLrYovLiCgIfz9wtWm6E6KA+1tLs= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c h1:onA2RpIyeCPvYAj1LFYiiMTrSpqVINWMfYFRS7lofJs= +github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= +github.com/elastic/go-elasticsearch/v8 v8.3.0 h1:RF4iRbvWkiT6UksZ+OwSLeCEtBg/HO8r88xNiSmhb8U= +github.com/elastic/go-elasticsearch/v8 v8.3.0/go.mod h1:Usvydt+x0dv9a1TzEUaovqbJor8rmOHy5dSmPeMAE2k= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.49.0 h1:ymWFBUkwN3JFPjvjcJJ5TSTwh84M66QrH+8vOytLgRY= +github.com/go-ini/ini v1.49.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= +github.com/go-redis/redis/v9 v9.0.0-beta.1 h1:oW3jlPic5HhGUbYMH0lidnP+72BgsT+lCwlVud6o2Mc= +github.com/go-redis/redis/v9 v9.0.0-beta.1/go.mod h1:6gNX1bXdwkpEG0M/hEBNK/Fp8zdyCkjwwKc6vBbfCDI= +github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b h1:dnUw9Ih14dCKzbtZxm+pwQRYIb+9ypiwtZgsCQN4zmg= +github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/clock v0.0.0-20180524022203-d293bb356ca4/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= +github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9 h1:Y+lzErDTURqeXqlqYi4YBYbDd7ycU74gW1ADt57/bgY= +github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/retry v0.0.0-20160928201858-1998d01ba1c3/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4= +github.com/juju/testing v0.0.0-20200510222523-6c8c298c77a0 h1:+WWUkhnTjV6RNOxkcwk79qrjeyHEHvBzlneueBsatX4= +github.com/juju/testing v0.0.0-20200510222523-6c8c298c77a0/go.mod h1:hpGvhGHPVbNBraRLZEhoQwFLMrjK8PSlO4D3nDjKYXo= +github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk= +github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= +github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= +github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncw/directio v1.0.4 h1:CojwI07mCEmRkajgx42Pf8jyCwTs1ji9/Ij9/PJG12k= +github.com/ncw/directio v1.0.4/go.mod h1:CKGdcN7StAaqjT7Qack3lAXeX4pjnyc46YeqZH1yWVY= +github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac h1:wyheT2lPXRQqYPWY2IVW5BTLrbqCsnhL61zK2R5goLA= +github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac/go.mod h1:ueVCjKQllPmX7uEvCYnZD5b8qjidGf1TCH61arVe4SU= +github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= +github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= +github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= +github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= +github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99 h1:KcEvVBAvyHkUdFAygKAzwB6LAcZ6LS32WHmRD2VyXMI= +github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= +github.com/pingcap-incubator/tinysql v0.0.0-20200518090433-a7d00f9e6aa7 h1:OmyyQb7aA70tZX+LFzixFtNocNl/VIx0QyfYaoDN9iA= +github.com/pingcap-incubator/tinysql v0.0.0-20200518090433-a7d00f9e6aa7/go.mod h1:yuleLQAIKLLe1wiHaX1WdEyCF7K3JKJ9KfaqglC7Nqc= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= +github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= +github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= +github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60 h1:aJPXrT1u4VfUSGFA2oQVwl4pOXzqe+YI6wed01cjDH4= +github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5 h1:d9vJ/8gXbVnNk8QFOxFZ7MN7TuHiuvolK1usz5KXVDo= +github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc= +go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/etcd/api/v3 v3.5.6 h1:Cy2qx3npLcYqTKqGJzMypnMv2tiRyifZJ17BlWIWA7A= +go.etcd.io/etcd/api/v3 v3.5.6/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/client/pkg/v3 v3.5.6 h1:TXQWYceBKqLp4sa87rcPs11SXxUA/mHwH975v+BDvLU= +go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/v3 v3.5.6 h1:coLs69PWCXE9G4FKquzNaSHrRyMCAXwF+IX1tAPVO8E= +go.etcd.io/etcd/client/v3 v3.5.6/go.mod h1:f6GRinRMCsFVv9Ht42EyY7nfsVGwrNO0WEoS2pRKzQk= +go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI= +go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= +go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0= +golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 h1:VnGaRqoLmqZH/3TMLJwYCEWkR4j1nuIU1U9TvbqsDUw= +golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191105231337-689d0f08e67a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.87.0 h1:pUQVF/F+X7Tl1lo4LJoJf5BOpjtmINU80p9XpYTU2p4= +google.golang.org/api v0.87.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220718134204-073382fd740c h1:xDUAhRezFnKF6wopxkOfdWYvz2XCiRQzndyDdpwFgbc= +google.golang.org/genproto v0.0.0-20220718134204-073382fd740c/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4 h1:hILp2hNrRnYjZpmIbx70psAHbBSEcQ1NIzDcUbJ1b6g= +gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs= +gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= +gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/go-ycsb/pkg/client/client.go b/go-ycsb/pkg/client/client.go new file mode 100644 index 000000000..71f4d2f0c --- /dev/null +++ b/go-ycsb/pkg/client/client.go @@ -0,0 +1,227 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + "fmt" + "math/rand" + "os" + "sync" + "time" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/measurement" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +type worker struct { + p *properties.Properties + workDB ycsb.DB + workload ycsb.Workload + doTransactions bool + doBatch bool + batchSize int + opCount int64 + targetOpsPerMs float64 + threadID int + targetOpsTickNs int64 + opsDone int64 +} + +func newWorker(p *properties.Properties, threadID int, threadCount int, workload ycsb.Workload, db ycsb.DB) *worker { + w := new(worker) + w.p = p + w.doTransactions = p.GetBool(prop.DoTransactions, true) + w.batchSize = p.GetInt(prop.BatchSize, prop.DefaultBatchSize) + if w.batchSize > 1 { + w.doBatch = true + } + w.threadID = threadID + w.workload = workload + w.workDB = db + + var totalOpCount int64 + if w.doTransactions { + totalOpCount = p.GetInt64(prop.OperationCount, 0) + } else { + if _, ok := p.Get(prop.InsertCount); ok { + totalOpCount = p.GetInt64(prop.InsertCount, 0) + } else { + totalOpCount = p.GetInt64(prop.RecordCount, 0) + } + } + + if totalOpCount < int64(threadCount) { + fmt.Printf("totalOpCount(%s/%s/%s): %d should be bigger than threadCount: %d", + prop.OperationCount, + prop.InsertCount, + prop.RecordCount, + totalOpCount, + threadCount) + + os.Exit(-1) + } + + w.opCount = totalOpCount / int64(threadCount) + + targetPerThreadPerms := float64(-1) + if v := p.GetInt64(prop.Target, 0); v > 0 { + targetPerThread := float64(v) / float64(threadCount) + targetPerThreadPerms = targetPerThread / 1000.0 + } + + if targetPerThreadPerms > 0 { + w.targetOpsPerMs = targetPerThreadPerms + w.targetOpsTickNs = int64(1000000.0 / w.targetOpsPerMs) + } + + return w +} + +func (w *worker) throttle(ctx context.Context, startTime time.Time) { + if w.targetOpsPerMs <= 0 { + return + } + + d := time.Duration(w.opsDone * w.targetOpsTickNs) + d = startTime.Add(d).Sub(time.Now()) + if d < 0 { + return + } + select { + case <-ctx.Done(): + case <-time.After(d): + } +} + +func (w *worker) run(ctx context.Context) { + // spread the thread operation out so they don't all hit the DB at the same time + if w.targetOpsPerMs > 0.0 && w.targetOpsPerMs <= 1.0 { + time.Sleep(time.Duration(rand.Int63n(w.targetOpsTickNs))) + } + + startTime := time.Now() + + for w.opCount == 0 || w.opsDone < w.opCount { + var err error + opsCount := 1 + if w.doTransactions { + if w.doBatch { + err = w.workload.DoBatchTransaction(ctx, w.batchSize, w.workDB) + opsCount = w.batchSize + } else { + err = w.workload.DoTransaction(ctx, w.workDB) + } + } else { + if w.doBatch { + err = w.workload.DoBatchInsert(ctx, w.batchSize, w.workDB) + opsCount = w.batchSize + } else { + err = w.workload.DoInsert(ctx, w.workDB) + } + } + + if err != nil && !w.p.GetBool(prop.Silence, prop.SilenceDefault) { + fmt.Printf("operation err: %v\n", err) + } + + if measurement.IsWarmUpFinished() { + w.opsDone += int64(opsCount) + w.throttle(ctx, startTime) + } + + select { + case <-ctx.Done(): + return + default: + } + } +} + +// Client is a struct which is used the run workload to a specific DB. +type Client struct { + p *properties.Properties + workload ycsb.Workload + db ycsb.DB +} + +// NewClient returns a client with the given workload and DB. +// The workload and db can't be nil. +func NewClient(p *properties.Properties, workload ycsb.Workload, db ycsb.DB) *Client { + return &Client{p: p, workload: workload, db: db} +} + +// Run runs the workload to the target DB, and blocks until all workers end. +func (c *Client) Run(ctx context.Context) { + var wg sync.WaitGroup + threadCount := c.p.GetInt(prop.ThreadCount, 1) + + wg.Add(threadCount) + measureCtx, measureCancel := context.WithCancel(ctx) + measureCh := make(chan struct{}, 1) + go func() { + defer func() { + measureCh <- struct{}{} + }() + // load stage no need to warm up + if c.p.GetBool(prop.DoTransactions, true) { + dur := c.p.GetInt64(prop.WarmUpTime, 0) + select { + case <-ctx.Done(): + return + case <-time.After(time.Duration(dur) * time.Second): + } + } + // finish warming up + measurement.EnableWarmUp(false) + + dur := c.p.GetInt64(prop.LogInterval, 10) + t := time.NewTicker(time.Duration(dur) * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + measurement.Summary() + case <-measureCtx.Done(): + return + } + } + }() + + for i := 0; i < threadCount; i++ { + go func(threadId int) { + defer wg.Done() + + w := newWorker(c.p, threadId, threadCount, c.workload, c.db) + ctx := c.workload.InitThread(ctx, threadId, threadCount) + ctx = c.db.InitThread(ctx, threadId, threadCount) + w.run(ctx) + c.db.CleanupThread(ctx) + c.workload.CleanupThread(ctx) + }(i) + } + + wg.Wait() + if !c.p.GetBool(prop.DoTransactions, true) { + // when loading is finished, try to analyze table if possible. + if analyzeDB, ok := c.db.(ycsb.AnalyzeDB); ok { + analyzeDB.Analyze(ctx, c.p.GetString(prop.TableName, prop.TableNameDefault)) + } + } + measureCancel() + <-measureCh +} diff --git a/go-ycsb/pkg/client/dbwrapper.go b/go-ycsb/pkg/client/dbwrapper.go new file mode 100644 index 000000000..d8ec94d25 --- /dev/null +++ b/go-ycsb/pkg/client/dbwrapper.go @@ -0,0 +1,174 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + "fmt" + "time" + + "github.com/pingcap/go-ycsb/pkg/measurement" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// DbWrapper stores the pointer to a implementation of ycsb.DB. +type DbWrapper struct { + DB ycsb.DB +} + +func measure(start time.Time, op string, err error) { + lan := time.Now().Sub(start) + if err != nil { + measurement.Measure(fmt.Sprintf("%s_ERROR", op), start, lan) + return + } + + measurement.Measure(op, start, lan) +} + +func (db DbWrapper) Close() error { + return db.DB.Close() +} + +func (db DbWrapper) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { + return db.DB.InitThread(ctx, threadID, threadCount) +} + +func (db DbWrapper) CleanupThread(ctx context.Context) { + db.DB.CleanupThread(ctx) +} + +func (db DbWrapper) Read(ctx context.Context, table string, key string, fields []string) (_ map[string][]byte, err error) { + start := time.Now() + defer func() { + measure(start, "READ", err) + }() + + return db.DB.Read(ctx, table, key, fields) +} + +func (db DbWrapper) BatchRead(ctx context.Context, table string, keys []string, fields []string) (_ []map[string][]byte, err error) { + batchDB, ok := db.DB.(ycsb.BatchDB) + if ok { + start := time.Now() + defer func() { + measure(start, "BATCH_READ", err) + }() + return batchDB.BatchRead(ctx, table, keys, fields) + } + for _, key := range keys { + _, err := db.DB.Read(ctx, table, key, fields) + if err != nil { + return nil, err + } + } + return nil, nil +} + +func (db DbWrapper) Scan(ctx context.Context, table string, startKey string, count int, fields []string) (_ []map[string][]byte, err error) { + start := time.Now() + defer func() { + measure(start, "SCAN", err) + }() + + return db.DB.Scan(ctx, table, startKey, count, fields) +} + +func (db DbWrapper) Update(ctx context.Context, table string, key string, values map[string][]byte) (err error) { + start := time.Now() + defer func() { + measure(start, "UPDATE", err) + }() + + return db.DB.Update(ctx, table, key, values) +} + +func (db DbWrapper) BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) (err error) { + batchDB, ok := db.DB.(ycsb.BatchDB) + if ok { + start := time.Now() + defer func() { + measure(start, "BATCH_UPDATE", err) + }() + return batchDB.BatchUpdate(ctx, table, keys, values) + } + for i := range keys { + err := db.DB.Update(ctx, table, keys[i], values[i]) + if err != nil { + return err + } + } + return nil +} + +func (db DbWrapper) Insert(ctx context.Context, table string, key string, values map[string][]byte) (err error) { + start := time.Now() + defer func() { + measure(start, "INSERT", err) + }() + + return db.DB.Insert(ctx, table, key, values) +} + +func (db DbWrapper) BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) (err error) { + batchDB, ok := db.DB.(ycsb.BatchDB) + if ok { + start := time.Now() + defer func() { + measure(start, "BATCH_INSERT", err) + }() + return batchDB.BatchInsert(ctx, table, keys, values) + } + for i := range keys { + err := db.DB.Insert(ctx, table, keys[i], values[i]) + if err != nil { + return err + } + } + return nil +} + +func (db DbWrapper) Delete(ctx context.Context, table string, key string) (err error) { + start := time.Now() + defer func() { + measure(start, "DELETE", err) + }() + + return db.DB.Delete(ctx, table, key) +} + +func (db DbWrapper) BatchDelete(ctx context.Context, table string, keys []string) (err error) { + batchDB, ok := db.DB.(ycsb.BatchDB) + if ok { + start := time.Now() + defer func() { + measure(start, "BATCH_DELETE", err) + }() + return batchDB.BatchDelete(ctx, table, keys) + } + for _, key := range keys { + err := db.DB.Delete(ctx, table, key) + if err != nil { + return err + } + } + return nil +} + +func (db DbWrapper) Analyze(ctx context.Context, table string) error { + if analyzeDB, ok := db.DB.(ycsb.AnalyzeDB); ok { + return analyzeDB.Analyze(ctx, table) + } + return nil +} diff --git a/go-ycsb/pkg/generator/acknowledged_counter.go b/go-ycsb/pkg/generator/acknowledged_counter.go new file mode 100644 index 000000000..885b99b82 --- /dev/null +++ b/go-ycsb/pkg/generator/acknowledged_counter.go @@ -0,0 +1,110 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import ( + "math/rand" + "sync/atomic" + + "github.com/pingcap/go-ycsb/pkg/util" +) + +const ( + // WindowSize is the size of window of pending acks. + WindowSize int64 = 1 << 20 + + // WindowMask is used to turn an ID into a slot in the window. + WindowMask int64 = WindowSize - 1 +) + +// AcknowledgedCounter reports generated integers via Last only +// after they have been acknoledged. +type AcknowledgedCounter struct { + c Counter + + lock util.SpinLock + + window []bool + limit int64 +} + +// NewAcknowledgedCounter creates the counter which starts at start. +func NewAcknowledgedCounter(start int64) *AcknowledgedCounter { + return &AcknowledgedCounter{ + c: Counter{counter: start}, + lock: util.SpinLock{}, + window: make([]bool, WindowSize), + limit: start - 1, + } +} + +// Next implements the Generator Next interface. +func (a *AcknowledgedCounter) Next(r *rand.Rand) int64 { + return a.c.Next(r) +} + +// Last implements the Generator Last interface. +func (a *AcknowledgedCounter) Last() int64 { + return atomic.LoadInt64(&a.limit) +} + +// Acknowledge makes a generated counter vaailable via Last. +func (a *AcknowledgedCounter) Acknowledge(value int64) { + currentSlot := value & WindowMask + if a.window[currentSlot] { + panic("Too many unacknowledged insertion keys.") + } + + a.window[currentSlot] = true + + if !a.lock.TryLock() { + return + } + + defer a.lock.Unlock() + + // move a contiguous sequence from the window + // over to the "limit" variable + + limit := atomic.LoadInt64(&a.limit) + beforeFirstSlot := limit & WindowMask + index := limit + 1 + for ; index != beforeFirstSlot; index++ { + slot := index & WindowMask + if !a.window[slot] { + break + } + + a.window[slot] = false + } + + atomic.StoreInt64(&a.limit, index-1) +} diff --git a/go-ycsb/pkg/generator/constant.go b/go-ycsb/pkg/generator/constant.go new file mode 100644 index 000000000..2bad6360c --- /dev/null +++ b/go-ycsb/pkg/generator/constant.go @@ -0,0 +1,53 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import "math/rand" + +// Constant is a trivial generator that always returns the same value. +type Constant struct { + value int64 +} + +// NewConstant creates a constant generator +func NewConstant(value int64) *Constant { + return &Constant{value: value} +} + +// Next implements the Generator Next interface. +func (c *Constant) Next(_ *rand.Rand) int64 { + return c.value +} + +// Last implements the Generator Last interface. +func (c *Constant) Last() int64 { + return c.value +} diff --git a/go-ycsb/pkg/generator/counter.go b/go-ycsb/pkg/generator/counter.go new file mode 100644 index 000000000..1c0f856a2 --- /dev/null +++ b/go-ycsb/pkg/generator/counter.go @@ -0,0 +1,58 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import ( + "math/rand" + "sync/atomic" +) + +// Counter generates a sequence of integers. [0, 1, ...] +type Counter struct { + counter int64 +} + +// NewCounter creates the Counter generator. +func NewCounter(start int64) *Counter { + return &Counter{ + counter: start, + } +} + +// Next implements Generator Next interface. +func (c *Counter) Next(_ *rand.Rand) int64 { + return atomic.AddInt64(&c.counter, 1) - 1 +} + +// Last implements Generator Last interface. +func (c *Counter) Last() int64 { + return atomic.LoadInt64(&c.counter) - 1 +} diff --git a/go-ycsb/pkg/generator/discrete.go b/go-ycsb/pkg/generator/discrete.go new file mode 100644 index 000000000..475bca622 --- /dev/null +++ b/go-ycsb/pkg/generator/discrete.go @@ -0,0 +1,77 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import "math/rand" + +type discretePair struct { + Weight float64 + Value int64 +} + +// Discrete generates a distribution by choosing from a discrete set of values. +type Discrete struct { + Number + values []discretePair +} + +// NewDiscrete creates the generator. +func NewDiscrete() *Discrete { + return &Discrete{} +} + +// Next implements the Generator Next interface. +func (d *Discrete) Next(r *rand.Rand) int64 { + sum := float64(0) + + for _, p := range d.values { + sum += p.Weight + } + + val := r.Float64() + + for _, p := range d.values { + pw := p.Weight / sum + if val < pw { + d.SetLastValue(p.Value) + return p.Value + } + + val -= pw + } + + panic("oops, should not get here.") +} + +// Add adds a value with weight. +func (d *Discrete) Add(weight float64, value int64) { + d.values = append(d.values, discretePair{Weight: weight, Value: value}) +} diff --git a/go-ycsb/pkg/generator/exponential.go b/go-ycsb/pkg/generator/exponential.go new file mode 100644 index 000000000..ec3e9a89c --- /dev/null +++ b/go-ycsb/pkg/generator/exponential.go @@ -0,0 +1,64 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import ( + "math" + "math/rand" +) + +// Exponential generates an exponential distribution. It produces a sequence +// of time intervals according to an expontential distribution. +type Exponential struct { + Number + gamma float64 +} + +// NewExponentialWithMean creates an exponential generator with a mean arrival rate +// of gamma. +func NewExponentialWithMean(mean float64) *Exponential { + return &Exponential{gamma: 1.0 / mean} +} + +// NewExponential creats an exponential generator with percential and range. +func NewExponential(percentile float64, rng float64) *Exponential { + gamma := -math.Log(1.0-percentile/100.0) / rng + return &Exponential{ + gamma: gamma, + } +} + +// Next implements the Generator Next interface. +func (e *Exponential) Next(r *rand.Rand) int64 { + v := int64(-math.Log(r.Float64()) / e.gamma) + e.SetLastValue(v) + return v +} diff --git a/go-ycsb/pkg/generator/histogram.go b/go-ycsb/pkg/generator/histogram.go new file mode 100644 index 000000000..0b3857d85 --- /dev/null +++ b/go-ycsb/pkg/generator/histogram.go @@ -0,0 +1,134 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import ( + "io/ioutil" + "math/rand" + "strconv" + "strings" + + "github.com/pingcap/go-ycsb/pkg/util" +) + +// Histogram generates integers according to a histogram distribution. +type Histogram struct { + Number + blockSize int64 + buckets []int64 + area int64 + weightedArea int64 +} + +// NewHistogram creates a Histogram generator. +func NewHistogram(buckets []int64, blockSize int64) *Histogram { + var ( + area int64 + weightedArea int64 + ) + + for i, b := range buckets { + area += b + weightedArea += int64(i) * b + } + + return &Histogram{ + buckets: buckets, + blockSize: blockSize, + area: area, + weightedArea: weightedArea, + } +} + +type bucketInfo struct { + location int64 + value int64 +} + +// NewHistogramFromFile creates a Histogram generator from file. +func NewHistogramFromFile(name string) *Histogram { + data, err := ioutil.ReadFile(name) + if err != nil { + util.Fatalf("load histogram file %s failed %v", name, err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + line := strings.Split(strings.TrimSpace(lines[0]), "\t") + if line[0] != "BlockSize" { + util.Fatalf("First line of histogram is not the BlockSize but %s", line) + } + + blockSize, err := strconv.ParseInt(line[1], 10, 64) + if err != nil { + util.Fatalf("parse BlockSize failed %v", err) + } + + ay := make([]bucketInfo, 0, len(lines[1:])) + maxLocation := int64(0) + for _, s := range lines[1:] { + s = strings.TrimSpace(s) + if len(s) == 0 { + break + } + + line = strings.Split(s, "\t") + location, _ := strconv.ParseInt(line[0], 10, 64) + if maxLocation < location { + maxLocation = location + } + value, _ := strconv.ParseInt(line[1], 10, 64) + ay = append(ay, bucketInfo{location: location, value: value}) + } + + buckets := make([]int64, maxLocation+1) + for _, b := range ay { + buckets[b.location] = b.value + } + + return NewHistogram(buckets, blockSize) +} + +// Next implements the Generator Next interface. +func (h *Histogram) Next(r *rand.Rand) int64 { + n := r.Int63n(h.area) + + i := int64(0) + for ; i < int64(len(h.buckets))-1; i++ { + n -= h.buckets[i] + if n <= 0 { + return (i + 1) * h.blockSize + } + } + + v := i * h.blockSize + h.SetLastValue(v) + return v +} diff --git a/go-ycsb/pkg/generator/hotspot.go b/go-ycsb/pkg/generator/hotspot.go new file mode 100644 index 000000000..874d9b20e --- /dev/null +++ b/go-ycsb/pkg/generator/hotspot.go @@ -0,0 +1,87 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import "math/rand" + +// Hotspot generates integers resembling a hotspot distribution where %x of operations +// access y% of data items. +type Hotspot struct { + Number + lowerBound int64 + upperBound int64 + hotInterval int64 + coldInterval int64 + hotsetFraction float64 + hotOpnFraction float64 +} + +// NewHotspot creates a Hotspot generator. +// lowerBound: the lower bound of the distribution. +// upperBound: the upper bound of the distribution. +// hotsetFraction: percentage of data itme. +// hotOpnFraction: percentage of operations accessing the hot set. +func NewHotspot(lowerBound int64, upperBound int64, hotsetFraction float64, hotOpnFraction float64) *Hotspot { + if hotsetFraction < 0.0 || hotsetFraction > 1.0 { + hotsetFraction = 0.0 + } + + if hotOpnFraction < 0.0 || hotOpnFraction > 1.0 { + hotOpnFraction = 0.0 + } + + if lowerBound > upperBound { + lowerBound, upperBound = upperBound, lowerBound + } + + interval := upperBound - lowerBound + 1 + hotInterval := int64(float64(interval) * hotsetFraction) + return &Hotspot{ + lowerBound: lowerBound, + upperBound: upperBound, + hotsetFraction: hotsetFraction, + hotOpnFraction: hotOpnFraction, + hotInterval: hotInterval, + coldInterval: interval - hotInterval, + } +} + +// Next implements the Generator Next interface. +func (h *Hotspot) Next(r *rand.Rand) int64 { + value := int64(0) + if r.Float64() < h.hotOpnFraction { + value = h.lowerBound + r.Int63n(h.hotInterval) + } else { + value = h.lowerBound + h.hotInterval + r.Int63n(h.coldInterval) + } + h.SetLastValue(value) + return value +} diff --git a/go-ycsb/pkg/generator/number.go b/go-ycsb/pkg/generator/number.go new file mode 100644 index 000000000..dced17ffa --- /dev/null +++ b/go-ycsb/pkg/generator/number.go @@ -0,0 +1,46 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +// Number is a common generator. +type Number struct { + LastValue int64 +} + +// SetLastValue sets the last value generated. +func (n *Number) SetLastValue(value int64) { + n.LastValue = value +} + +// Last implements the Generator Last interface. +func (n *Number) Last() int64 { + return n.LastValue +} diff --git a/go-ycsb/pkg/generator/scrambled_zipfian.go b/go-ycsb/pkg/generator/scrambled_zipfian.go new file mode 100644 index 000000000..a9bdf7d92 --- /dev/null +++ b/go-ycsb/pkg/generator/scrambled_zipfian.go @@ -0,0 +1,76 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import ( + "math/rand" + + "github.com/pingcap/go-ycsb/pkg/util" +) + +// ScrambledZipfian produces a sequence of items, such that some items are more popular than +// others, according to a zipfian distribution +type ScrambledZipfian struct { + Number + gen *Zipfian + min int64 + max int64 + itemCount int64 +} + +// NewScrambledZipfian creates a ScrambledZipfian generator. +func NewScrambledZipfian(min int64, max int64, zipfianConstant float64) *ScrambledZipfian { + const ( + zetan = float64(26.46902820178302) + usedZipfianConstant = float64(0.99) + itemCount = int64(10000000000) + ) + + s := new(ScrambledZipfian) + s.min = min + s.max = max + s.itemCount = max - min + 1 + if zipfianConstant == usedZipfianConstant { + s.gen = NewZipfian(0, itemCount, zipfianConstant, zetan) + } else { + s.gen = NewZipfianWithRange(0, itemCount, zipfianConstant) + } + return s +} + +// Next implements the Generator Next interface. +func (s *ScrambledZipfian) Next(r *rand.Rand) int64 { + n := s.gen.Next(r) + + n = s.min + util.Hash64(n)%s.itemCount + s.SetLastValue(n) + return n +} diff --git a/go-ycsb/pkg/generator/sequential.go b/go-ycsb/pkg/generator/sequential.go new file mode 100644 index 000000000..7ef1a5881 --- /dev/null +++ b/go-ycsb/pkg/generator/sequential.go @@ -0,0 +1,63 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import ( + "math/rand" + "sync/atomic" +) + +// Sequential generates a sequence of integers 0, 1, ... +type Sequential struct { + counter int64 + interval int64 + start int64 +} + +// NewSequential creates a counter that starts at countStart. +func NewSequential(countStart int64, countEnd int64) *Sequential { + return &Sequential{ + counter: 0, + start: countStart, + interval: countEnd - countStart + 1, + } +} + +// Next implements the Generator Next interface. +func (s *Sequential) Next(_ *rand.Rand) int64 { + n := s.start + atomic.AddInt64(&s.counter, 1)%s.interval + return n +} + +// Last implements the Generator Last interface. +func (s *Sequential) Last() int64 { + return atomic.LoadInt64(&s.counter) + 1 +} diff --git a/go-ycsb/pkg/generator/skewedlatest.go b/go-ycsb/pkg/generator/skewedlatest.go new file mode 100644 index 000000000..d4d2258ac --- /dev/null +++ b/go-ycsb/pkg/generator/skewedlatest.go @@ -0,0 +1,68 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import ( + "math/rand" + "time" + + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +// SkewedLatest generates a popularity distribution of items, +// skewed to favor recent items significantly more than older items. +type SkewedLatest struct { + Number + basis ycsb.Generator + zipfian *Zipfian +} + +// NewSkewedLatest creates the SkewedLatest generator. +// basis is Counter or AcknowledgedCounter +func NewSkewedLatest(basis ycsb.Generator) *SkewedLatest { + zipfian := NewZipfianWithItems(basis.Last(), ZipfianConstant) + s := &SkewedLatest{ + basis: basis, + zipfian: zipfian, + } + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + s.Next(r) + return s +} + +// Next implements the Generator Next interface. +func (s *SkewedLatest) Next(r *rand.Rand) int64 { + max := s.basis.Last() + next := max - s.zipfian.next(r, max) + s.SetLastValue(next) + return next +} diff --git a/go-ycsb/pkg/generator/uniform.go b/go-ycsb/pkg/generator/uniform.go new file mode 100644 index 000000000..10678895d --- /dev/null +++ b/go-ycsb/pkg/generator/uniform.go @@ -0,0 +1,57 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import "math/rand" + +// Uniform generates integers randomly. +type Uniform struct { + Number + lb int64 + ub int64 + interval int64 +} + +// NewUniform creates the Uniform generator. +func NewUniform(lb int64, ub int64) *Uniform { + return &Uniform{ + lb: lb, + ub: ub, + interval: ub - lb + 1, + } +} + +// Next implements the Generator Next interface. +func (u *Uniform) Next(r *rand.Rand) int64 { + n := r.Int63n(u.interval) + u.lb + u.SetLastValue(n) + return n +} diff --git a/go-ycsb/pkg/generator/zipfian.go b/go-ycsb/pkg/generator/zipfian.go new file mode 100644 index 000000000..4bac60750 --- /dev/null +++ b/go-ycsb/pkg/generator/zipfian.go @@ -0,0 +1,170 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package generator + +import ( + "fmt" + "math" + "math/rand" + "time" + + "github.com/pingcap/go-ycsb/pkg/util" +) + +const ( + // ZipfianConstant is the default constant for the zipfian. + ZipfianConstant = float64(0.99) +) + +// Zipfian generates the zipfian distribution. It produces a sequence of items, such that some items are more popular than +// others, according to a zipfian distribution. When you construct an instance of this class, you specify the number +// of items in the set to draw from, either by specifying an itemcount (so that the sequence is of items from 0 to +// itemcount-1) or by specifying a min and a max (so that the sequence is of items from min to max inclusive). After +// you construct the instance, you can change the number of items by calling Next(*rand.Rand). +// +// Note that the popular items will be clustered together, e.g. item 0 is the most popular, item 1 the second most +// popular, and so on (or min is the most popular, min+1 the next most popular, etc.) If you don't want this clustering, +// and instead want the popular items scattered throughout the item space, then use ScrambledZipfian(located in scrambled_zipfian.go) instead. +// +// Be aware: initializing this generator may take a long time if there are lots of items to choose from (e.g. over a +// minute for 100 million objects). This is because certain mathematical values need to be computed to properly +// generate a zipfian skew, and one of those values (zeta) is a sum sequence from 1 to n, where n is the itemcount. +// Note that if you increase the number of items in the set, we can compute a new zeta incrementally, so it should be +// fast unless you have added millions of items. However, if you decrease the number of items, we recompute zeta from +// scratch, so this can take a long time. +// +// The algorithm used here is from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994. +type Zipfian struct { + Number + + lock util.SpinLock + + items int64 + base int64 + + zipfianConstant float64 + + alpha float64 + zetan float64 + theta float64 + eta float64 + zeta2Theta float64 + + countForZeta int64 + + allowItemCountDecrease bool +} + +// NewZipfianWithItems creates the Zipfian generator. +func NewZipfianWithItems(items int64, zipfianConstant float64) *Zipfian { + return NewZipfianWithRange(0, items-1, zipfianConstant) +} + +// NewZipfianWithRange creates the Zipfian generator. +func NewZipfianWithRange(min int64, max int64, zipfianConstant float64) *Zipfian { + return NewZipfian(min, max, zipfianConstant, zetaStatic(0, max-min+1, zipfianConstant, 0)) +} + +// NewZipfian creates the Zipfian generator. +func NewZipfian(min int64, max int64, zipfianConstant float64, zetan float64) *Zipfian { + items := max - min + 1 + z := new(Zipfian) + + z.items = items + z.base = min + + z.zipfianConstant = zipfianConstant + theta := z.zipfianConstant + z.theta = theta + + z.zeta2Theta = z.zeta(0, 2, theta, 0) + + z.alpha = 1.0 / (1.0 - theta) + z.zetan = zetan + z.countForZeta = items + z.eta = (1 - math.Pow(2.0/float64(items), 1-theta)) / (1 - z.zeta2Theta/z.zetan) + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + z.Next(r) + return z +} + +func (z *Zipfian) zeta(st int64, n int64, thetaVal float64, initialSum float64) float64 { + z.countForZeta = n + return zetaStatic(st, n, thetaVal, initialSum) +} + +func zetaStatic(st int64, n int64, theta float64, initialSum float64) float64 { + sum := initialSum + + for i := st; i < n; i++ { + sum += 1 / math.Pow(float64(i+1), theta) + } + + return sum +} + +func (z *Zipfian) next(r *rand.Rand, itemCount int64) int64 { + if itemCount != z.countForZeta { + z.lock.Lock() + if itemCount > z.countForZeta { + //we have added more items. can compute zetan incrementally, which is cheaper + z.zetan = z.zeta(z.countForZeta, itemCount, z.theta, z.zetan) + z.eta = (1 - math.Pow(2.0/float64(z.items), 1-z.theta)) / (1 - z.zeta2Theta/z.zetan) + } else if itemCount < z.countForZeta && z.allowItemCountDecrease { + //note : for large itemsets, this is very slow. so don't do it! + fmt.Printf("recomputing Zipfian distribution, should be avoided,item count %v, count for zeta %v\n", itemCount, z.countForZeta) + z.zetan = z.zeta(0, itemCount, z.theta, 0) + z.eta = (1 - math.Pow(2.0/float64(z.items), 1-z.theta)) / (1 - z.zeta2Theta/z.zetan) + } + z.lock.Unlock() + } + + u := r.Float64() + uz := u * z.zetan + + if uz < 1.0 { + return z.base + } + + if uz < 1.0+math.Pow(0.5, z.theta) { + return z.base + 1 + } + + ret := z.base + int64(float64(itemCount)*math.Pow(z.eta*u-z.eta+1, z.alpha)) + z.SetLastValue(ret) + return ret +} + +// Next implements the Generator Next interface. +func (z *Zipfian) Next(r *rand.Rand) int64 { + return z.next(r, z.items) +} diff --git a/go-ycsb/pkg/measurement/csv.go b/go-ycsb/pkg/measurement/csv.go new file mode 100644 index 000000000..0014c6fe6 --- /dev/null +++ b/go-ycsb/pkg/measurement/csv.go @@ -0,0 +1,51 @@ +package measurement + +import ( + "fmt" + "io" + "time" +) + +type csventry struct { + // start time of the operation in us from unix epoch + startUs int64 + // latency of the operation in us + latencyUs int64 +} + +type csvs struct { + opCsv map[string][]csventry +} + +func InitCSV() *csvs { + return &csvs{ + opCsv: make(map[string][]csventry), + } +} + +func (c *csvs) Measure(op string, start time.Time, lan time.Duration) { + c.opCsv[op] = append(c.opCsv[op], csventry{ + startUs: start.UnixMicro(), + latencyUs: lan.Microseconds(), + }) +} + +func (c *csvs) Output(w io.Writer) error { + _, err := fmt.Fprintln(w, "operation,timestamp_us,latency_us") + if err != nil { + return err + } + for op, entries := range c.opCsv { + for _, entry := range entries { + _, err := fmt.Fprintf(w, "%s,%d,%d\n", op, entry.startUs, entry.latencyUs) + if err != nil { + return err + } + } + } + return nil +} + +func (c *csvs) Summary() { + // do nothing as csvs don't keep a summary +} diff --git a/go-ycsb/pkg/measurement/histogram.go b/go-ycsb/pkg/measurement/histogram.go new file mode 100644 index 000000000..5da22c9e8 --- /dev/null +++ b/go-ycsb/pkg/measurement/histogram.go @@ -0,0 +1,97 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurement + +import ( + "sort" + "time" + + hdrhistogram "github.com/HdrHistogram/hdrhistogram-go" + "github.com/pingcap/go-ycsb/pkg/util" +) + +type histogram struct { + boundCounts util.ConcurrentMap + startTime time.Time + hist *hdrhistogram.Histogram +} + +// Metric name. +const ( + ELAPSED = "ELAPSED" + COUNT = "COUNT" + QPS = "QPS" + AVG = "AVG" + MIN = "MIN" + MAX = "MAX" + PER99TH = "PER99TH" + PER999TH = "PER999TH" + PER9999TH = "PER9999TH" +) + +func newHistogram() *histogram { + h := new(histogram) + h.startTime = time.Now() + h.hist = hdrhistogram.New(1, 24*60*60*1000*1000, 3) + return h +} + +func (h *histogram) Measure(latency time.Duration) { + h.hist.RecordValue(latency.Microseconds()) +} + +func (h *histogram) Summary() []string { + res := h.getInfo() + + return []string{ + util.FloatToOneString(res[ELAPSED]), + util.IntToString(res[COUNT]), + util.FloatToOneString(res[QPS]), + util.IntToString(res[AVG]), + util.IntToString(res[MIN]), + util.IntToString(res[MAX]), + util.IntToString(res[PER99TH]), + util.IntToString(res[PER999TH]), + util.IntToString(res[PER9999TH]), + } +} + +func (h *histogram) getInfo() map[string]interface{} { + min := h.hist.Min() + max := h.hist.Max() + avg := int64(h.hist.Mean()) + count := h.hist.TotalCount() + + bounds := h.boundCounts.Keys() + sort.Ints(bounds) + + per99 := h.hist.ValueAtPercentile(99) + per999 := h.hist.ValueAtPercentile(99.9) + per9999 := h.hist.ValueAtPercentile(99.99) + + elapsed := time.Now().Sub(h.startTime).Seconds() + qps := float64(count) / elapsed + res := make(map[string]interface{}) + res[ELAPSED] = elapsed + res[COUNT] = count + res[QPS] = qps + res[AVG] = avg + res[MIN] = min + res[MAX] = max + res[PER99TH] = per99 + res[PER999TH] = per999 + res[PER9999TH] = per9999 + + return res +} diff --git a/go-ycsb/pkg/measurement/histograms.go b/go-ycsb/pkg/measurement/histograms.go new file mode 100644 index 000000000..4e0d0a054 --- /dev/null +++ b/go-ycsb/pkg/measurement/histograms.go @@ -0,0 +1,76 @@ +package measurement + +import ( + "io" + "os" + "sort" + "time" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" +) + +type histograms struct { + p *properties.Properties + + histograms map[string]*histogram +} + +func (h *histograms) Measure(op string, start time.Time, lan time.Duration) { + opM, ok := h.histograms[op] + if !ok { + opM = newHistogram() + h.histograms[op] = opM + } + + opM.Measure(lan) +} + +func (h *histograms) summary() map[string][]string { + summaries := make(map[string][]string, len(h.histograms)) + for op, opM := range h.histograms { + summaries[op] = opM.Summary() + } + return summaries +} + +func (h *histograms) Summary() { + h.Output(os.Stdout) +} + +func (h *histograms) Output(w io.Writer) error { + summaries := h.summary() + keys := make([]string, 0, len(summaries)) + for k := range summaries { + keys = append(keys, k) + } + sort.Strings(keys) + + lines := [][]string{} + for _, op := range keys { + line := []string{op} + line = append(line, summaries[op]...) + lines = append(lines, line) + } + + outputStyle := h.p.GetString(prop.OutputStyle, util.OutputStylePlain) + switch outputStyle { + case util.OutputStylePlain: + util.RenderString(w, "%-6s - %s\n", header, lines) + case util.OutputStyleJson: + util.RenderJson(w, header, lines) + case util.OutputStyleTable: + util.RenderTable(w, header, lines) + default: + panic("unsupported outputstyle: " + outputStyle) + } + return nil +} + +func InitHistograms(p *properties.Properties) *histograms { + return &histograms{ + p: p, + histograms: make(map[string]*histogram, 16), + } +} diff --git a/go-ycsb/pkg/measurement/measurement.go b/go-ycsb/pkg/measurement/measurement.go new file mode 100644 index 000000000..7260bb31d --- /dev/null +++ b/go-ycsb/pkg/measurement/measurement.go @@ -0,0 +1,126 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurement + +import ( + "bufio" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +var header = []string{"Operation", "Takes(s)", "Count", "OPS", "Avg(us)", "Min(us)", "Max(us)", "99th(us)", "99.9th(us)", "99.99th(us)"} + +type measurement struct { + sync.RWMutex + + p *properties.Properties + + measurer ycsb.Measurer +} + +func (m *measurement) measure(op string, start time.Time, lan time.Duration) { + m.Lock() + m.measurer.Measure(op, start, lan) + m.Unlock() +} + +func (m *measurement) output() { + m.RLock() + defer m.RUnlock() + + outFile := m.p.GetString(prop.MeasurementRawOutputFile, "") + var w *bufio.Writer + if outFile == "" { + w = bufio.NewWriter(os.Stdout) + } else { + f, err := os.Create(outFile) + if err != nil { + panic("failed to create output file: " + err.Error()) + } + defer f.Close() + w = bufio.NewWriter(f) + } + + err := globalMeasure.measurer.Output(w) + if err != nil { + panic("failed to write output: " + err.Error()) + } + + err = w.Flush() + if err != nil { + panic("failed to flush output: " + err.Error()) + } +} + +func (m *measurement) summary() { + m.RLock() + globalMeasure.measurer.Summary() + m.RUnlock() +} + +// InitMeasure initializes the global measurement. +func InitMeasure(p *properties.Properties) { + globalMeasure = new(measurement) + globalMeasure.p = p + measurementType := p.GetString(prop.MeasurementType, prop.MeasurementTypeDefault) + switch measurementType { + case "histogram": + globalMeasure.measurer = InitHistograms(p) + case "raw", "csv": + globalMeasure.measurer = InitCSV() + default: + panic("unsupported measurement type: " + measurementType) + } + EnableWarmUp(p.GetInt64(prop.WarmUpTime, 0) > 0) +} + +// Output prints the complete measurements. +func Output() { + globalMeasure.output() +} + +// Summary prints the measurement summary. +func Summary() { + globalMeasure.summary() +} + +// EnableWarmUp sets whether to enable warm-up. +func EnableWarmUp(b bool) { + if b { + atomic.StoreInt32(&warmUp, 1) + } else { + atomic.StoreInt32(&warmUp, 0) + } +} + +// IsWarmUpFinished returns whether warm-up is finished or not. +func IsWarmUpFinished() bool { + return atomic.LoadInt32(&warmUp) == 0 +} + +// Measure measures the operation. +func Measure(op string, start time.Time, lan time.Duration) { + if IsWarmUpFinished() { + globalMeasure.measure(op, start, lan) + } +} + +var globalMeasure *measurement +var warmUp int32 // use as bool, 1 means in warmup progress, 0 means warmup finished. diff --git a/go-ycsb/pkg/prop/prop.go b/go-ycsb/pkg/prop/prop.go new file mode 100644 index 000000000..7ca5d6c17 --- /dev/null +++ b/go-ycsb/pkg/prop/prop.go @@ -0,0 +1,119 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package prop + +// Properties +const ( + InsertStart = "insertstart" + InsertCount = "insertcount" + InsertStartDefault = int64(0) + + OperationCount = "operationcount" + RecordCount = "recordcount" + RecordCountDefault = int64(0) + Workload = "workload" + DB = "db" + Exporter = "exporter" + ExportFile = "exportfile" + ThreadCount = "threadcount" + ThreadCountDefault = int64(200) + Target = "target" + MaxExecutiontime = "maxexecutiontime" + WarmUpTime = "warmuptime" + DoTransactions = "dotransactions" + Status = "status" + Label = "label" + // batch mode + BatchSize = "batch.size" + DefaultBatchSize = int(1) + + TableName = "table" + TableNameDefault = "usertable" + FieldCount = "fieldcount" + FieldCountDefault = int64(10) + // "uniform", "zipfian", "constant", "histogram" + FieldLengthDistribution = "fieldlengthdistribution" + FieldLengthDistributionDefault = "constant" + FieldLength = "fieldlength" + FieldLengthDefault = int64(100) + // Used if fieldlengthdistribution is "histogram" + FieldLengthHistogramFile = "fieldlengthhistogram" + FieldLengthHistogramFileDefault = "hist.txt" + ReadAllFields = "readallfields" + ReadALlFieldsDefault = true + WriteAllFields = "writeallfields" + WriteAllFieldsDefault = false + DataIntegrity = "dataintegrity" + DataIntegrityDefault = false + ReadProportion = "readproportion" + ReadProportionDefault = float64(0.95) + UpdateProportion = "updateproportion" + UpdateProportionDefault = float64(0.05) + InsertProportion = "insertproportion" + InsertProportionDefault = float64(0.0) + ScanProportion = "scanproportion" + ScanProportionDefault = float64(0.0) + ReadModifyWriteProportion = "readmodifywriteproportion" + ReadModifyWriteProportionDefault = float64(0.0) + // "uniform", "zipfian", "latest" + RequestDistribution = "requestdistribution" + RequestDistributionDefault = "uniform" + ZeroPadding = "zeropadding" + ZeroPaddingDefault = int64(1) + MaxScanLength = "maxscanlength" + MaxScanLengthDefault = int64(1000) + // "uniform", "zipfian" + ScanLengthDistribution = "scanlengthdistribution" + ScanLengthDistributionDefault = "uniform" + // "ordered", "hashed" + InsertOrder = "insertorder" + InsertOrderDefault = "hashed" + HotspotDataFraction = "hotspotdatafraction" + HotspotDataFractionDefault = float64(0.2) + HotspotOpnFraction = "hotspotopnfraction" + HotspotOpnFractionDefault = float64(0.8) + InsertionRetryLimit = "core_workload_insertion_retry_limit" + InsertionRetryLimitDefault = int64(0) + InsertionRetryInterval = "core_workload_insertion_retry_interval" + InsertionRetryIntervalDefault = int64(3) + + ExponentialPercentile = "exponential.percentile" + ExponentialPercentileDefault = float64(95) + ExponentialFrac = "exponential.frac" + ExponentialFracDefault = float64(0.8571428571) + + DebugPprof = "debug.pprof" + DebugPprofDefault = ":6060" + + Verbose = "verbose" + VerboseDefault = false + DropData = "dropdata" + DropDataDefault = false + + Silence = "silence" + SilenceDefault = true + + KeyPrefix = "keyprefix" + KeyPrefixDefault = "user" + + LogInterval = "measurement.interval" + + MeasurementType = "measurementtype" + MeasurementTypeDefault = "histogram" + MeasurementRawOutputFile = "measurement.output_file" + + Command = "command" + + OutputStyle = "outputstyle" +) diff --git a/go-ycsb/pkg/util/concurrent_map.go b/go-ycsb/pkg/util/concurrent_map.go new file mode 100644 index 000000000..e50e5b025 --- /dev/null +++ b/go-ycsb/pkg/util/concurrent_map.go @@ -0,0 +1,319 @@ +package util + +import ( + "encoding/json" + "sync" +) + +/* + This module is modified from github.com/orcaman/concurrent-map. + MIT license. +*/ + +// A "thread" safe map of type int:int64. +// To avoid lock bottlenecks this map is dived to several (SHARD_COUNT) map shards. +type ConcurrentMap struct { + shards []*ConcurrentMapShared + shardCount int +} + +// A "thread" safe int to int64 map. +type ConcurrentMapShared struct { + items map[int]int64 + sync.RWMutex // Read Write mutex, guards access to internal map. +} + +// Creates a new concurrent map. +func New(shardCount int) ConcurrentMap { + m := new(ConcurrentMap) + m.shardCount = shardCount + m.shards = make([]*ConcurrentMapShared, shardCount) + for i := 0; i < shardCount; i++ { + m.shards[i] = &ConcurrentMapShared{items: make(map[int]int64)} + } + return *m +} + +// GetShard returns shard under given key +func (m ConcurrentMap) GetShard(key int) *ConcurrentMapShared { + return m.shards[uint(m.fnv32(key))%uint(m.shardCount)] +} + +func (m ConcurrentMap) MSet(data map[int]int64) { + for key, value := range data { + m.Set(key, value) + } +} + +// Sets the given value under the specified key. +func (m ConcurrentMap) Set(key int, value int64) { + // Get map shard. + shard := m.GetShard(key) + shard.Lock() + shard.items[key] = value + shard.Unlock() +} + +// Callback to return new element to be inserted into the map +// It is called while lock is held, therefore it MUST NOT +// try to access other keys in same map, as it can lead to deadlock since +// Go sync.RWLock is not reentrant +type UpsertCb func(exist bool, valueInMap int64, newValue int64) int64 + +// Insert or Update - updates existing element or inserts a new one using UpsertCb +func (m ConcurrentMap) Upsert(key int, value int64, cb UpsertCb) (res int64) { + shard := m.GetShard(key) + shard.Lock() + v, ok := shard.items[key] + res = cb(ok, v, value) + shard.items[key] = res + shard.Unlock() + return res +} + +// Sets the given value under the specified key if no value was associated with it. +func (m ConcurrentMap) SetIfAbsent(key int, value int64) bool { + // Get map shard. + shard := m.GetShard(key) + shard.Lock() + _, ok := shard.items[key] + if !ok { + shard.items[key] = value + } + shard.Unlock() + return !ok +} + +// Get retrieves an element from map under given key. +func (m ConcurrentMap) Get(key int) (int64, bool) { + // Get shard + shard := m.GetShard(key) + shard.RLock() + // Get item from shard. + val, ok := shard.items[key] + shard.RUnlock() + return val, ok +} + +// Count returns the number of elements within the map. +func (m ConcurrentMap) Count() int { + count := 0 + for i := 0; i < m.shardCount; i++ { + shard := m.shards[i] + shard.RLock() + count += len(shard.items) + shard.RUnlock() + } + return count +} + +// Looks up an item under specified key +func (m ConcurrentMap) Has(key int) bool { + // Get shard + shard := m.GetShard(key) + shard.RLock() + // See if element is within shard. + _, ok := shard.items[key] + shard.RUnlock() + return ok +} + +// Remove removes an element from the map. +func (m ConcurrentMap) Remove(key int) { + // Try to get shard. + shard := m.GetShard(key) + shard.Lock() + delete(shard.items, key) + shard.Unlock() +} + +// RemoveCb is a callback executed in a map.RemoveCb() call, while Lock is held +// If returns true, the element will be removed from the map +type RemoveCb func(key int, v int64, exists bool) bool + +// RemoveCb locks the shard containing the key, retrieves its current value and calls the callback with those params +// If callback returns true and element exists, it will remove it from the map +// Returns the value returned by the callback (even if element was not present in the map) +func (m ConcurrentMap) RemoveCb(key int, cb RemoveCb) bool { + // Try to get shard. + shard := m.GetShard(key) + shard.Lock() + v, ok := shard.items[key] + remove := cb(key, v, ok) + if remove && ok { + delete(shard.items, key) + } + shard.Unlock() + return remove +} + +// Pop removes an element from the map and returns it +func (m ConcurrentMap) Pop(key int) (v int64, exists bool) { + // Try to get shard. + shard := m.GetShard(key) + shard.Lock() + v, exists = shard.items[key] + delete(shard.items, key) + shard.Unlock() + return v, exists +} + +// IsEmpty checks if map is empty. +func (m ConcurrentMap) IsEmpty() bool { + return m.Count() == 0 +} + +// Used by the Iter & IterBuffered functions to wrap two variables together over a channel, +type Tuple struct { + Key int + Val int64 +} + +// Iter returns an iterator which could be used in a for range loop. +// +// Deprecated: using IterBuffered() will get a better performence +func (m ConcurrentMap) Iter() <-chan Tuple { + chans := snapshot(m) + ch := make(chan Tuple) + go fanIn(chans, ch) + return ch +} + +// IterBuffered returns a buffered iterator which could be used in a for range loop. +func (m ConcurrentMap) IterBuffered() <-chan Tuple { + chans := snapshot(m) + total := 0 + for _, c := range chans { + total += cap(c) + } + ch := make(chan Tuple, total) + go fanIn(chans, ch) + return ch +} + +// Returns a array of channels that contains elements in each shard, +// which likely takes a snapshot of `m`. +// It returns once the size of each buffered channel is determined, +// before all the channels are populated using goroutines. +func snapshot(m ConcurrentMap) (chans []chan Tuple) { + chans = make([]chan Tuple, m.shardCount) + wg := sync.WaitGroup{} + wg.Add(m.shardCount) + // Foreach shard. + for index, shard := range m.shards { + go func(index int, shard *ConcurrentMapShared) { + // Foreach key, value pair. + shard.RLock() + chans[index] = make(chan Tuple, len(shard.items)) + wg.Done() + for key, val := range shard.items { + chans[index] <- Tuple{key, val} + } + shard.RUnlock() + close(chans[index]) + }(index, shard) + } + wg.Wait() + return chans +} + +// fanIn reads elements from channels `chans` into channel `out` +func fanIn(chans []chan Tuple, out chan Tuple) { + wg := sync.WaitGroup{} + wg.Add(len(chans)) + for _, ch := range chans { + go func(ch chan Tuple) { + for t := range ch { + out <- t + } + wg.Done() + }(ch) + } + wg.Wait() + close(out) +} + +// Items returns all items as map[int]int64 +func (m ConcurrentMap) Items() map[int]int64 { + tmp := make(map[int]int64) + + // Insert items to temporary map. + for item := range m.IterBuffered() { + tmp[item.Key] = item.Val + } + + return tmp +} + +// Iterator callback,called for every key,value found in +// maps. RLock is held for all calls for a given shard +// therefore callback sess consistent view of a shard, +// but not across the shards +type IterCb func(key int, v int64) + +// Callback based iterator, cheapest way to read +// all elements in a map. +func (m ConcurrentMap) IterCb(fn IterCb) { + for idx := range m.shards { + shard := m.shards[idx] + shard.RLock() + for key, value := range shard.items { + fn(key, value) + } + shard.RUnlock() + } +} + +// Keys returns all keys as []int +func (m ConcurrentMap) Keys() []int { + count := m.Count() + ch := make(chan int, count) + go func() { + // Foreach shard. + wg := sync.WaitGroup{} + wg.Add(m.shardCount) + for _, shard := range m.shards { + go func(shard *ConcurrentMapShared) { + // Foreach key, value pair. + shard.RLock() + for key := range shard.items { + ch <- key + } + shard.RUnlock() + wg.Done() + }(shard) + } + wg.Wait() + close(ch) + }() + + // Generate keys + keys := make([]int, 0, count) + for k := range ch { + keys = append(keys, k) + } + return keys +} + +//Reviles ConcurrentMap "private" variables to json marshal. +func (m ConcurrentMap) MarshalJSON() ([]byte, error) { + // Create a temporary map, which will hold all item spread across shards. + tmp := make(map[int]int64) + + // Insert items to temporary map. + for item := range m.IterBuffered() { + tmp[item.Key] = item.Val + } + return json.Marshal(tmp) +} + +func (m ConcurrentMap) fnv32(key int) uint32 { + key = (key + 0x7ed55d16) + (key << 12) + key = (key ^ 0xc761c23c) ^ (key >> 19) + key = (key + 0x165667b1) + (key << 5) + key = (key + 0xd3a2646c) ^ (key << 9) + key = (key + 0xfd7046c5) + (key << 3) + key = (key ^ 0xb55a4f09) ^ (key >> 16) + key = key % m.shardCount + return uint32(key) +} diff --git a/go-ycsb/pkg/util/core.go b/go-ycsb/pkg/util/core.go new file mode 100644 index 000000000..87d44b12f --- /dev/null +++ b/go-ycsb/pkg/util/core.go @@ -0,0 +1,134 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + "sort" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/prop" +) + +// createFieldIndices is a helper function to create a field -> index mapping +// for the core workload +func createFieldIndices(p *properties.Properties) map[string]int64 { + fieldCount := p.GetInt64(prop.FieldCount, prop.FieldCountDefault) + m := make(map[string]int64, fieldCount) + for i := int64(0); i < fieldCount; i++ { + field := fmt.Sprintf("field%d", i) + m[field] = i + } + return m +} + +// allFields is a helper function to create all fields +func allFields(p *properties.Properties) []string { + fieldCount := p.GetInt64(prop.FieldCount, prop.FieldCountDefault) + fields := make([]string, 0, fieldCount) + for i := int64(0); i < fieldCount; i++ { + field := fmt.Sprintf("field%d", i) + fields = append(fields, field) + } + return fields +} + +// RowCodec is a helper struct to encode and decode TiDB format row +type RowCodec struct { + fieldIndices map[string]int64 + fields []string +} + +// NewRowCodec creates the RowCodec +func NewRowCodec(p *properties.Properties) *RowCodec { + return &RowCodec{ + fieldIndices: createFieldIndices(p), + fields: allFields(p), + } +} + +// Decode decodes the row and returns a field-value map +func (r *RowCodec) Decode(row []byte, fields []string) (map[string][]byte, error) { + if len(fields) == 0 { + fields = r.fields + } + + data, err := DecodeRow(row) + if err != nil { + return nil, err + } + + res := make(map[string][]byte, len(fields)) + for _, field := range fields { + i := r.fieldIndices[field] + if v, ok := data[i]; ok { + res[field] = v + } + } + + return res, nil +} + +// Encode encodes the values +func (r *RowCodec) Encode(buf []byte, values map[string][]byte) ([]byte, error) { + cols := make([][]byte, 0, len(values)) + colIDs := make([]int64, 0, len(values)) + + for k, v := range values { + i := r.fieldIndices[k] + cols = append(cols, v) + colIDs = append(colIDs, i) + } + + rowData, err := EncodeRow(cols, colIDs, buf) + return rowData, err +} + +// FieldPair is a pair to hold field + value. +type FieldPair struct { + Field string + Value []byte +} + +// FieldPairs implements sort interface for []FieldPair +type FieldPairs []FieldPair + +// Len implements sort interface Len +func (s FieldPairs) Len() int { + return len(s) +} + +// Len implements sort interface Swap +func (s FieldPairs) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +// Len implements sort interface Less +func (s FieldPairs) Less(i, j int) bool { + return s[i].Field < s[j].Field +} + +// NewFieldPairs sorts the map by fields and return a sorted slice of FieldPair. +func NewFieldPairs(values map[string][]byte) FieldPairs { + pairs := make(FieldPairs, 0, len(values)) + for field, value := range values { + pairs = append(pairs, FieldPair{ + Field: field, + Value: value, + }) + } + + sort.Sort(pairs) + return pairs +} diff --git a/go-ycsb/pkg/util/core_test.go b/go-ycsb/pkg/util/core_test.go new file mode 100644 index 000000000..0bae2e2c6 --- /dev/null +++ b/go-ycsb/pkg/util/core_test.go @@ -0,0 +1,24 @@ +package util + +import ( + "reflect" + "testing" +) + +func TestFieldPair(t *testing.T) { + m := map[string][]byte{ + "f2": []byte("b"), + "f1": []byte("a"), + } + + p := NewFieldPairs(m) + + check := FieldPairs{ + {"f1", []byte("a")}, + {"f2", []byte("b")}, + } + + if !reflect.DeepEqual(p, check) { + t.Errorf("want %v, but got %v", check, p) + } +} diff --git a/go-ycsb/pkg/util/hack.go b/go-ycsb/pkg/util/hack.go new file mode 100644 index 000000000..4cce8d695 --- /dev/null +++ b/go-ycsb/pkg/util/hack.go @@ -0,0 +1,43 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "reflect" + "unsafe" +) + +// String converts slice to string without copy. +// Use at your own risk. +func String(b []byte) (s string) { + if len(b) == 0 { + return "" + } + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) + pstring.Data = pbytes.Data + pstring.Len = pbytes.Len + return +} + +// Slice converts string to slice without copy. +// Use at your own risk. +func Slice(s string) (b []byte) { + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) + pbytes.Data = pstring.Data + pbytes.Len = pstring.Len + pbytes.Cap = pstring.Len + return +} diff --git a/go-ycsb/pkg/util/hash.go b/go-ycsb/pkg/util/hash.go new file mode 100644 index 000000000..c7ee6138e --- /dev/null +++ b/go-ycsb/pkg/util/hash.go @@ -0,0 +1,46 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "encoding/binary" + "hash/fnv" +) + +// Hash64 returns a fnv Hash of the integer. +func Hash64(n int64) int64 { + var b [8]byte + binary.BigEndian.PutUint64(b[0:8], uint64(n)) + hash := fnv.New64a() + hash.Write(b[0:8]) + result := int64(hash.Sum64()) + if result < 0 { + return -result + } + return result +} + +// BytesHash64 returns the fnv hash of a bytes +func BytesHash64(b []byte) int64 { + hash := fnv.New64a() + hash.Write(b) + return int64(hash.Sum64()) +} + +// StringHash64 returns the fnv hash of a string +func StringHash64(s string) int64 { + hash := fnv.New64a() + hash.Write(Slice(s)) + return int64(hash.Sum64()) +} diff --git a/go-ycsb/pkg/util/output.go b/go-ycsb/pkg/util/output.go new file mode 100644 index 000000000..c982d6e5b --- /dev/null +++ b/go-ycsb/pkg/util/output.go @@ -0,0 +1,77 @@ +package util + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/olekukonko/tablewriter" +) + +// output style +const ( + OutputStylePlain = "plain" + OutputStyleTable = "table" + OutputStyleJson = "json" +) + +// RenderString renders headers and values according to the format provided +func RenderString(w io.Writer, format string, headers []string, values [][]string) { + if len(values) == 0 { + return + } + + buf := new(bytes.Buffer) + for _, value := range values { + args := make([]string, len(headers)-1) + for i, header := range headers[1:] { + args[i] = header + ": " + value[i+1] + } + buf.WriteString(fmt.Sprintf(format, value[0], strings.Join(args, ", "))) + } + fmt.Fprint(w, buf.String()) +} + +// RenderTable will use given headers and values to render a table style output +func RenderTable(w io.Writer, headers []string, values [][]string) { + if len(values) == 0 { + return + } + tb := tablewriter.NewWriter(w) + tb.SetHeader(headers) + tb.AppendBulk(values) + tb.Render() +} + +// RnederJson will combine the headers and values and print a json string +func RenderJson(w io.Writer, headers []string, values [][]string) { + if len(values) == 0 { + return + } + data := make([]map[string]string, 0, len(values)) + for _, value := range values { + line := make(map[string]string, 0) + for i, header := range headers { + line[header] = value[i] + } + data = append(data, line) + } + outStr, err := json.Marshal(data) + if err != nil { + fmt.Fprintln(w, err) + return + } + fmt.Fprintln(w, string(outStr)) +} + +// IntToString formats int value to string +func IntToString(i interface{}) string { + return fmt.Sprintf("%d", i) +} + +// FloatToOneString formats float into string with one digit after dot +func FloatToOneString(f interface{}) string { + return fmt.Sprintf("%.1f", f) +} diff --git a/go-ycsb/pkg/util/row.go b/go-ycsb/pkg/util/row.go new file mode 100644 index 000000000..c2ecd9417 --- /dev/null +++ b/go-ycsb/pkg/util/row.go @@ -0,0 +1,115 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "encoding/binary" + + "github.com/pingcap/errors" +) + +// EncodeRow encodes row data and column ids into a slice of byte. +// Row layout: colID1, value1, colID2, value2, ... +// valBuf and values pass by caller, for reducing EncodeRow allocates temporary bufs. If you pass valBuf and values as nil, +// EncodeRow will allocate it. +// It is a simplified and specialized version of `github.com/pingcap/tidb/tablecodec.EncodeRow`. +func EncodeRow(cols [][]byte, colIDs []int64, valBuf []byte) ([]byte, error) { + if len(cols) != len(colIDs) { + return nil, errors.Errorf("EncodeRow error: cols and colIDs count not match %d vs %d", len(cols), len(colIDs)) + } + valBuf = valBuf[:0] + if len(cols) == 0 { + return append(valBuf, 0), nil + } + for i := range cols { + valBuf = encodeInt64(valBuf, colIDs[i]) + valBuf = encodeBytes(valBuf, cols[i]) + } + return valBuf, nil +} + +const ( + compactBytesFlag byte = 2 + varintFlag byte = 8 +) + +func encodeInt64(b []byte, v int64) []byte { + b = append(b, varintFlag) + return appendVarint(b, v) +} + +func encodeBytes(b []byte, v []byte) []byte { + b = append(b, compactBytesFlag) + b = appendVarint(b, int64(len(v))) + return append(b, v...) +} + +func appendVarint(b []byte, v int64) []byte { + var data [binary.MaxVarintLen64]byte + n := binary.PutVarint(data[:], v) + return append(b, data[:n]...) +} + +// DecodeRow decodes a byte slice into columns. +// Row layout: colID1, value1, colID2, value2, ..... +// It is a simplified and specialized version of `github.com/pingcap/tidb/tablecodec.DecodeRow`. +func DecodeRow(b []byte) (map[int64][]byte, error) { + row := make(map[int64][]byte) + if len(b) == 0 { + return row, nil + } + if len(b) == 1 && b[0] == 0 { + return row, nil + } + for len(b) > 0 { + remain, rowID, err := decodeInt64(b) + if err != nil { + return row, err + } + var v []byte + remain, v, err = decodeBytes(remain) + if err != nil { + return row, err + } + row[rowID] = v + b = remain + } + return row, nil +} + +func decodeInt64(b []byte) ([]byte, int64, error) { + return decodeVarint(b[1:]) +} + +func decodeVarint(b []byte) ([]byte, int64, error) { + v, n := binary.Varint(b) + if n > 0 { + return b[n:], v, nil + } + if n < 0 { + return nil, 0, errors.New("value larger than 64 bits") + } + return nil, 0, errors.New("insufficient bytes to decode value") +} + +func decodeBytes(b []byte) ([]byte, []byte, error) { + remain, n, err := decodeVarint(b[1:]) + if err != nil { + return nil, nil, err + } + if int64(len(remain)) < n { + return nil, nil, errors.Errorf("insufficient bytes to decode value, expected length: %v", n) + } + return remain[n:], remain[:n], nil +} diff --git a/go-ycsb/pkg/util/row_test.go b/go-ycsb/pkg/util/row_test.go new file mode 100644 index 000000000..2f292f925 --- /dev/null +++ b/go-ycsb/pkg/util/row_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "bytes" + "testing" +) + +func TestCodec(t *testing.T) { + colIDs := []int64{1, 4, 7, 2, 5, 8} + cols := [][]byte{[]byte("147"), []byte("258"), []byte("147258"), []byte(""), []byte("258147"), []byte("369")} + + buf, err := EncodeRow(cols, colIDs, nil) + if err != nil { + t.Fatal(err) + } + row, err := DecodeRow(buf) + if err != nil { + t.Fatal(err) + } + for i, id := range colIDs { + if !bytes.Equal(cols[i], row[id]) { + t.Fatalf("id:%v, before:%q, after:%q", id, cols[i], row[id]) + } + } +} diff --git a/go-ycsb/pkg/util/spinlock.go b/go-ycsb/pkg/util/spinlock.go new file mode 100644 index 000000000..a65a23558 --- /dev/null +++ b/go-ycsb/pkg/util/spinlock.go @@ -0,0 +1,50 @@ +// Copyright 2014 OneOfOne +// https://github.com/OneOfOne/go-utils +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "runtime" + "sync/atomic" +) + +// SpinLock implements a simple atomic spin lock, the zero value for a SpinLock is an unlocked spinlock. +type SpinLock struct { + f uint32 +} + +// Lock locks sl. If the lock is already in use, the caller blocks until Unlock is called +func (sl *SpinLock) Lock() { + for !sl.TryLock() { + runtime.Gosched() //allow other goroutines to do stuff. + } +} + +// Unlock unlocks sl, unlike [Mutex.Unlock](http://golang.org/pkg/sync/#Mutex.Unlock), +// there's no harm calling it on an unlocked SpinLock +func (sl *SpinLock) Unlock() { + atomic.StoreUint32(&sl.f, 0) +} + +// TryLock will try to lock sl and return whether it succeed or not without blocking. +func (sl *SpinLock) TryLock() bool { + return atomic.CompareAndSwapUint32(&sl.f, 0, 1) +} + +func (sl *SpinLock) String() string { + if atomic.LoadUint32(&sl.f) == 1 { + return "Locked" + } + return "Unlocked" +} diff --git a/go-ycsb/pkg/util/tls.go b/go-ycsb/pkg/util/tls.go new file mode 100644 index 000000000..389218233 --- /dev/null +++ b/go-ycsb/pkg/util/tls.go @@ -0,0 +1,59 @@ +package util + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" +) + +// CreateTLSConfig creats a TLS configuration. +func CreateTLSConfig(caPath, certPath, keyPath string, insecureSkipVerify bool) (*tls.Config, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: insecureSkipVerify, + Renegotiation: tls.RenegotiateNever, + } + + if caPath != "" { + pool, err := makeCertPool([]string{caPath}) + if err != nil { + return nil, err + } + tlsConfig.RootCAs = pool + } + + if certPath != "" && keyPath != "" { + err := loadCertificate(tlsConfig, certPath, keyPath) + if err != nil { + return nil, err + } + } + + return tlsConfig, nil +} + +func makeCertPool(certFiles []string) (*x509.CertPool, error) { + pool := x509.NewCertPool() + for _, certFile := range certFiles { + pem, err := ioutil.ReadFile(certFile) + if err != nil { + return nil, fmt.Errorf("could not read certificate %q: %v", certFile, err) + } + ok := pool.AppendCertsFromPEM(pem) + if !ok { + return nil, fmt.Errorf("could not parse any PEM certificates %q: %v", certFile, err) + } + } + return pool, nil +} + +func loadCertificate(config *tls.Config, certFile, keyFile string) error { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("could not load keypair %s:%s: %v", certFile, keyFile, err) + } + + config.Certificates = []tls.Certificate{cert} + config.BuildNameToCertificate() + return nil +} diff --git a/go-ycsb/pkg/util/util.go b/go-ycsb/pkg/util/util.go new file mode 100644 index 000000000..2fed70072 --- /dev/null +++ b/go-ycsb/pkg/util/util.go @@ -0,0 +1,73 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + "math/rand" + "os" + "sync" +) + +// Fatalf prints the message and exits the program. +func Fatalf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, format, args...) + fmt.Println("") + os.Exit(1) +} + +// Fatal prints the message and exits the program. +func Fatal(args ...interface{}) { + fmt.Fprint(os.Stderr, args...) + fmt.Println("") + os.Exit(1) +} + +var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +// RandBytes fills the bytes with alphabetic characters randomly +func RandBytes(r *rand.Rand, b []byte) { + for i := range b { + b[i] = letters[r.Intn(len(letters))] + } +} + +// BufPool is a bytes.Buffer pool +type BufPool struct { + p *sync.Pool +} + +// NewBufPool creates a buffer pool. +func NewBufPool() *BufPool { + p := &sync.Pool{ + New: func() interface{} { + return []byte(nil) + }, + } + return &BufPool{ + p: p, + } +} + +// Get gets a buffer. +func (b *BufPool) Get() []byte { + buf := b.p.Get().([]byte) + buf = buf[:0] + return buf +} + +// Put returns a buffer. +func (b *BufPool) Put(buf []byte) { + b.p.Put(buf) +} diff --git a/go-ycsb/pkg/workload/core.go b/go-ycsb/pkg/workload/core.go new file mode 100644 index 000000000..1f0f2991c --- /dev/null +++ b/go-ycsb/pkg/workload/core.go @@ -0,0 +1,705 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package workload + +import ( + "bytes" + "context" + "fmt" + "math" + "math/rand" + "strconv" + "strings" + "sync" + "time" + + "github.com/magiconair/properties" + "github.com/pingcap/go-ycsb/pkg/generator" + "github.com/pingcap/go-ycsb/pkg/measurement" + "github.com/pingcap/go-ycsb/pkg/prop" + "github.com/pingcap/go-ycsb/pkg/util" + "github.com/pingcap/go-ycsb/pkg/ycsb" +) + +type contextKey string + +const stateKey = contextKey("core") + +type coreState struct { + r *rand.Rand + // fieldNames is a copy of core.fieldNames to be goroutine-local + fieldNames []string +} + +type operationType int64 + +const ( + read operationType = iota + 1 + update + insert + scan + readModifyWrite +) + +// Core is the core benchmark scenario. Represents a set of clients doing simple CRUD operations. +type core struct { + p *properties.Properties + + table string + fieldCount int64 + fieldNames []string + + fieldLengthGenerator ycsb.Generator + readAllFields bool + writeAllFields bool + dataIntegrity bool + + keySequence ycsb.Generator + operationChooser *generator.Discrete + keyChooser ycsb.Generator + fieldChooser ycsb.Generator + transactionInsertKeySequence *generator.AcknowledgedCounter + scanLength ycsb.Generator + orderedInserts bool + recordCount int64 + zeroPadding int64 + insertionRetryLimit int64 + insertionRetryInterval int64 + + valuePool sync.Pool +} + +func getFieldLengthGenerator(p *properties.Properties) ycsb.Generator { + var fieldLengthGenerator ycsb.Generator + fieldLengthDistribution := p.GetString(prop.FieldLengthDistribution, prop.FieldLengthDistributionDefault) + fieldLength := p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) + fieldLengthHistogram := p.GetString(prop.FieldLengthHistogramFile, prop.FieldLengthHistogramFileDefault) + + switch strings.ToLower(fieldLengthDistribution) { + case "constant": + fieldLengthGenerator = generator.NewConstant(fieldLength) + case "uniform": + fieldLengthGenerator = generator.NewUniform(1, fieldLength) + case "zipfian": + fieldLengthGenerator = generator.NewZipfianWithRange(1, fieldLength, generator.ZipfianConstant) + case "histogram": + fieldLengthGenerator = generator.NewHistogramFromFile(fieldLengthHistogram) + default: + util.Fatalf("unknown field length distribution %s", fieldLengthDistribution) + } + + return fieldLengthGenerator +} + +func createOperationGenerator(p *properties.Properties) *generator.Discrete { + readProportion := p.GetFloat64(prop.ReadProportion, prop.ReadProportionDefault) + updateProportion := p.GetFloat64(prop.UpdateProportion, prop.UpdateProportionDefault) + insertProportion := p.GetFloat64(prop.InsertProportion, prop.InsertProportionDefault) + scanProportion := p.GetFloat64(prop.ScanProportion, prop.ScanProportionDefault) + readModifyWriteProportion := p.GetFloat64(prop.ReadModifyWriteProportion, prop.ReadModifyWriteProportionDefault) + + operationChooser := generator.NewDiscrete() + if readProportion > 0 { + operationChooser.Add(readProportion, int64(read)) + } + + if updateProportion > 0 { + operationChooser.Add(updateProportion, int64(update)) + } + + if insertProportion > 0 { + operationChooser.Add(insertProportion, int64(insert)) + } + + if scanProportion > 0 { + operationChooser.Add(scanProportion, int64(scan)) + } + + if readModifyWriteProportion > 0 { + operationChooser.Add(readModifyWriteProportion, int64(readModifyWrite)) + } + + return operationChooser +} + +// Load implements the Workload Load interface. +func (c *core) Load(ctx context.Context, db ycsb.DB, totalCount int64) error { + return nil +} + +// InitThread implements the Workload InitThread interface. +func (c *core) InitThread(ctx context.Context, _ int, _ int) context.Context { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + fieldNames := make([]string, len(c.fieldNames)) + copy(fieldNames, c.fieldNames) + state := &coreState{ + r: r, + fieldNames: fieldNames, + } + return context.WithValue(ctx, stateKey, state) +} + +// CleanupThread implements the Workload CleanupThread interface. +func (c *core) CleanupThread(_ context.Context) { + +} + +// Close implements the Workload Close interface. +func (c *core) Close() error { + return nil +} + +func (c *core) buildKeyName(keyNum int64) string { + if !c.orderedInserts { + keyNum = util.Hash64(keyNum) + } + + prefix := c.p.GetString(prop.KeyPrefix, prop.KeyPrefixDefault) + return fmt.Sprintf("%s%0[3]*[2]d", prefix, keyNum, c.zeroPadding) +} + +func (c *core) buildSingleValue(state *coreState, key string) map[string][]byte { + values := make(map[string][]byte, 1) + + r := state.r + fieldKey := state.fieldNames[c.fieldChooser.Next(r)] + + var buf []byte + if c.dataIntegrity { + buf = c.buildDeterministicValue(state, key, fieldKey) + } else { + buf = c.buildRandomValue(state) + } + + values[fieldKey] = buf + + return values +} + +func (c *core) buildValues(state *coreState, key string) map[string][]byte { + values := make(map[string][]byte, c.fieldCount) + + for _, fieldKey := range state.fieldNames { + var buf []byte + if c.dataIntegrity { + buf = c.buildDeterministicValue(state, key, fieldKey) + } else { + buf = c.buildRandomValue(state) + } + + values[fieldKey] = buf + } + return values +} + +func (c *core) getValueBuffer(size int) []byte { + buf := c.valuePool.Get().([]byte) + if cap(buf) >= size { + return buf[0:size] + } + + return make([]byte, size) +} + +func (c *core) putValues(values map[string][]byte) { + for _, value := range values { + c.valuePool.Put(value) + } +} + +func (c *core) buildRandomValue(state *coreState) []byte { + // TODO: use pool for the buffer + r := state.r + buf := c.getValueBuffer(int(c.fieldLengthGenerator.Next(r))) + util.RandBytes(r, buf) + return buf +} + +func (c *core) buildDeterministicValue(state *coreState, key string, fieldKey string) []byte { + // TODO: use pool for the buffer + r := state.r + size := c.fieldLengthGenerator.Next(r) + buf := c.getValueBuffer(int(size + 21)) + b := bytes.NewBuffer(buf[0:0]) + b.WriteString(key) + b.WriteByte(':') + b.WriteString(strings.ToLower(fieldKey)) + for int64(b.Len()) < size { + b.WriteByte(':') + n := util.BytesHash64(b.Bytes()) + b.WriteString(strconv.FormatUint(uint64(n), 10)) + } + b.Truncate(int(size)) + return b.Bytes() +} + +func (c *core) verifyRow(state *coreState, key string, values map[string][]byte) { + if len(values) == 0 { + // null data here, need panic? + return + } + + for fieldKey, value := range values { + expected := c.buildDeterministicValue(state, key, fieldKey) + if !bytes.Equal(expected, value) { + util.Fatalf("unexpected deterministic value, expect %q, but got %q", expected, value) + } + } +} + +// DoInsert implements the Workload DoInsert interface. +func (c *core) DoInsert(ctx context.Context, db ycsb.DB) error { + state := ctx.Value(stateKey).(*coreState) + r := state.r + keyNum := c.keySequence.Next(r) + dbKey := c.buildKeyName(keyNum) + values := c.buildValues(state, dbKey) + defer c.putValues(values) + + numOfRetries := int64(0) + + var err error + for { + err = db.Insert(ctx, c.table, dbKey, values) + if err == nil { + break + } + + select { + case <-ctx.Done(): + if ctx.Err() == context.Canceled { + return nil + } + default: + } + + // Retry if configured. Without retrying, the load process will fail + // even if one single insertion fails. User can optionally configure + // an insertion retry limit (default is 0) to enable retry. + numOfRetries++ + if numOfRetries > c.insertionRetryLimit { + break + } + + // Sleep for a random time betweensz [0.8, 1.2)*insertionRetryInterval + sleepTimeMs := float64((c.insertionRetryInterval * 1000)) * (0.8 + 0.4*r.Float64()) + + time.Sleep(time.Duration(sleepTimeMs) * time.Millisecond) + } + + return err +} + +// DoBatchInsert implements the Workload DoBatchInsert interface. +func (c *core) DoBatchInsert(ctx context.Context, batchSize int, db ycsb.DB) error { + batchDB, ok := db.(ycsb.BatchDB) + if !ok { + return fmt.Errorf("the %T does't implement the batchDB interface", db) + } + state := ctx.Value(stateKey).(*coreState) + r := state.r + var keys []string + var values []map[string][]byte + for i := 0; i < batchSize; i++ { + keyNum := c.keySequence.Next(r) + dbKey := c.buildKeyName(keyNum) + keys = append(keys, dbKey) + values = append(values, c.buildValues(state, dbKey)) + } + defer func() { + for _, value := range values { + c.putValues(value) + } + }() + + numOfRetries := int64(0) + var err error + for { + err = batchDB.BatchInsert(ctx, c.table, keys, values) + if err == nil { + break + } + + select { + case <-ctx.Done(): + if ctx.Err() == context.Canceled { + return nil + } + default: + } + + // Retry if configured. Without retrying, the load process will fail + // even if one single insertion fails. User can optionally configure + // an insertion retry limit (default is 0) to enable retry. + numOfRetries++ + if numOfRetries > c.insertionRetryLimit { + break + } + + // Sleep for a random time betweensz [0.8, 1.2)*insertionRetryInterval + sleepTimeMs := float64((c.insertionRetryInterval * 1000)) * (0.8 + 0.4*r.Float64()) + + time.Sleep(time.Duration(sleepTimeMs) * time.Millisecond) + } + return err +} + +// DoTransaction implements the Workload DoTransaction interface. +func (c *core) DoTransaction(ctx context.Context, db ycsb.DB) error { + state := ctx.Value(stateKey).(*coreState) + r := state.r + + operation := operationType(c.operationChooser.Next(r)) + switch operation { + case read: + return c.doTransactionRead(ctx, db, state) + case update: + return c.doTransactionUpdate(ctx, db, state) + case insert: + return c.doTransactionInsert(ctx, db, state) + case scan: + return c.doTransactionScan(ctx, db, state) + default: + return c.doTransactionReadModifyWrite(ctx, db, state) + } +} + +// DoBatchTransaction implements the Workload DoBatchTransaction interface +func (c *core) DoBatchTransaction(ctx context.Context, batchSize int, db ycsb.DB) error { + batchDB, ok := db.(ycsb.BatchDB) + if !ok { + return fmt.Errorf("the %T does't implement the batchDB interface", db) + } + state := ctx.Value(stateKey).(*coreState) + r := state.r + + operation := operationType(c.operationChooser.Next(r)) + switch operation { + case read: + return c.doBatchTransactionRead(ctx, batchSize, batchDB, state) + case insert: + return c.doBatchTransactionInsert(ctx, batchSize, batchDB, state) + case update: + return c.doBatchTransactionUpdate(ctx, batchSize, batchDB, state) + case scan: + panic("The batch mode don't support the scan operation") + default: + return nil + } +} + +func (c *core) nextKeyNum(state *coreState) int64 { + r := state.r + keyNum := int64(0) + if _, ok := c.keyChooser.(*generator.Exponential); ok { + keyNum = -1 + for keyNum < 0 { + keyNum = c.transactionInsertKeySequence.Last() - c.keyChooser.Next(r) + } + } else { + keyNum = c.keyChooser.Next(r) + } + return keyNum +} + +func (c *core) doTransactionRead(ctx context.Context, db ycsb.DB, state *coreState) error { + r := state.r + keyNum := c.nextKeyNum(state) + keyName := c.buildKeyName(keyNum) + + var fields []string + if !c.readAllFields { + fieldName := state.fieldNames[c.fieldChooser.Next(r)] + fields = append(fields, fieldName) + } else { + fields = state.fieldNames + } + + values, err := db.Read(ctx, c.table, keyName, fields) + if err != nil { + return err + } + + if c.dataIntegrity { + c.verifyRow(state, keyName, values) + } + + return nil +} + +func (c *core) doTransactionReadModifyWrite(ctx context.Context, db ycsb.DB, state *coreState) error { + start := time.Now() + defer func() { + measurement.Measure("READ_MODIFY_WRITE", start, time.Now().Sub(start)) + }() + + r := state.r + keyNum := c.nextKeyNum(state) + keyName := c.buildKeyName(keyNum) + + var fields []string + if !c.readAllFields { + fieldName := state.fieldNames[c.fieldChooser.Next(r)] + fields = append(fields, fieldName) + } else { + fields = state.fieldNames + } + + var values map[string][]byte + if c.writeAllFields { + values = c.buildValues(state, keyName) + } else { + values = c.buildSingleValue(state, keyName) + } + defer c.putValues(values) + + readValues, err := db.Read(ctx, c.table, keyName, fields) + if err != nil { + return err + } + + if err := db.Update(ctx, c.table, keyName, values); err != nil { + return err + } + + if c.dataIntegrity { + c.verifyRow(state, keyName, readValues) + } + + return nil +} + +func (c *core) doTransactionInsert(ctx context.Context, db ycsb.DB, state *coreState) error { + r := state.r + keyNum := c.transactionInsertKeySequence.Next(r) + defer c.transactionInsertKeySequence.Acknowledge(keyNum) + dbKey := c.buildKeyName(keyNum) + values := c.buildValues(state, dbKey) + defer c.putValues(values) + + return db.Insert(ctx, c.table, dbKey, values) +} + +func (c *core) doTransactionScan(ctx context.Context, db ycsb.DB, state *coreState) error { + r := state.r + keyNum := c.nextKeyNum(state) + startKeyName := c.buildKeyName(keyNum) + + scanLen := c.scanLength.Next(r) + + var fields []string + if !c.readAllFields { + fieldName := state.fieldNames[c.fieldChooser.Next(r)] + fields = append(fields, fieldName) + } else { + fields = state.fieldNames + } + + _, err := db.Scan(ctx, c.table, startKeyName, int(scanLen), fields) + + return err +} + +func (c *core) doTransactionUpdate(ctx context.Context, db ycsb.DB, state *coreState) error { + keyNum := c.nextKeyNum(state) + keyName := c.buildKeyName(keyNum) + + var values map[string][]byte + if c.writeAllFields { + values = c.buildValues(state, keyName) + } else { + values = c.buildSingleValue(state, keyName) + } + + defer c.putValues(values) + + return db.Update(ctx, c.table, keyName, values) +} + +func (c *core) doBatchTransactionRead(ctx context.Context, batchSize int, db ycsb.BatchDB, state *coreState) error { + r := state.r + var fields []string + + if !c.readAllFields { + fieldName := state.fieldNames[c.fieldChooser.Next(r)] + fields = append(fields, fieldName) + } else { + fields = state.fieldNames + } + + keys := make([]string, batchSize) + for i := 0; i < batchSize; i++ { + keys[i] = c.buildKeyName(c.nextKeyNum(state)) + } + + _, err := db.BatchRead(ctx, c.table, keys, fields) + if err != nil { + return err + } + + // TODO should we verify the result? + return nil +} + +func (c *core) doBatchTransactionInsert(ctx context.Context, batchSize int, db ycsb.BatchDB, state *coreState) error { + r := state.r + keys := make([]string, batchSize) + values := make([]map[string][]byte, batchSize) + for i := 0; i < batchSize; i++ { + keyNum := c.transactionInsertKeySequence.Next(r) + keyName := c.buildKeyName(keyNum) + keys[i] = keyName + if c.writeAllFields { + values[i] = c.buildValues(state, keyName) + } else { + values[i] = c.buildSingleValue(state, keyName) + } + c.transactionInsertKeySequence.Acknowledge(keyNum) + } + + defer func() { + for _, value := range values { + c.putValues(value) + } + }() + + return db.BatchInsert(ctx, c.table, keys, values) +} + +func (c *core) doBatchTransactionUpdate(ctx context.Context, batchSize int, db ycsb.BatchDB, state *coreState) error { + keys := make([]string, batchSize) + values := make([]map[string][]byte, batchSize) + for i := 0; i < batchSize; i++ { + keyNum := c.nextKeyNum(state) + keyName := c.buildKeyName(keyNum) + keys[i] = keyName + if c.writeAllFields { + values[i] = c.buildValues(state, keyName) + } else { + values[i] = c.buildSingleValue(state, keyName) + } + } + + defer func() { + for _, value := range values { + c.putValues(value) + } + }() + + return db.BatchUpdate(ctx, c.table, keys, values) +} + +// CoreCreator creates the Core workload. +type coreCreator struct { +} + +// Create implements the WorkloadCreator Create interface. +func (coreCreator) Create(p *properties.Properties) (ycsb.Workload, error) { + c := new(core) + c.p = p + c.table = p.GetString(prop.TableName, prop.TableNameDefault) + c.fieldCount = p.GetInt64(prop.FieldCount, prop.FieldCountDefault) + c.fieldNames = make([]string, c.fieldCount) + for i := int64(0); i < c.fieldCount; i++ { + c.fieldNames[i] = fmt.Sprintf("field%d", i) + } + c.fieldLengthGenerator = getFieldLengthGenerator(p) + c.recordCount = p.GetInt64(prop.RecordCount, prop.RecordCountDefault) + if c.recordCount == 0 { + c.recordCount = int64(math.MaxInt32) + } + + requestDistrib := p.GetString(prop.RequestDistribution, prop.RequestDistributionDefault) + maxScanLength := p.GetInt64(prop.MaxScanLength, prop.MaxScanLengthDefault) + scanLengthDistrib := p.GetString(prop.ScanLengthDistribution, prop.ScanLengthDistributionDefault) + + insertStart := p.GetInt64(prop.InsertStart, prop.InsertStartDefault) + insertCount := p.GetInt64(prop.InsertCount, c.recordCount-insertStart) + if c.recordCount < insertStart+insertCount { + util.Fatalf("record count %d must be bigger than insert start %d + count %d", + c.recordCount, insertStart, insertCount) + } + c.zeroPadding = p.GetInt64(prop.ZeroPadding, prop.ZeroPaddingDefault) + c.readAllFields = p.GetBool(prop.ReadAllFields, prop.ReadALlFieldsDefault) + c.writeAllFields = p.GetBool(prop.WriteAllFields, prop.WriteAllFieldsDefault) + c.dataIntegrity = p.GetBool(prop.DataIntegrity, prop.DataIntegrityDefault) + fieldLengthDistribution := p.GetString(prop.FieldLengthDistribution, prop.FieldLengthDistributionDefault) + if c.dataIntegrity && fieldLengthDistribution != "constant" { + util.Fatal("must have constant field size to check data integrity") + } + + if p.GetString(prop.InsertOrder, prop.InsertOrderDefault) == "hashed" { + c.orderedInserts = false + } else { + c.orderedInserts = true + } + + c.keySequence = generator.NewCounter(insertStart) + c.operationChooser = createOperationGenerator(p) + var keyrangeLowerBound int64 = insertStart + var keyrangeUpperBound int64 = insertStart + insertCount - 1 + + c.transactionInsertKeySequence = generator.NewAcknowledgedCounter(c.recordCount) + switch requestDistrib { + case "uniform": + c.keyChooser = generator.NewUniform(keyrangeLowerBound, keyrangeUpperBound) + case "sequential": + c.keyChooser = generator.NewSequential(keyrangeLowerBound, keyrangeUpperBound) + case "zipfian": + insertProportion := p.GetFloat64(prop.InsertProportion, prop.InsertProportionDefault) + opCount := p.GetInt64(prop.OperationCount, 0) + expectedNewKeys := int64(float64(opCount) * insertProportion * 2.0) + keyrangeUpperBound = insertStart + insertCount + expectedNewKeys + c.keyChooser = generator.NewScrambledZipfian(keyrangeLowerBound, keyrangeUpperBound, generator.ZipfianConstant) + case "latest": + c.keyChooser = generator.NewSkewedLatest(c.transactionInsertKeySequence) + case "hotspot": + hotsetFraction := p.GetFloat64(prop.HotspotDataFraction, prop.HotspotDataFractionDefault) + hotopnFraction := p.GetFloat64(prop.HotspotOpnFraction, prop.HotspotOpnFractionDefault) + c.keyChooser = generator.NewHotspot(keyrangeLowerBound, keyrangeUpperBound, hotsetFraction, hotopnFraction) + case "exponential": + percentile := p.GetFloat64(prop.ExponentialPercentile, prop.ExponentialPercentileDefault) + frac := p.GetFloat64(prop.ExponentialFrac, prop.ExponentialFracDefault) + c.keyChooser = generator.NewExponential(percentile, float64(c.recordCount)*frac) + default: + util.Fatalf("unknown request distribution %s", requestDistrib) + } + fmt.Println(fmt.Sprintf("Using request distribution '%s' a keyrange of [%d %d]", requestDistrib, keyrangeLowerBound, keyrangeUpperBound)) + + c.fieldChooser = generator.NewUniform(0, c.fieldCount-1) + switch scanLengthDistrib { + case "uniform": + c.scanLength = generator.NewUniform(1, maxScanLength) + case "zipfian": + c.scanLength = generator.NewZipfianWithRange(1, maxScanLength, generator.ZipfianConstant) + default: + util.Fatalf("distribution %s not allowed for scan length", scanLengthDistrib) + } + + c.insertionRetryLimit = p.GetInt64(prop.InsertionRetryLimit, prop.InsertionRetryLimitDefault) + c.insertionRetryInterval = p.GetInt64(prop.InsertionRetryInterval, prop.InsertionRetryIntervalDefault) + + fieldLength := p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) + c.valuePool = sync.Pool{ + New: func() interface{} { + return make([]byte, fieldLength) + }, + } + + return c, nil +} + +func init() { + ycsb.RegisterWorkloadCreator("core", coreCreator{}) +} diff --git a/go-ycsb/pkg/ycsb/db.go b/go-ycsb/pkg/ycsb/db.go new file mode 100644 index 000000000..c4545c835 --- /dev/null +++ b/go-ycsb/pkg/ycsb/db.go @@ -0,0 +1,120 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package ycsb + +import ( + "context" + "fmt" + + "github.com/magiconair/properties" +) + +// DBCreator creates a database layer. +type DBCreator interface { + Create(p *properties.Properties) (DB, error) +} + +// DB is the layer to access the database to be benchmarked. +type DB interface { + // Close closes the database layer. + Close() error + + // InitThread initializes the state associated to the goroutine worker. + // The Returned context will be passed to the following usage. + InitThread(ctx context.Context, threadID int, threadCount int) context.Context + + // CleanupThread cleans up the state when the worker finished. + CleanupThread(ctx context.Context) + + // Read reads a record from the database and returns a map of each field/value pair. + // table: The name of the table. + // key: The record key of the record to read. + // fields: The list of fields to read, nil|empty for reading all. + Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) + + // Scan scans records from the database. + // table: The name of the table. + // startKey: The first record key to read. + // count: The number of records to read. + // fields: The list of fields to read, nil|empty for reading all. + Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) + + // Update updates a record in the database. Any field/value pairs will be written into the + // database or overwritten the existing values with the same field name. + // table: The name of the table. + // key: The record key of the record to update. + // values: A map of field/value pairs to update in the record. + Update(ctx context.Context, table string, key string, values map[string][]byte) error + + // Insert inserts a record in the database. Any field/value pairs will be written into the + // database. + // table: The name of the table. + // key: The record key of the record to insert. + // values: A map of field/value pairs to insert in the record. + Insert(ctx context.Context, table string, key string, values map[string][]byte) error + + // Delete deletes a record from the database. + // table: The name of the table. + // key: The record key of the record to delete. + Delete(ctx context.Context, table string, key string) error +} + +type BatchDB interface { + // BatchInsert inserts batch records in the database. + // table: The name of the table. + // keys: The keys of batch records. + // values: The values of batch records. + BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) error + + // BatchRead reads records from the database. + // table: The name of the table. + // keys: The keys of records to read. + // fields: The list of fields to read, nil|empty for reading all. + BatchRead(ctx context.Context, table string, keys []string, fields []string) ([]map[string][]byte, error) + + // BatchUpdate updates records in the database. + // table: The name of table. + // keys: The keys of records to update. + // values: The values of records to update. + BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) error + + // BatchDelete deletes records from the database. + // table: The name of the table. + // keys: The keys of the records to delete. + BatchDelete(ctx context.Context, table string, keys []string) error +} + +// AnalyzeDB is the interface for the DB that can perform an analysis on given table. +type AnalyzeDB interface { + // Analyze performs a key distribution analysis for the table. + // table: The name of the table. + Analyze(ctx context.Context, table string) error +} + +var dbCreators = map[string]DBCreator{} + +// RegisterDBCreator registers a creator for the database +func RegisterDBCreator(name string, creator DBCreator) { + _, ok := dbCreators[name] + if ok { + panic(fmt.Sprintf("duplicate register database %s", name)) + } + + dbCreators[name] = creator +} + +// GetDBCreator gets the DBCreator for the database +func GetDBCreator(name string) DBCreator { + return dbCreators[name] +} diff --git a/go-ycsb/pkg/ycsb/generator.go b/go-ycsb/pkg/ycsb/generator.go new file mode 100644 index 000000000..cce4cf6cc --- /dev/null +++ b/go-ycsb/pkg/ycsb/generator.go @@ -0,0 +1,28 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package ycsb + +import "math/rand" + +// Generator generates a sequence of values, following some distribution (Uniform, Zipfian, etc.). +type Generator interface { + // Next generates the next value in the distribution. + Next(r *rand.Rand) int64 + + // Last returns the previous value generated by the distribution, e.g. the + // last Next call. + // Calling Last should not advance the distribution or have any side effect. + // If Next has not been called, Last should return something reasonable. + Last() int64 +} diff --git a/go-ycsb/pkg/ycsb/measurement.go b/go-ycsb/pkg/ycsb/measurement.go new file mode 100644 index 000000000..60e4c46c8 --- /dev/null +++ b/go-ycsb/pkg/ycsb/measurement.go @@ -0,0 +1,31 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package ycsb + +import ( + "io" + "time" +) + +// Measurer is used to capture measurements. +type Measurer interface { + // Measure measures the latency of an operation. + Measure(op string, start time.Time, latency time.Duration) + + // Summary writes a summary of the current measurement results to stdout. + Summary() + + // Output writes the measurement results to the specified writer. + Output(w io.Writer) error +} diff --git a/go-ycsb/pkg/ycsb/workload.go b/go-ycsb/pkg/ycsb/workload.go new file mode 100644 index 000000000..7be59ed4a --- /dev/null +++ b/go-ycsb/pkg/ycsb/workload.go @@ -0,0 +1,71 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package ycsb + +import ( + "context" + "fmt" + + "github.com/magiconair/properties" +) + +// WorkloadCreator creates a Workload +type WorkloadCreator interface { + Create(p *properties.Properties) (Workload, error) +} + +// Workload defines different workload for YCSB. +type Workload interface { + // Close closes the workload. + Close() error + + // InitThread initializes the state associated to the goroutine worker. + // The Returned context will be passed to the following DoInsert and DoTransaction. + InitThread(ctx context.Context, threadID int, threadCount int) context.Context + + // CleanupThread cleans up the state when the worker finished. + CleanupThread(ctx context.Context) + + // Load data into DB. + Load(ctx context.Context, db DB, totalCount int64) error + + // DoInsert does one insert operation. + DoInsert(ctx context.Context, db DB) error + + // DoBatchInsert does batch insert. + DoBatchInsert(ctx context.Context, batchSize int, db DB) error + + // DoTransaction does one transaction operation. + DoTransaction(ctx context.Context, db DB) error + + // DoBatchTransaction does the batch transaction operation. + DoBatchTransaction(ctx context.Context, batchSize int, db DB) error +} + +var workloadCreators = map[string]WorkloadCreator{} + +// RegisterWorkloadCreator registers a creator for the workload +func RegisterWorkloadCreator(name string, creator WorkloadCreator) { + _, ok := workloadCreators[name] + if ok { + panic(fmt.Sprintf("duplicate register workload %s", name)) + } + + workloadCreators[name] = creator +} + +// GetWorkloadCreator gets the WorkloadCreator for the database +func GetWorkloadCreator(name string) WorkloadCreator { + return workloadCreators[name] +} diff --git a/go-ycsb/tool/binary/bench.sh b/go-ycsb/tool/binary/bench.sh new file mode 100755 index 000000000..864a3084f --- /dev/null +++ b/go-ycsb/tool/binary/bench.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +TYPE=$1 +DB=$2 + +# Directory to save data +DATA=./data +CMD=../../bin/go-ycsb +# Direcotry to save logs +LOG=./logs + +RECORDCOUNT=100000000 +OPERATIONCOUNT=100000000 +THREADCOUNT=16 +FIELDCOUNT=10 +FIELDLENGTH=100 +MAXSCANLENGTH=10 + +PROPS="-p recordcount=${RECORDCOUNT} \ + -p operationcount=${OPERATIONCOUNT} \ + -p threadcount=${THREADCOUNT} \ + -p fieldcount=${FIELDCOUNT} \ + -p fieldlength=${FIELDLENGTH} \ + -p maxscanlength=${MAXSCANLENGTH}" +PROPS+=" ${@:3}" +WORKLOADS= + +mkdir -p ${LOG} + +DBDATA=${DATA}/${DB} + +if [ ${DB} == 'rocksdb' ]; then + PROPS+=" -p rocksdb.dir=${DBDATA}" + WORKLOADS="-P property/rocksdb" +elif [ ${DB} == 'badger' ]; then + PROPS+=" -p badger.dir=${DBDATA}" + WORKLOADS="-P property/badger" +fi + + +if [ ${TYPE} == 'load' ]; then + echo "clear data before load" + PROPS+=" -p dropdata=true" +fi + +echo ${TYPE} ${DB} ${WORKLOADS} ${PROPS} + +if [ ${TYPE} == 'load' ]; then + $CMD load ${DB} ${WORKLOADS} -p=workload=core ${PROPS} | tee ${LOG}/${DB}_load.log +elif [ ${TYPE} == 'run' ]; then + for workload in a b c d e f + do + $CMD run ${DB} -P ../../workloads/workload${workload} ${WORKLOADS} ${PROPS} | tee ${LOG}/${DB}_workload${workload}.log + done +else + echo "invalid type ${TYPE}" + exit 1 +fi + diff --git a/go-ycsb/tool/binary/property/badger b/go-ycsb/tool/binary/property/badger new file mode 100644 index 000000000..3028b9214 --- /dev/null +++ b/go-ycsb/tool/binary/property/badger @@ -0,0 +1,16 @@ +badger.sync_writes = false +badger.num_versions_to_keep = 1 +badger.max_table_size = 67108864 +badger.level_size_multiplier = 10 +badger.max_levels = 7 +badger.value_threshold = 32 +badger.num_memtables = 5 +badger.num_level0_tables = 5 +badger.num_level0_tables_stall = 10 +badger.level_one_size = 268435456 +badger.value_log_file_size = 1073741824 +badger.value_log_max_entries = 1000000 +badger.num_compactors = 3 +badger.do_not_compact = false +badger.table_loading_mode = LoadToRAM +badger.value_log_loading_mode = MemoryMap \ No newline at end of file diff --git a/go-ycsb/tool/binary/property/rocksdb b/go-ycsb/tool/binary/property/rocksdb new file mode 100644 index 000000000..778addd9e --- /dev/null +++ b/go-ycsb/tool/binary/property/rocksdb @@ -0,0 +1,27 @@ +rocksdb.allow_concurrent_memtable_writes = true +rocksdb.allow_mmap_reads = false +rocksdb.allow_mmap_writes = false +rocksdb.arena_block_size = 0 +rocksdb.db_write_buffer_size = 0 +rocksdb.hard_pending_compaction_bytes_limit = 274877906944 +rocksdb.level0_file_num_compaction_trigger = 4 +rocksdb.level0_slowdown_writes_trigger = 20 +rocksdb.level0_stop_writes_trigger = 36 +rocksdb.max_background_flushes = 1 +rocksdb.max_bytes_for_level_base = 268435456 +rocksdb.max_bytes_for_level_multiplier = 10 +rocksdb.max_total_wal_size = 0 +rocksdb.memtable_huge_page_size = 0 +rocksdb.num_levels = 7 +rocksdb.use_direct_reads = false +rocksdb.use_fsync = false +rocksdb.write_buffer_size = 67108864 + +# TableOptions/BlockBasedTable +rocksdb.block_size = 4096 +rocksdb.block_size_deviation = 10 +rocksdb.cache_index_and_filter_blocks = false +rocksdb.no_block_cache = false +rocksdb.pin_l0_filter_and_index_blocks_in_cache = false +rocksdb.whole_key_filtering = true +rocksdb.block_restart_interval = 16 diff --git a/go-ycsb/tool/docker/bench.sh b/go-ycsb/tool/docker/bench.sh new file mode 100755 index 000000000..0dea3c38a --- /dev/null +++ b/go-ycsb/tool/docker/bench.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +TYPE=$1 +DB=$2 + +# Direcotry to save logs +LOG=./logs + +RECORDCOUNT=100000 +OPERATIONCOUNT=100000 +THREADCOUNT=20 +FIELDCOUNT=5 +FIELDLENGTH=16 +MAXSCANLENGTH=10 + +PROPS="-p recordcount=${RECORDCOUNT} \ + -p operationcount=${OPERATIONCOUNT} \ + -p threadcount=${THREADCOUNT} \ + -p fieldcount=${FIELDCOUNT} \ + -p fieldlength=${FIELDLENGTH} \ + -p maxscanlength=${MAXSCANLENGTH}" +PROPS+=" ${@:3}" +WORKLOADS= +SLEEPTIME=10 + +mkdir -p ${LOG} + +BENCH_DB=${DB} + +case ${DB} in + mysql) + PROPS+=" -p mysql.host=mysql" + SLEEPTIME=30 + ;; + mysql8) + PROPS+=" -p mysql.host=mysql" + SLEEPTIME=30 + DB="mysql" + ;; + mariadb) + PROPS+=" -p mysql.host=mariadb" + SLEEPTIME=60 + DB="mysql" + ;; + pg) + PROPS+=" -p pg.host=pg" + SLEEPTIME=30 + ;; + tikv) + PROPS+=" -p tikv.pd=pd:2379 -p tikv.type=txn" + ;; + raw) + PROPS+=" -p tikv.pd=pd:2379 -p tikv.type=raw" + DB="tikv" + ;; + tidb) + PROPS+=" -p mysql.host=tidb -p mysql.port=4000" + ;; + cockroach) + PROPS+=" -p pg.host=cockroach -p pg.port=26257" + ;; + sqlite) + PROPS+=" -p sqlite.db=/data/sqlite.db" + ;; + cassandra) + PROPS+=" -p cassandra.cluster=cassandra" + SLEEPTIME=30 + ;; + scylla) + PROPS+=" -p cassandra.cluster=scylla" + SLEEPTIME=30 + ;; + *) + ;; +esac + +echo ${TYPE} ${DB} ${WORKLOADS} ${PROPS} + +CMD="docker-compose -f ${BENCH_DB}.yml" + +if [ ${TYPE} == 'load' ]; then + $CMD down --remove-orphans + rm -rf ./data/${BENCH_DB} + $CMD up -d + sleep ${SLEEPTIME} + + $CMD run ycsb load ${DB} ${WORKLOADS} -p workload=core ${PROPS} | tee ${LOG}/${BENCH_DB}_load.log + + $CMD down +elif [ ${TYPE} == 'run' ]; then + $CMD up -d + sleep ${SLEEPTIME} + + for workload in a b c d e f + do + $CMD run --rm ycsb run ${DB} -P ../../workloads/workload${workload} ${WORKLOADS} ${PROPS} | tee ${LOG}/${BENCH_DB}_workload${workload}.log + done + + $CMD down +else + echo "invalid type ${TYPE}" + exit 1 +fi diff --git a/go-ycsb/tool/docker/cassandra.yml b/go-ycsb/tool/docker/cassandra.yml new file mode 100644 index 000000000..0e43a65a4 --- /dev/null +++ b/go-ycsb/tool/docker/cassandra.yml @@ -0,0 +1,24 @@ +version: '2.1' + +services: + cassandra: + image: cassandra:latest + volumes: + - ./data/cassandra:/var/lib/cassandra + ports: + - "9042:9042" + restart: on-failure + + db-init: + image: cassandra:latest + volumes: + - ./config/init_cassandra.sh:/init_cassandra.sh + entrypoint: + - "/bin/bash" + command: /init_cassandra.sh cassandra + depends_on: + - "cassandra" + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/clear.sh b/go-ycsb/tool/docker/clear.sh new file mode 100755 index 000000000..ddf27ad2f --- /dev/null +++ b/go-ycsb/tool/docker/clear.sh @@ -0,0 +1,7 @@ + +for db in pg cockroach mysql mysql8 tidb tikv +do + docker-compose -f ${db}.yml down --remove-orphans +done + +rm -rf ./data \ No newline at end of file diff --git a/go-ycsb/tool/docker/cockroach.yml b/go-ycsb/tool/docker/cockroach.yml new file mode 100644 index 000000000..a6181636e --- /dev/null +++ b/go-ycsb/tool/docker/cockroach.yml @@ -0,0 +1,26 @@ +version: '2.1' + +services: + cockroach: + image: cockroachdb/cockroach:latest + volumes: + - ./data/cockroach:/cockroach/cockroach-data + ports: + - "26257:26257" + - "8080:8080" + command: start --insecure + restart: on-failure + + db-init: + image: cockroachdb/cockroach:latest + volumes: + - ./config/init_cockroach.sh:/init_cockroach.sh + entrypoint: + - "/bin/bash" + command: /init_cockroach.sh + depends_on: + - "cockroach" + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/config/init_cassandra.sh b/go-ycsb/tool/docker/config/init_cassandra.sh new file mode 100755 index 000000000..c6e64ee5c --- /dev/null +++ b/go-ycsb/tool/docker/config/init_cassandra.sh @@ -0,0 +1,14 @@ +#!/bin/bash +echo "Wait for servers to be up" + +for _ in {1..10} +do + sleep 5 + cqlsh $1 -e "CREATE KEYSPACE test WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor' : 1};" + if [ $? -eq 0 ] + then + echo "create keyspace ok" + break + fi + echo "create keyspace failed, wait 5s" +done diff --git a/go-ycsb/tool/docker/config/init_cockroach.sh b/go-ycsb/tool/docker/config/init_cockroach.sh new file mode 100755 index 000000000..1996f2689 --- /dev/null +++ b/go-ycsb/tool/docker/config/init_cockroach.sh @@ -0,0 +1,8 @@ +#!/bin/bash +echo "Wait for servers to be up" +sleep 5 + +HOSTPARAMS="--host cockroach --insecure" +SQL="/cockroach/cockroach.sh sql $HOSTPARAMS" + +$SQL -e "CREATE DATABASE test;" \ No newline at end of file diff --git a/go-ycsb/tool/docker/config/pd.toml b/go-ycsb/tool/docker/config/pd.toml new file mode 100644 index 000000000..e091e695b --- /dev/null +++ b/go-ycsb/tool/docker/config/pd.toml @@ -0,0 +1,91 @@ +# PD Configuration. + +name = "pd" +data-dir = "default.pd" + +client-urls = "http://127.0.0.1:2379" +# if not set, use ${client-urls} +advertise-client-urls = "" + +peer-urls = "http://127.0.0.1:2380" +# if not set, use ${peer-urls} +advertise-peer-urls = "" + +initial-cluster = "pd=http://127.0.0.1:2380" +initial-cluster-state = "new" + +lease = 3 +tso-save-interval = "3s" + +namespace-classifier = "table" + +enable-prevote = true + +[security] +# Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty +cacert-path = "" +# Path of file that contains X509 certificate in PEM format. +cert-path = "" +# Path of file that contains X509 key in PEM format. +key-path = "" + +[log] +level = "info" + +# log format, one of json, text, console +#format = "text" + +# disable automatic timestamps in output +#disable-timestamp = false + +# file logging +[log.file] +#filename = "" +# max log file size in MB +#max-size = 300 +# max log file keep days +#max-days = 28 +# maximum number of old log files to retain +#max-backups = 7 +# rotate log by day +#log-rotate = true + +[metric] +# prometheus client push interval, set "0s" to disable prometheus. +interval = "15s" +# prometheus pushgateway address, leaves it empty will disable prometheus. +address = "" + +[schedule] +max-merge-region-size = 20 +max-merge-region-keys = 200000 +split-merge-interval = "1h" +max-snapshot-count = 3 +max-pending-peer-count = 16 +max-store-down-time = "30m" +leader-schedule-limit = 4 +region-schedule-limit = 4 +replica-schedule-limit = 8 +merge-schedule-limit = 8 +tolerant-size-ratio = 5.0 + +# customized schedulers, the format is as below +# if empty, it will use balance-leader, balance-region, hot-region as default +# [[schedule.schedulers]] +# type = "evict-leader" +# args = ["1"] + +[replication] +# The number of replicas for each region. +max-replicas = 3 +# The label keys specified the location of a store. +# The placement priorities is implied by the order of label keys. +# For example, ["zone", "rack"] means that we should place replicas to +# different zones first, then to different racks if we don't have enough zones. +location-labels = [] + +[label-property] +# Do not assign region leaders to stores that have these tags. +# [[label-property.reject-leader]] +# key = "zone" +# value = "cn1 diff --git a/go-ycsb/tool/docker/config/tidb.toml b/go-ycsb/tool/docker/config/tidb.toml new file mode 100644 index 000000000..376052d49 --- /dev/null +++ b/go-ycsb/tool/docker/config/tidb.toml @@ -0,0 +1,260 @@ +# TiDB Configuration. + +# TiDB server host. +host = "0.0.0.0" + +# tidb server advertise IP. +advertise-address = "" + +# TiDB server port. +port = 4000 + +# Registered store name, [tikv, mocktikv] +store = "mocktikv" + +# TiDB storage path. +path = "/tmp/tidb" + +# The socket file to use for connection. +socket = "" + +# Run ddl worker on this tidb-server. +run-ddl = true + +# Schema lease duration, very dangerous to change only if you know what you do. +lease = "45s" + +# When create table, split a separated region for it. It is recommended to +# turn off this option if there will be a large number of tables created. +split-table = true + +# The limit of concurrent executed sessions. +token-limit = 1000 + +# Only print a log when out of memory quota. +# Valid options: ["log", "cancel"] +oom-action = "log" + +# Set the memory quota for a query in bytes. Default: 32GB +mem-quota-query = 34359738368 + +# Enable coprocessor streaming. +enable-streaming = false + +# Set system variable 'lower_case_table_names' +lower-case-table-names = 2 + +# Make "kill query" behavior compatible with MySQL. It's not recommend to +# turn on this option when TiDB server is behind a proxy. +compatible-kill-query = false + +[log] +# Log level: debug, info, warn, error, fatal. +level = "info" + +# Log format, one of json, text, console. +format = "text" + +# Disable automatic timestamp in output +disable-timestamp = false + +# Stores slow query log into separated files. +slow-query-file = "" + +# Queries with execution time greater than this value will be logged. (Milliseconds) +slow-threshold = 300 + +# Queries with internal result greater than this value will be logged. +expensive-threshold = 10000 + +# Maximum query length recorded in log. +query-log-max-len = 2048 + +# File logging. +[log.file] +# Log file name. +filename = "" + +# Max log file size in MB (upper limit to 4096MB). +max-size = 300 + +# Max log file keep days. No clean up by default. +max-days = 0 + +# Maximum number of old log files to retain. No clean up by default. +max-backups = 0 + +# Rotate log by day +log-rotate = true + +[security] +# Path of file that contains list of trusted SSL CAs for connection with mysql client. +ssl-ca = "" + +# Path of file that contains X509 certificate in PEM format for connection with mysql client. +ssl-cert = "" + +# Path of file that contains X509 key in PEM format for connection with mysql client. +ssl-key = "" + +# Path of file that contains list of trusted SSL CAs for connection with cluster components. +cluster-ssl-ca = "" + +# Path of file that contains X509 certificate in PEM format for connection with cluster components. +cluster-ssl-cert = "" + +# Path of file that contains X509 key in PEM format for connection with cluster components. +cluster-ssl-key = "" + +[status] +# If enable status report HTTP service. +report-status = true + +# TiDB status port. +status-port = 10080 + +# Prometheus pushgateway address, leaves it empty will disable prometheus push. +metrics-addr = "" + +# Prometheus client push interval in second, set \"0\" to disable prometheus push. +metrics-interval = 15 + +[performance] +# Max CPUs to use, 0 use number of CPUs in the machine. +max-procs = 0 + +# Max memory size to use, 0 use the total usable memory in the machine. +max-memory = 0 + +# StmtCountLimit limits the max count of statement inside a transaction. +stmt-count-limit = 5000 + +# Set keep alive option for tcp connection. +tcp-keep-alive = true + +# Whether support cartesian product. +cross-join = true + +# Stats lease duration, which influences the time of analyze and stats load. +stats-lease = "3s" + +# Run auto analyze worker on this tidb-server. +run-auto-analyze = true + +# Probability to use the query feedback to update stats, 0 or 1 for always false/true. +feedback-probability = 0.05 + +# The max number of query feedback that cache in memory. +query-feedback-limit = 1024 + +# Pseudo stats will be used if the ratio between the modify count and +# row count in statistics of a table is greater than it. +pseudo-estimate-ratio = 0.8 + +# Force the priority of all statements in a specified priority. +# The value could be "NO_PRIORITY", "LOW_PRIORITY", "HIGH_PRIORITY" or "DELAYED". +force-priority = "NO_PRIORITY" + +[proxy-protocol] +# PROXY protocol acceptable client networks. +# Empty string means disable PROXY protocol, * means all networks. +networks = "" + +# PROXY protocol header read timeout, unit is second +header-timeout = 5 + +[prepared-plan-cache] +enabled = false +capacity = 100 +memory-guard-ratio = 0.1 + +[opentracing] +# Enable opentracing. +enable = false + +# Whether to enable the rpc metrics. +rpc-metrics = false + +[opentracing.sampler] +# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote +type = "const" + +# Param is a value passed to the sampler. +# Valid values for Param field are: +# - for "const" sampler, 0 or 1 for always false/true respectively +# - for "probabilistic" sampler, a probability between 0 and 1 +# - for "rateLimiting" sampler, the number of spans per second +# - for "remote" sampler, param is the same as for "probabilistic" +# and indicates the initial sampling rate before the actual one +# is received from the mothership +param = 1.0 + +# SamplingServerURL is the address of jaeger-agent's HTTP sampling server +sampling-server-url = "" + +# MaxOperations is the maximum number of operations that the sampler +# will keep track of. If an operation is not tracked, a default probabilistic +# sampler will be used rather than the per operation specific sampler. +max-operations = 0 + +# SamplingRefreshInterval controls how often the remotely controlled sampler will poll +# jaeger-agent for the appropriate sampling strategy. +sampling-refresh-interval = 0 + +[opentracing.reporter] +# QueueSize controls how many spans the reporter can keep in memory before it starts dropping +# new spans. The queue is continuously drained by a background go-routine, as fast as spans +# can be sent out of process. +queue-size = 0 + +# BufferFlushInterval controls how often the buffer is force-flushed, even if it's not full. +# It is generally not useful, as it only matters for very low traffic services. +buffer-flush-interval = 0 + +# LogSpans, when true, enables LoggingReporter that runs in parallel with the main reporter +# and logs all submitted spans. Main Configuration.Logger must be initialized in the code +# for this option to have any effect. +log-spans = false + +# LocalAgentHostPort instructs reporter to send spans to jaeger-agent at this address +local-agent-host-port = "" + +[tikv-client] +# Max gRPC connections that will be established with each tikv-server. +grpc-connection-count = 16 + +# After a duration of this time in seconds if the client doesn't see any activity it pings +# the server to see if the transport is still alive. +grpc-keepalive-time = 10 + +# After having pinged for keepalive check, the client waits for a duration of Timeout in seconds +# and if no activity is seen even after that the connection is closed. +grpc-keepalive-timeout = 3 + +# max time for commit command, must be twice bigger than raft election timeout. +commit-timeout = "41s" + +# The max time a Txn may use (in seconds) from its startTS to commitTS. +# We use it to guarantee GC worker will not influence any active txn. Please make sure that this +# value is less than gc_life_time - 10s. +max-txn-time-use = 590 + +[txn-local-latches] +# Enable local latches for transactions. Enable it when +# there are lots of conflicts between transactions. +enabled = true +capacity = 2048000 + +[binlog] +# enable to write binlog. +enable = false + +# WriteTimeout specifies how long it will wait for writing binlog to pump. +write-timeout = "15s" + +# If IgnoreError is true, when writting binlog meets error, TiDB would stop writting binlog, +# but still provide service. +ignore-error = false + +# use socket file to write binlog, for compatible with kafka version tidb-binlog. +binlog-socket = "" diff --git a/go-ycsb/tool/docker/config/tikv.toml b/go-ycsb/tool/docker/config/tikv.toml new file mode 100644 index 000000000..29deb85ff --- /dev/null +++ b/go-ycsb/tool/docker/config/tikv.toml @@ -0,0 +1,689 @@ +## TiKV config template +## Human-readable big numbers: +## File size(based on byte): KB, MB, GB, TB, PB +## e.g.: 1_048_576 = "1MB" +## Time(based on ms): ms, s, m, h +## e.g.: 78_000 = "1.3m" + +## Log levels: trace, debug, info, warning, error, critical. +## Note that `debug` and `trace` are only available in development builds. +# log-level = "info" + +## File to store logs. +## If it is not set, logs will be appended to stderr. +# log-file = "" + +## Timespan between rotating the log files. +## Once this timespan passes, log files will be rotated, i.e. existing log file will have a +## timestamp appended to its name and a new file will be created. +# log-rotation-timespan = "24h" + +[readpool.storage] +## Size of the thread pool for high-priority operations. +# high-concurrency = 4 + +## Size of the thread pool for normal-priority operations. +# normal-concurrency = 4 + +## Size of the thread pool for low-priority operations. +# low-concurrency = 4 + +## Max running high-priority operations of each worker, reject if exceeded. +# max-tasks-per-worker-high = 2000 + +## Max running normal-priority operations of each worker, reject if exceeded. +# max-tasks-per-worker-normal = 2000 + +## Max running low-priority operations of each worker, reject if exceeded. +# max-tasks-per-worker-low = 2000 + +## Size of the stack for each thread in the thread pool. +# stack-size = "10MB" + +[readpool.coprocessor] +## Most read requests from TiDB are sent to the coprocessor of TiKV. high/normal/low-concurrency is +## used to set the number of threads of the coprocessor. +## If there are many read requests, you can increase these config values (but keep it within the +## number of system CPU cores). For example, for a 32-core machine deployed with TiKV, you can even +## set these config to 30 in heavy read scenarios. +## If CPU_NUM > 8, the default thread pool size for coprocessors is set to CPU_NUM * 0.8. + +# high-concurrency = 8 +# normal-concurrency = 8 +# low-concurrency = 8 +# max-tasks-per-worker-high = 2000 +# max-tasks-per-worker-normal = 2000 +# max-tasks-per-worker-low = 2000 +# stack-size = "10MB" + +[server] +## Listening address. +# addr = "127.0.0.1:20160" + +## Advertise listening address for client communication. +## If not set, `addr` will be used. +# advertise-addr = "" + +## Status address. +## This is used for reporting the status of TiKV directly through the HTTP address. +## Empty string means disabling it. +# status-addr = "127.0.0.1:20180" + +## Set the maximum number of worker threads for the status report HTTP service. +# status-thread-pool-size = 1 + +## Compression type for gRPC channel: none, deflate or gzip. +# grpc-compression-type = "none" + +## Size of the thread pool for the gRPC server. +# grpc-concurrency = 4 + +## The number of max concurrent streams/requests on a client connection. +# grpc-concurrent-stream = 1024 + +## The number of connections with each TiKV server to send Raft messages. +# grpc-raft-conn-num = 10 + +## Amount to read ahead on individual gRPC streams. +# grpc-stream-initial-window-size = "2MB" + +## Time to wait before sending out a ping to check if server is still alive. +## This is only for communications between TiKV instances. +# grpc-keepalive-time = "10s" + +## Time to wait before closing the connection without receiving KeepAlive ping Ack. +# grpc-keepalive-timeout = "3s" + +## How many snapshots can be sent concurrently. +# concurrent-send-snap-limit = 32 + +## How many snapshots can be received concurrently. +# concurrent-recv-snap-limit = 32 + +## Max allowed recursion level when decoding Coprocessor DAG expression. +# end-point-recursion-limit = 1000 + +## Max time to handle Coprocessor requests before timeout. +# end-point-request-max-handle-duration = "60s" + +## Max bytes that snapshot can be written to disk in one second. +## It should be set based on your disk performance. +# snap-max-write-bytes-per-sec = "100MB" + +## Attributes about this server, e.g. `{ zone = "us-west-1", disk = "ssd" }`. +# labels = {} + +[storage] +## The path to RocksDB directory. +# data-dir = "/tmp/tikv/store" + +## Internal notify capacity of Scheduler's channel. +# scheduler-notify-capacity = 10240 + +## The number of slots in Scheduler latches, which controls write concurrency. +## In most cases you can use the default value. When importing data, you can set it to a larger +## value. +# scheduler-concurrency = 2048000 + +## Scheduler's worker pool size, i.e. the number of write threads. +## It should be less than total CPU cores. When there are frequent write operations, set it to a +## higher value. More specifically, you can run `top -H -p tikv-pid` to check whether the threads +## named `sched-worker-pool` are busy. +# scheduler-worker-pool-size = 4 + +## When the pending write bytes exceeds this threshold, the "scheduler too busy" error is displayed. +# scheduler-pending-write-threshold = "100MB" + +[pd] +## PD endpoints. +# endpoints = [] + +[raftstore] +## Whether to force to flush logs. +## Set to `true` (default) for best reliability, which prevents data loss when there is a power +## failure. Set to `false` for higher performance (ensure that you run multiple TiKV nodes!). +# sync-log = true + +## Whether to enable Raft prevote. +## Prevote minimizes disruption when a partitioned node rejoins the cluster by using a two phase +## election. +# prevote = true + +## The path to RaftDB directory. +## If not set, it will be `{data-dir}/raft`. +## If there are multiple disks on the machine, storing the data of Raft RocksDB on differen disks +## can improve TiKV performance. +# raftdb-path = "" + +## Store capacity, i.e. max data size allowed. +## If it is not set, disk capacity is used. +# capacity = 0 + +## Internal notify capacity. +## 40960 is suitable for about 7000 Regions. It is recommended to use the default value. +# notify-capacity = 40960 + +## Maximum number of internal messages to process in a tick. +# messages-per-tick = 4096 + +## Region heartbeat tick interval for reporting to PD. +# pd-heartbeat-tick-interval = "60s" + +## Store heartbeat tick interval for reporting to PD. +# pd-store-heartbeat-tick-interval = "10s" + +## The threshold of triggering Region split check. +## When Region size change exceeds this config, TiKV will check whether the Region should be split +## or not. To reduce the cost of scanning data in the checking process, you can set the value to +## 32MB during checking and set it back to the default value in normal operations. +# region-split-check-diff = "6MB" + +## The interval of triggering Region split check. +# split-region-check-tick-interval = "10s" + +## When the number of Raft entries exceeds the max size, TiKV rejects to propose the entry. +# raft-entry-max-size = "8MB" + +## Interval to GC unnecessary Raft log. +# raft-log-gc-tick-interval = "10s" + +## Threshold to GC stale Raft log, must be >= 1. +# raft-log-gc-threshold = 50 + +## When the entry count exceeds this value, GC will be forced to trigger. +# raft-log-gc-count-limit = 72000 + +## When the approximate size of Raft log entries exceeds this value, GC will be forced trigger. +## It's recommanded to set it to 3/4 of `region-split-size`. +# raft-log-gc-size-limit = "72MB" + +## How long the peer will be considered down and reported to PD when it hasn't been active for this +## time. +# max-peer-down-duration = "5m" + +## Interval to check whether to start manual compaction for a Region. +# region-compact-check-interval = "5m" + +## Number of Regions for each time to check. +# region-compact-check-step = 100 + +## The minimum number of delete tombstones to trigger manual compaction. +# region-compact-min-tombstones = 10000 + +## The minimum percentage of delete tombstones to trigger manual compaction. +## It should be set between 1 and 100. Manual compaction is only triggered when the number of +## delete tombstones exceeds `region-compact-min-tombstones` and the percentage of delete tombstones +## exceeds `region-compact-tombstones-percent`. +# region-compact-tombstones-percent = 30 + +## Interval to check whether to start a manual compaction for Lock Column Family. +## If written bytes reach `lock-cf-compact-bytes-threshold` for Lock Column Family, TiKV will +## trigger a manual compaction for Lock Column Family. +# lock-cf-compact-interval = "10m" +# lock-cf-compact-bytes-threshold = "256MB" + +## Interval (s) to check Region whether the data are consistent. +# consistency-check-interval = 0 + +## Delay time before deleting a stale peer. +# clean-stale-peer-delay = "10m" + +## Interval to clean up import SST files. +# cleanup-import-sst-interval = "10m" + +[coprocessor] +## When it is set to `true`, TiKV will try to split a Region with table prefix if that Region +## crosses tables. +## It is recommended to turn off this option if there will be a large number of tables created. +# split-region-on-table = true + +## One split check produces several split keys in batch. This config limits the number of produced +## split keys in one batch. +# batch-split-limit = 10 + +## When Region [a,e) size exceeds `region_max_size`, it will be split into several Regions [a,b), +## [b,c), [c,d), [d,e) and the size of [a,b), [b,c), [c,d) will be `region_split_size` (or a +## little larger). +# region-max-size = "144MB" +# region-split-size = "96MB" + +## When the number of keys in Region [a,e) exceeds the `region_max_keys`, it will be split into +## several Regions [a,b), [b,c), [c,d), [d,e) and the number of keys in [a,b), [b,c), [c,d) will be +## `region_split_keys`. +# region-max-keys = 1440000 +# region-split-keys = 960000 + +[rocksdb] +## Maximum number of threads of RocksDB background jobs. +## The background tasks include compaction and flush. For detailed information why RocksDB needs to +## do compaction, see RocksDB-related materials. When write traffic (like the importing data size) +## is big, it is recommended to enable more threads. But set the number of the enabled threads +## smaller than that of CPU cores. For example, when importing data, for a machine with a 32-core +## CPU, set the value to 28. +# max-background-jobs = 8 + +## Represents the maximum number of threads that will concurrently perform a sub-compaction job by +## breaking it into multiple, smaller ones running simultaneously. +# max-sub-compactions = 1 + +## Number of open files that can be used by the DB. +## Value -1 means files opened are always kept open and RocksDB will prefetch index and filter +## blocks into block cache at startup. So if your database has a large working set, it will take +## several minutes to open the DB. You may need to increase this if your database has a large +## working set. You can estimate the number of files based on `target-file-size-base` and +## `target_file_size_multiplier` for level-based compaction. +max-open-files = 1024 + +## Max size of RocksDB's MANIFEST file. +## For detailed explanation, please refer to https://github.com/facebook/rocksdb/wiki/MANIFEST +# max-manifest-file-size = "20MB" + +## If the value is `true`, the database will be created if it is missing. +# create-if-missing = true + +## RocksDB Write-Ahead Logs (WAL) recovery mode. +## 0 : TolerateCorruptedTailRecords, tolerate incomplete record in trailing data on all logs; +## 1 : AbsoluteConsistency, We don't expect to find any corruption in the WAL; +## 2 : PointInTimeRecovery, Recover to point-in-time consistency; +## 3 : SkipAnyCorruptedRecords, Recovery after a disaster; +# wal-recovery-mode = 2 + +## RocksDB WAL directory. +## This config specifies the absolute directory path for WAL. +## If it is not set, the log files will be in the same directory as data. When you set the path to +## RocksDB directory in memory like in `/dev/shm`, you may want to set`wal-dir` to a directory on a +## persistent storage. See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database . +## If there are two disks on the machine, storing RocksDB data and WAL logs on different disks can +## improve performance. +# wal-dir = "/tmp/tikv/store" + +## The following two fields affect how archived WAL will be deleted. +## 1. If both values are set to 0, logs will be deleted ASAP and will not get into the archive. +## 2. If `wal-ttl-seconds` is 0 and `wal-size-limit` is not 0, WAL files will be checked every 10 +## min and if total size is greater than `wal-size-limit`, they will be deleted starting with the +## earliest until `wal-size-limit` is met. All empty files will be deleted. +## 3. If `wal-ttl-seconds` is not 0 and `wal-size-limit` is 0, then WAL files will be checked every +## `wal-ttl-seconds / 2` and those that are older than `wal-ttl-seconds` will be deleted. +## 4. If both are not 0, WAL files will be checked every 10 min and both checks will be performed +## with ttl being first. +## When you set the path to RocksDB directory in memory like in `/dev/shm`, you may want to set +## `wal-ttl-seconds` to a value greater than 0 (like 86400) and backup your DB on a regular basis. +## See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database . +# wal-ttl-seconds = 0 +# wal-size-limit = 0 + +## Max RocksDB WAL size in total +# max-total-wal-size = "4GB" + +## RocksDB Statistics provides cumulative stats over time. +## Turning statistics on will introduce about 5%-10% overhead for RocksDB, but it can help you to +## know the internal status of RocksDB. +# enable-statistics = true + +## Dump statistics periodically in information logs. +## Same as RocksDB's default value (10 min). +# stats-dump-period = "10m" + +## Refer to: https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ +## If you want to use RocksDB on multi disks or spinning disks, you should set value at least 2MB. +# compaction-readahead-size = 0 + +## Max buffer size that is used by WritableFileWrite. +# writable-file-max-buffer-size = "1MB" + +## Use O_DIRECT for both reads and writes in background flush and compactions. +# use-direct-io-for-flush-and-compaction = false + +## Limit the disk IO of compaction and flush. +## Compaction and flush can cause terrible spikes if they exceed a certain threshold. Consider +## setting this to 50% ~ 80% of the disk throughput for a more stable result. However, in heavy +## write workload, limiting compaction and flush speed can cause write stalls too. +# rate-bytes-per-sec = 0 + +## Enable or disable the pipelined write. +# enable-pipelined-write = true + +## Allows OS to incrementally sync files to disk while they are being written, asynchronously, +## in the background. +# bytes-per-sync = "1MB" + +## Allows OS to incrementally sync WAL to disk while it is being written. +# wal-bytes-per-sync = "512KB" + +## Specify the maximal size of the RocksDB info log file. +## If the log file is larger than this config, a new info log file will be created. +## If it is set to 0, all logs will be written to one log file. +# info-log-max-size = "1GB" + +## Time for the RocksDB info log file to roll (in seconds). +## If the log file has been active longer than this config, it will be rolled. +## If it is set to 0, rolling will be disabled. +# info-log-roll-time = "0" + +## Maximal RocksDB info log files to be kept. +# info-log-keep-log-file-num = 10 + +## Specifies the RocksDB info log directory. +## If it is empty, the log files will be in the same directory as data. +## If it is not empty, the log files will be in the specified directory, and the DB data directory's +## absolute path will be used as the log file name's prefix. +# info-log-dir = "" + +## Options for `Titan`. +[rocksdb.titan] +## Enables `Titan` +## default: false +# enabled = false + +## Specifies `Titan` blob files directory +## default: "titandb" (if not specific or empty) +# dirname = "" + +## Disable blob file gc +# default: false +# disable-gc = false + +## Maximum number of threads of `Titan` background gc jobs. +# default: 1 +# max-background-gc = 1 + +## Options for "Default" Column Family, which stores actual user data. +[rocksdb.defaultcf] +## Compression method (if any) is used to compress a block. +## no: kNoCompression +## snappy: kSnappyCompression +## zlib: kZlibCompression +## bzip2: kBZip2Compression +## lz4: kLZ4Compression +## lz4hc: kLZ4HCCompression +## zstd: kZSTD +## `lz4` is a compression algorithm with moderate speed and compression ratio. The compression +## ratio of `zlib` is high. It is friendly to the storage space, but its compression speed is +## slow. This compression occupies many CPU resources. + +## Per level compression. +## This config should be chosen carefully according to CPU and I/O resources. For example, if you +## use the compression mode of "no:no:lz4:lz4:lz4:zstd:zstd" and find much I/O pressure of the +## system (run the `iostat` command to find %util lasts 100%, or run the `top` command to find many +## iowaits) when writing (importing) a lot of data while the CPU resources are adequate, you can +## compress level-0 and level-1 and exchange CPU resources for I/O resources. If you use the +## compression mode of "no:no:lz4:lz4:lz4:zstd:zstd" and you find the I/O pressure of the system is +## not big when writing a lot of data, but CPU resources are inadequate. Then run the `top` command +## and choose the `-H` option. If you find a lot of bg threads (namely the compression thread of +## RocksDB) are running, you can exchange I/O resources for CPU resources and change the compression +## mode to "no:no:no:lz4:lz4:zstd:zstd". In a word, it aims at making full use of the existing +## resources of the system and improving TiKV performance in terms of the current resources. +# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] + +## The data block size. RocksDB compresses data based on the unit of block. +## Similar to page in other databases, block is the smallest unit cached in block-cache. Note that +## the block size specified here corresponds to uncompressed data. +# block-size = "64KB" + +## If you're doing point lookups you definitely want to turn bloom filters on. We use bloom filters +## to avoid unnecessary disk reads. Default bits_per_key is 10, which yields ~1% false positive +## rate. Larger `bloom-filter-bits-per-key` values will reduce false positive rate, but increase +## memory usage and space amplification. +# bloom-filter-bits-per-key = 10 + +## `false` means one SST file one bloom filter, `true` means every block has a corresponding bloom +## filter. +# block-based-bloom-filter = false + +# level0-file-num-compaction-trigger = 4 + +## Soft limit on number of level-0 files. +## When the number of SST files of level-0 reaches the limit of `level0-slowdown-writes-trigger`, +## RocksDB tries to slow down the write operation, because too many SST files of level-0 can cause +## higher read pressure of RocksDB. +# level0-slowdown-writes-trigger = 20 + +## Maximum number of level-0 files. +## When the number of SST files of level-0 reaches the limit of `level0-stop-writes-trigger`, +## RocksDB stalls the new write operation. +# level0-stop-writes-trigger = 36 + +## Amount of data to build up in memory (backed by an unsorted log on disk) before converting to a +## sorted on-disk file. It is the RocksDB MemTable size. +# write-buffer-size = "128MB" + +## The maximum number of the MemTables. The data written into RocksDB is first recorded in the WAL +## log, and then inserted into MemTables. When the MemTable reaches the size limit of +## `write-buffer-size`, it turns into read only and generates a new MemTable receiving new write +## operations. The flush threads of RocksDB will flush the read only MemTable to the disks to become +## an SST file of level0. `max-background-flushes` controls the maximum number of flush threads. +## When the flush threads are busy, resulting in the number of the MemTables waiting to be flushed +## to the disks reaching the limit of `max-write-buffer-number`, RocksDB stalls the new operation. +## "Stall" is a flow control mechanism of RocksDB. When importing data, you can set the +## `max-write-buffer-number` value higher, like 10. +# max-write-buffer-number = 5 + +## The minimum number of write buffers that will be merged together before writing to storage. +# min-write-buffer-number-to-merge = 1 + +## Control maximum total data size for base level (level 1). +## When the level-1 data size reaches the limit value of `max-bytes-for-level-base`, the SST files +## of level-1 and their overlap SST files of level-2 will be compacted. The golden rule: the first +## reference principle of setting `max-bytes-for-level-base` is guaranteeing that the +## `max-bytes-for-level-base` value is roughly equal to the data volume of level-0. Thus +## unnecessary compaction is reduced. For example, if the compression mode is +## "no:no:lz4:lz4:lz4:lz4:lz4", the `max-bytes-for-level-base` value can be `write-buffer-size * 4`, +## because there is no compression of level-0 and level-1 and the trigger condition of compaction +## for level-0 is that the number of the SST files reaches 4 (the default value). When both level-0 +## and level-1 adopt compaction, it is necessary to analyze RocksDB logs to know the size of an SST +## file compressed from a MemTable. For example, if the file size is 32MB, the proposed value of +## `max-bytes-for-level-base` is 32MB * 4 = 128MB. +# max-bytes-for-level-base = "512MB" + +## Target file size for compaction. +## The SST file size of level-0 is influenced by the compaction algorithm of `write-buffer-size` +## and level0. `target-file-size-base` is used to control the size of a single SST file of level1 to +## level6. +# target-file-size-base = "8MB" + +## Max bytes for `compaction.max_compaction_bytes`. +# max-compaction-bytes = "2GB" + +## There are four different compaction priorities. +## 0 : ByCompensatedSize +## 1 : OldestLargestSeqFirst +## 2 : OldestSmallestSeqFirst +## 3 : MinOverlappingRatio +# compaction-pri = 3 + +## Block cache used to cache uncompressed blocks. +## Big block-cache can speed up read. Normally it should be tuned to 30%-50% system's total memory. +## When the config is not set, TiKV sets the value to 40% of the system memory size. +## To deploy multiple TiKV nodes on one physical machine, configure this parameter explicitly. +## Otherwise, the OOM problem might occur in TiKV. +# block-cache-size = "1GB" + +## Indicating if we'd put index/filter blocks to the block cache. +## If not specified, each "table reader" object will pre-load index/filter block during table +## initialization. +# cache-index-and-filter-blocks = true + +## Pin level-0 filter and index blocks in cache. +# pin-l0-filter-and-index-blocks = true + +## Enable read amplification statistics. +## value => memory usage (percentage of loaded blocks memory) +## 1 => 12.50 % +## 2 => 06.25 % +## 4 => 03.12 % +## 8 => 01.56 % +## 16 => 00.78 % +# read-amp-bytes-per-bit = 0 + +## Pick target size of each level dynamically. +# dynamic-level-bytes = true + +## Optimizes bloom filters. If true, RocksDB won't create bloom filters for the max level of +## the LSM to reduce metadata that should fit in RAM. +## This value is setted to true for `default` cf by default because its kv data could be determined +## whether really exists by upper logic instead of bloom filters. But we suggest to set it to false +## while using `Raw` mode. +# optimize-filters-for-hits = true + +## Options for "Default" Column Family for `Titan`. +[rocksdb.defaultcf.titan] +## The smallest value to store in blob files. Value smaller than +## this threshold will be inlined in base DB. +## default: 1KB +# min-blob-size = "1KB" + +## The compression algorithm used to compress data in blob files. +## Compression method. +## no: kNoCompression +## snappy: kSnappyCompression +## zlib: kZlibCompression +## bzip2: kBZip2Compression +## lz4: kLZ4Compression +## lz4hc: kLZ4HCCompression +## zstd: kZSTD +# default: lz4 +# blob-file-compression = "lz4" + +## Specifics cache size for blob records +# default: 0 +# blob-cache-size = "0GB" + +## The minimum batch size of one gc job. The total blob file size +## of one gc job cannot smaller than this threshold. +## default: 16MB +# min-gc-batch-size = "16MB" + +## The maximum batch size of one gc job. The total blob file size +## of one gc job cannot exceed this threshold. +# max-gc-batch-size = "64MB" + +## If the ratio of discardable size of a blob file is larger than +## this threshold, the blob file will be GCed out. +# default: 0.5 +# discardable-ratio = 0.5 + +## The gc job will sample the target blob files to see if its +## discardable ratio is smaller than discardable-ratio metioned +## above before gc start, if so the blob file will be exclude. +# sample-ratio = 0.1 + +## If the size of the blob file is smaller than this threshold, +## the blob file will be merge. +# default: 8MB +# merge-small-file-threshold = "8MB" + +## Options for "Write" Column Family, which stores MVCC commit information +[rocksdb.writecf] +## Recommend to set it the same as `rocksdb.defaultcf.compression-per-level`. +# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] +# block-size = "64KB" + +## Recommend to set it the same as `rocksdb.defaultcf.write-buffer-size`. +# write-buffer-size = "128MB" +# max-write-buffer-number = 5 +# min-write-buffer-number-to-merge = 1 + +## Recommend to set it the same as `rocksdb.defaultcf.max-bytes-for-level-base`. +# max-bytes-for-level-base = "512MB" +# target-file-size-base = "8MB" + +## In normal cases, this config should be tuned to 10%-30% of the system's total memory. +## When this config is not set, TiKV sets it to 15% of the system memory size. +## To deploy multiple TiKV nodes on a single physical machine, configure this parameter explicitly. +## The related data of the version information (MVCC) and the index-related data are recorded in +## Write CF. In scenarios that include many single table indexes, set this parameter value higher. +# block-cache-size = "256MB" + +# level0-file-num-compaction-trigger = 4 +# level0-slowdown-writes-trigger = 20 +# level0-stop-writes-trigger = 36 +# cache-index-and-filter-blocks = true +# pin-l0-filter-and-index-blocks = true +# compaction-pri = 3 +# read-amp-bytes-per-bit = 0 +# dynamic-level-bytes = true +# optimize-filters-for-hits = false + +[rocksdb.lockcf] +# compression-per-level = ["no", "no", "no", "no", "no", "no", "no"] +# block-size = "16KB" +# write-buffer-size = "128MB" +# max-write-buffer-number = 5 +# min-write-buffer-number-to-merge = 1 +# max-bytes-for-level-base = "128MB" +# target-file-size-base = "8MB" +# block-cache-size = "256MB" +# level0-file-num-compaction-trigger = 1 +# level0-slowdown-writes-trigger = 20 +# level0-stop-writes-trigger = 36 +# cache-index-and-filter-blocks = true +# pin-l0-filter-and-index-blocks = true +# compaction-pri = 0 +# read-amp-bytes-per-bit = 0 +# dynamic-level-bytes = true +# optimize-filters-for-hits = false + +[raftdb] +# max-background-jobs = 2 +# max-sub-compactions = 1 +max-open-files = 1024 +# max-manifest-file-size = "20MB" +# create-if-missing = true + +# enable-statistics = true +# stats-dump-period = "10m" + +# compaction-readahead-size = 0 +# writable-file-max-buffer-size = "1MB" +# use-direct-io-for-flush-and-compaction = false +# enable-pipelined-write = true +# allow-concurrent-memtable-write = false +# bytes-per-sync = "1MB" +# wal-bytes-per-sync = "512KB" + +# info-log-max-size = "1GB" +# info-log-roll-time = "0" +# info-log-keep-log-file-num = 10 +# info-log-dir = "" +# optimize-filters-for-hits = true + +[raftdb.defaultcf] +## Recommend to set it the same as `rocksdb.defaultcf.compression-per-level`. +# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] +# block-size = "64KB" + +## Recommend to set it the same as `rocksdb.defaultcf.write-buffer-size`. +# write-buffer-size = "128MB" +# max-write-buffer-number = 5 +# min-write-buffer-number-to-merge = 1 + +## Recommend to set it the same as `rocksdb.defaultcf.max-bytes-for-level-base`. +# max-bytes-for-level-base = "512MB" +# target-file-size-base = "8MB" + +## Generally, you can set it from 256MB to 2GB. In most cases, you can use the default value. But +## if the system resources are adequate, you can set it higher. +# block-cache-size = "256MB" + +# level0-file-num-compaction-trigger = 4 +# level0-slowdown-writes-trigger = 20 +# level0-stop-writes-trigger = 36 +# cache-index-and-filter-blocks = true +# pin-l0-filter-and-index-blocks = true +# compaction-pri = 0 +# read-amp-bytes-per-bit = 0 +# dynamic-level-bytes = true +# optimize-filters-for-hits = true + +[security] +## The path for TLS certificates. Empty string means disabling secure connections. +# ca-path = "" +# cert-path = "" +# key-path = "" + +[import] +## Number of threads to handle RPC requests. +# num-threads = 8 + +## Stream channel window size, stream will be blocked on channel full. +# stream-channel-window = 128 diff --git a/go-ycsb/tool/docker/mariadb.yml b/go-ycsb/tool/docker/mariadb.yml new file mode 100644 index 000000000..d8f0dada4 --- /dev/null +++ b/go-ycsb/tool/docker/mariadb.yml @@ -0,0 +1,17 @@ +version: '2.1' + +services: + mariadb: + image: mariadb + ports: + - "3308:3306" + volumes: + - ./data/mariadb:/var/lib/mysql + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: "test" + restart: on-failure + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/mysql.yml b/go-ycsb/tool/docker/mysql.yml new file mode 100644 index 000000000..bd0a2012e --- /dev/null +++ b/go-ycsb/tool/docker/mysql.yml @@ -0,0 +1,17 @@ +version: '2.1' + +services: + mysql: + image: mysql:5.7 + ports: + - "3306:3306" + volumes: + - ./data/mysql:/var/lib/mysql + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: "test" + restart: on-failure + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/mysql8.yml b/go-ycsb/tool/docker/mysql8.yml new file mode 100644 index 000000000..5116614f5 --- /dev/null +++ b/go-ycsb/tool/docker/mysql8.yml @@ -0,0 +1,17 @@ +version: '2.1' + +services: + mysql: + image: mysql:8.0 + ports: + - "3307:3306" + volumes: + - ./data/mysql8:/var/lib/mysql + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: "test" + restart: on-failure + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/pg.yml b/go-ycsb/tool/docker/pg.yml new file mode 100644 index 000000000..b2691b1bb --- /dev/null +++ b/go-ycsb/tool/docker/pg.yml @@ -0,0 +1,18 @@ +version: '2.1' + +services: + pg: + image: postgres:latest + ports: + - "5432:5432" + volumes: + - ./data/pg:/var/lib/postgresql/data + environment: + POSTGRES_USER: "root" + POSTGRES_PASSWORD: "" + POSTGRES_DB: "test" + restart: on-failure + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/raw.yml b/go-ycsb/tool/docker/raw.yml new file mode 100644 index 000000000..f97af7b02 --- /dev/null +++ b/go-ycsb/tool/docker/raw.yml @@ -0,0 +1,44 @@ +version: '2.1' + +services: + pd: + image: pingcap/pd:latest + ports: + - "2379" + volumes: + - ./config/pd.toml:/pd.toml:ro + - ./data/txn:/data + - ./logs/txn:/logs + command: + - --name=pd + - --client-urls=http://0.0.0.0:2379 + - --peer-urls=http://0.0.0.0:2380 + - --advertise-client-urls=http://pd:2379 + - --advertise-peer-urls=http://pd:2380 + - --initial-cluster=pd=http://pd:2380 + - --data-dir=/data/pd + - --config=/pd.toml + - --log-file=/logs/pd.log + restart: on-failure + + tikv: + image: pingcap/tikv:latest + volumes: + - ./config/tikv.toml:/tikv.toml:ro + - ./data/txn:/data + - ./logs/txn:/logs + command: + - --addr=0.0.0.0:20160 + - --advertise-addr=tikv:20160 + - --data-dir=/data/tikv + - --pd=pd:2379 + - --config=/tikv.toml + - --log-file=/logs/tikv.log + depends_on: + - "pd" + restart: on-failure + + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/run_bench.sh b/go-ycsb/tool/docker/run_bench.sh new file mode 100755 index 000000000..05f879858 --- /dev/null +++ b/go-ycsb/tool/docker/run_bench.sh @@ -0,0 +1,10 @@ +rm -rf ./data +rm -rf ./logs + +for db in pg cockroach mysql mysql8 mariadb tidb tikv raw cassandra scylla +do + ./bench.sh load ${db} + ./bench.sh run ${db} +done + +./clear.sh diff --git a/go-ycsb/tool/docker/scylla.yml b/go-ycsb/tool/docker/scylla.yml new file mode 100644 index 000000000..6aa08b843 --- /dev/null +++ b/go-ycsb/tool/docker/scylla.yml @@ -0,0 +1,24 @@ +version: '2.1' + +services: + scylla: + image: scylladb/scylla:latest + volumes: + - ./data/scylla:/var/lib/scylla + ports: + - "9042:9042" + restart: on-failure + + db-init: + image: scylladb/scylla:latest + volumes: + - ./config/init_cassandra.sh:/init_cassandra.sh + entrypoint: + - "/bin/bash" + command: /init_cassandra.sh scylla + depends_on: + - "scylla" + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/sqlite.yml b/go-ycsb/tool/docker/sqlite.yml new file mode 100644 index 000000000..f6e32c11c --- /dev/null +++ b/go-ycsb/tool/docker/sqlite.yml @@ -0,0 +1,8 @@ +version: '2.1' + +services: + ycsb: + image: pingcap/go-ycsb + volumes: + - ./data/sqlite:/data + diff --git a/go-ycsb/tool/docker/tidb.yml b/go-ycsb/tool/docker/tidb.yml new file mode 100644 index 000000000..8f1b08bbf --- /dev/null +++ b/go-ycsb/tool/docker/tidb.yml @@ -0,0 +1,60 @@ +version: '2.1' + +services: + pd: + image: pingcap/pd:latest + ports: + - "2379" + volumes: + - ./config/pd.toml:/pd.toml:ro + - ./data/tidb:/data + - ./logs/tidb:/logs + command: + - --name=pd + - --client-urls=http://0.0.0.0:2379 + - --peer-urls=http://0.0.0.0:2380 + - --advertise-client-urls=http://pd:2379 + - --advertise-peer-urls=http://pd:2380 + - --initial-cluster=pd=http://pd:2380 + - --data-dir=/data/pd + - --config=/pd.toml + - --log-file=/logs/pd.log + restart: on-failure + + tikv: + image: pingcap/tikv:latest + volumes: + - ./config/tikv.toml:/tikv.toml:ro + - ./data/tidb:/data + - ./logs/tidb:/logs + command: + - --addr=0.0.0.0:20160 + - --advertise-addr=tikv:20160 + - --data-dir=/data/tikv + - --pd=pd:2379 + - --config=/tikv.toml + - --log-file=/logs/tikv.log + depends_on: + - "pd" + restart: on-failure + + tidb: + image: pingcap/tidb:latest + ports: + - "4000:4000" + - "10080:10080" + volumes: + - ./config/tidb.toml:/tidb.toml:ro + - ./logs/tidb:/logs + command: + - --store=tikv + - --path=pd:2379 + - --config=/tidb.toml + - --log-file=/logs/tidb.log + depends_on: + - "tikv" + restart: on-failure + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/tikv.yml b/go-ycsb/tool/docker/tikv.yml new file mode 100644 index 000000000..1c98b81e0 --- /dev/null +++ b/go-ycsb/tool/docker/tikv.yml @@ -0,0 +1,44 @@ +version: '2.1' + +services: + pd: + image: pingcap/pd:latest + ports: + - "2379" + volumes: + - ./config/pd.toml:/pd.toml:ro + - ./data/tikv:/data + - ./logs/tikv:/logs + command: + - --name=pd + - --client-urls=http://0.0.0.0:2379 + - --peer-urls=http://0.0.0.0:2380 + - --advertise-client-urls=http://pd:2379 + - --advertise-peer-urls=http://pd:2380 + - --initial-cluster=pd=http://pd:2380 + - --data-dir=/data/pd + - --config=/pd.toml + - --log-file=/logs/pd.log + restart: on-failure + + tikv: + image: pingcap/tikv:latest + volumes: + - ./config/tikv.toml:/tikv.toml:ro + - ./data/tikv:/data + - ./logs/tikv:/logs + command: + - --addr=0.0.0.0:20160 + - --advertise-addr=tikv:20160 + - --data-dir=/data/tikv + - --pd=pd:2379 + - --config=/tikv.toml + - --log-file=/logs/tikv.log + depends_on: + - "pd" + restart: on-failure + + + # monitors + ycsb: + image: pingcap/go-ycsb diff --git a/go-ycsb/tool/report.go b/go-ycsb/tool/report.go new file mode 100644 index 000000000..f17499ca8 --- /dev/null +++ b/go-ycsb/tool/report.go @@ -0,0 +1,245 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path" + "sort" + "strconv" + "strings" +) + +var ( + logsPath = "./logs" + operations = map[string]struct{}{ + "INSERT": struct{}{}, + "READ": struct{}{}, + "UPDATE": struct{}{}, + "SCAN": struct{}{}, + "READ_MODIFY_WRITE": struct{}{}, + "DELETE": struct{}{}, + } + workloads = map[string]struct{}{ + "load": struct{}{}, + "workloada": struct{}{}, + "workloadb": struct{}{}, + "workloadc": struct{}{}, + "workloadd": struct{}{}, + "workloade": struct{}{}, + "workloadf": struct{}{}, + } +) + +type stat struct { + OPS float64 + P99 float64 +} + +func statFieldFunc(c rune) bool { + return c == ':' || c == ',' +} + +func newStat(line string) (*stat, error) { + kvs := strings.FieldsFunc(line, statFieldFunc) + s := stat{} + if len(kvs)%2 != 0 { + println(line) + } + for i := 0; i < len(kvs); i += 2 { + v, err := strconv.ParseFloat(strings.TrimSpace(kvs[i+1]), 64) + if err != nil { + return nil, err + } + switch strings.TrimSpace(kvs[i]) { + case "OPS": + s.OPS = v + case "99th(us)": + s.P99 = v + default: + } + } + return &s, nil +} + +type dbStat struct { + db string + workload string + summary map[string]*stat + // progress map[string][]*stat +} + +func parseDBStat(pathName string) (*dbStat, error) { + // check db and workload from file name, the name format is: + // 1. db_load.log + // 2. db_run_workloadx.log + s := new(dbStat) + s.summary = make(map[string]*stat, 1) + // s.progress = make(map[string][]*stat, 1) + + fileName := path.Base(pathName) + seps := strings.Split(fileName, "_") + + if len(seps) != 2 { + return nil, nil + } + + s.db = seps[0] + workload := strings.TrimSuffix(seps[1], ".log") + if _, ok := workloads[workload]; !ok { + return nil, nil + } + + s.workload = workload + file, err := os.Open(pathName) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + handleSummary := false + + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Run finished") { + handleSummary = true + continue + } + + seps := strings.Split(line, "-") + op := strings.TrimSpace(seps[0]) + if _, ok := operations[op]; !ok { + continue + } + + stat, err := newStat(strings.TrimSpace(seps[1])) + if err != nil { + return nil, err + } + + if handleSummary { + // handle summary logs + s.summary[op] = stat + } + + // TODO handle progress logs + // s.progress[op] = append(s.progress[op], stat) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + return s, nil +} + +type dbStats []*dbStat + +func (a dbStats) Len() int { return len(a) } +func (a dbStats) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a dbStats) Less(i, j int) bool { return a[i].db < a[j].db } + +func reportDBStats(logsPath string, workload string, stats dbStats) error { + dir := path.Join(logsPath, "report") + os.MkdirAll(dir, 0755) + + fileName := path.Join(dir, fmt.Sprintf("%s_summary.csv", workload)) + sort.Sort(stats) + + file, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return err + } + defer file.Close() + + var fields []string + switch workload { + case "load": + fields = []string{"INSERT"} + case "workloada": + fields = []string{"READ", "UPDATE"} + case "workloadb": + fields = []string{"READ", "UPDATE"} + case "workloadc": + fields = []string{"READ"} + case "workloadd": + fields = []string{"READ", "INSERT"} + case "workloade": + fields = []string{"SCAN", "INSERT"} + case "workloadf": + fields = []string{"READ_MODIFY_WRITE"} + default: + } + + fmt.Fprintf(file, "DB") + for _, field := range fields { + fmt.Fprintf(file, ",%s OPS,%s P99(us)", field, field) + } + fmt.Fprint(file, "\n") + + for _, stat := range stats { + fmt.Fprintf(file, "%s", stat.db) + for _, field := range fields { + s := stat.summary[field] + if s == nil { + fmt.Fprintf(file, ",0.0") + } else { + fmt.Fprintf(file, ",%.f,%.f", s.OPS, s.P99) + } + } + fmt.Fprintf(file, "\n") + } + + return nil +} + +func main() { + if len(os.Args) >= 2 { + logsPath = os.Args[1] + } + + files, err := ioutil.ReadDir(logsPath) + if err != nil { + println(err.Error()) + return + } + + stats := make(map[string][]*dbStat) + for _, file := range files { + if file.IsDir() { + continue + } + + s, err := parseDBStat(path.Join(logsPath, file.Name())) + + if err != nil { + fmt.Printf("parse %s failed %v\n", file.Name(), err.Error()) + } + + if s == nil { + continue + } + + stats[s.workload] = append(stats[s.workload], s) + } + + for workload, s := range stats { + if err := reportDBStats(logsPath, workload, s); err != nil { + fmt.Printf("report %s failed %v\n", workload, err) + } + } +} diff --git a/go-ycsb/workloads/minio b/go-ycsb/workloads/minio new file mode 100644 index 000000000..c4017fbec --- /dev/null +++ b/go-ycsb/workloads/minio @@ -0,0 +1,4 @@ +minio.access-key=minio +minio.secret-key=myminio +minio.endpoint=127.0.0.1:9000 +minio.secure=false diff --git a/go-ycsb/workloads/workload_template b/go-ycsb/workloads/workload_template new file mode 100644 index 000000000..51aa1f2b0 --- /dev/null +++ b/go-ycsb/workloads/workload_template @@ -0,0 +1,206 @@ +# Copyright (c) 2012-2016 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Yahoo! Cloud System Benchmark +# Workload Template: Default Values +# +# File contains all properties that can be set to define a +# YCSB session. All properties are set to their default +# value if one exists. If not, the property is commented +# out. When a property has a finite number of settings, +# the default is enabled and the alternates are shown in +# comments below it. +# +# Use of most explained through comments in Client.java or +# CoreWorkload.java or on the YCSB wiki page: +# https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties + +# The name of the workload class to use +workload=core + +# There is no default setting for recordcount but it is +# required to be set. +# The number of records in the table to be inserted in +# the load phase or the number of records already in the +# table before the run phase. +recordcount=1000000 + +# There is no default setting for operationcount but it is +# required to be set. +# The number of operations to use during the run phase. +operationcount=3000000 + +# The number of thread. +threadcount=500 + +# The number of insertions to do, if different from recordcount. +# Used with insertstart to grow an existing table. +#insertcount= + +# The offset of the first insertion +insertstart=0 + +# The number of fields in a record +fieldcount=10 + +# The size of each field (in bytes) +fieldlength=100 + +# Should read all fields +readallfields=true + +# Should write all fields on update +writeallfields=false + +# The distribution used to choose the length of a field +fieldlengthdistribution=constant +#fieldlengthdistribution=uniform +#fieldlengthdistribution=zipfian + +# What proportion of operations are reads +readproportion=0.95 + +# What proportion of operations are updates +updateproportion=0.05 + +# What proportion of operations are inserts +insertproportion=0 + +# What proportion of operations read then modify a record +readmodifywriteproportion=0 + +# What proportion of operations are scans +scanproportion=0 + +# On a single scan, the maximum number of records to access +maxscanlength=1000 + +# The distribution used to choose the number of records to access on a scan +scanlengthdistribution=uniform +#scanlengthdistribution=zipfian + +# Should records be inserted in order or pseudo-randomly +insertorder=hashed +#insertorder=ordered + +# The distribution of requests across the keyspace +requestdistribution=zipfian +#requestdistribution=uniform +#requestdistribution=latest + +# Percentage of data items that constitute the hot set +hotspotdatafraction=0.2 + +# Percentage of operations that access the hot set +hotspotopnfraction=0.8 + +# Maximum execution time in seconds +#maxexecutiontime= + +# The name of the database table to run queries against +table=usertable + +# The column family of fields (required by some databases) +#columnfamily= + +# How the latency measurements are presented +measurementtype=histogram +#measurementtype=timeseries +#measurementtype=raw +# When measurementtype is set to raw, measurements will be output +# as RAW datapoints in the following csv format: +# "operation, timestamp of the measurement, latency in us" +# +# Raw datapoints are collected in-memory while the test is running. Each +# data point consumes about 50 bytes (including java object overhead). +# For a typical run of 1 million to 10 million operations, this should +# fit into memory most of the time. If you plan to do 100s of millions of +# operations per run, consider provisioning a machine with larger RAM when using +# the RAW measurement type, or split the run into multiple runs. +# +# Optionally, you can specify an output file to save raw datapoints. +# Otherwise, raw datapoints will be written to stdout. +# The output file will be appended to if it already exists, otherwise +# a new output file will be created. +#measurement.raw.output_file = /tmp/your_output_file_for_this_run + +# JVM Reporting. +# +# Measure JVM information over time including GC counts, max and min memory +# used, max and min thread counts, max and min system load and others. This +# setting must be enabled in conjunction with the "-s" flag to run the status +# thread. Every "status.interval", the status thread will capture JVM +# statistics and record the results. At the end of the run, max and mins will +# be recorded. +# measurement.trackjvm = false + +# The range of latencies to track in the histogram (milliseconds) +histogram.buckets=1000 + +# Granularity for time series (in milliseconds) +timeseries.granularity=1000 + +# Latency reporting. +# +# YCSB records latency of failed operations separately from successful ones. +# Latency of all OK operations will be reported under their operation name, +# such as [READ], [UPDATE], etc. +# +# For failed operations: +# By default we don't track latency numbers of specific error status. +# We just report latency of all failed operation under one measurement name +# such as [READ-FAILED]. But optionally, user can configure to have either: +# 1. Record and report latency for each and every error status code by +# setting reportLatencyForEachError to true, or +# 2. Record and report latency for a select set of error status codes by +# providing a CSV list of Status codes via the "latencytrackederrors" +# property. +# reportlatencyforeacherror=false +# latencytrackederrors="" + +# Insertion error retry for the core workload. +# +# By default, the YCSB core workload does not retry any operations. +# However, during the load process, if any insertion fails, the entire +# load process is terminated. +# If a user desires to have more robust behavior during this phase, they can +# enable retry for insertion by setting the following property to a positive +# number. +# core_workload_insertion_retry_limit = 0 +# +# the following number controls the interval between retries (in seconds): +# core_workload_insertion_retry_interval = 3 + +# Distributed Tracing via Apache HTrace (http://htrace.incubator.apache.org/) +# +# Defaults to blank / no tracing +# Below sends to a local file, sampling at 0.1% +# +# htrace.sampler.classes=ProbabilitySampler +# htrace.sampler.fraction=0.001 +# htrace.span.receiver.classes=org.apache.htrace.core.LocalFileSpanReceiver +# htrace.local.file.span.receiver.path=/some/path/to/local/file +# +# To capture all spans, use the AlwaysSampler +# +# htrace.sampler.classes=AlwaysSampler +# +# To send spans to an HTraced receiver, use the below and ensure +# your classpath contains the htrace-htraced jar (i.e. when invoking the ycsb +# command add -cp /path/to/htrace-htraced.jar) +# +# htrace.span.receiver.classes=org.apache.htrace.impl.HTracedSpanReceiver +# htrace.htraced.receiver.address=example.com:9075 +# htrace.htraced.error.log.period.ms=10000 diff --git a/go-ycsb/workloads/workloada b/go-ycsb/workloads/workloada new file mode 100644 index 000000000..ce71eb7cc --- /dev/null +++ b/go-ycsb/workloads/workloada @@ -0,0 +1,37 @@ +# Copyright (c) 2010 Yahoo! Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + + +# Yahoo! Cloud System Benchmark +# Workload A: Update heavy workload +# Application example: Session store recording recent actions +# +# Read/update ratio: 50/50 +# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) +# Request distribution: zipfian + +recordcount=1000 +operationcount=1000 +workload=core + +readallfields=true + +readproportion=0.5 +updateproportion=0.5 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform + diff --git a/go-ycsb/workloads/workloadb b/go-ycsb/workloads/workloadb new file mode 100644 index 000000000..9294450b6 --- /dev/null +++ b/go-ycsb/workloads/workloadb @@ -0,0 +1,36 @@ +# Copyright (c) 2010 Yahoo! Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Yahoo! Cloud System Benchmark +# Workload B: Read mostly workload +# Application example: photo tagging; add a tag is an update, but most operations are to read tags +# +# Read/update ratio: 95/5 +# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) +# Request distribution: zipfian + +recordcount=1000 +operationcount=1000 +workload=core + +readallfields=true + +readproportion=0.95 +updateproportion=0.05 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform + diff --git a/go-ycsb/workloads/workloadc b/go-ycsb/workloads/workloadc new file mode 100644 index 000000000..df49bccbc --- /dev/null +++ b/go-ycsb/workloads/workloadc @@ -0,0 +1,38 @@ +# Copyright (c) 2010 Yahoo! Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Yahoo! Cloud System Benchmark +# Workload C: Read only +# Application example: user profile cache, where profiles are constructed elsewhere (e.g., Hadoop) +# +# Read/update ratio: 100/0 +# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) +# Request distribution: zipfian + +recordcount=1000 +operationcount=1000 +workload=core + +readallfields=true + +readproportion=1 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform + + + diff --git a/go-ycsb/workloads/workloadd b/go-ycsb/workloads/workloadd new file mode 100644 index 000000000..bc09b75bd --- /dev/null +++ b/go-ycsb/workloads/workloadd @@ -0,0 +1,41 @@ +# Copyright (c) 2010 Yahoo! Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Yahoo! Cloud System Benchmark +# Workload D: Read latest workload +# Application example: user status updates; people want to read the latest +# +# Read/update/insert ratio: 95/0/5 +# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) +# Request distribution: latest + +# The insert order for this is hashed, not ordered. The "latest" items may be +# scattered around the keyspace if they are keyed by userid.timestamp. A workload +# which orders items purely by time, and demands the latest, is very different than +# workload here (which we believe is more typical of how people build systems.) + +recordcount=1000 +operationcount=1000 +workload=core + +readallfields=true + +readproportion=0.95 +updateproportion=0 +scanproportion=0 +insertproportion=0.05 + +requestdistribution=latest + diff --git a/go-ycsb/workloads/workloade b/go-ycsb/workloads/workloade new file mode 100644 index 000000000..bfae98833 --- /dev/null +++ b/go-ycsb/workloads/workloade @@ -0,0 +1,46 @@ +# Copyright (c) 2010 Yahoo! Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Yahoo! Cloud System Benchmark +# Workload E: Short ranges +# Application example: threaded conversations, where each scan is for the posts in a given thread (assumed to be clustered by thread id) +# +# Scan/insert ratio: 95/5 +# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) +# Request distribution: zipfian + +# The insert order is hashed, not ordered. Although the scans are ordered, it does not necessarily +# follow that the data is inserted in order. For example, posts for thread 342 may not be inserted contiguously, but +# instead interspersed with posts from lots of other threads. The way the YCSB client works is that it will pick a start +# key, and then request a number of records; this works fine even for hashed insertion. + +recordcount=1000 +operationcount=1000 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0.95 +insertproportion=0.05 + +requestdistribution=uniform + +maxscanlength=1 + +scanlengthdistribution=uniform + + diff --git a/go-ycsb/workloads/workloadf b/go-ycsb/workloads/workloadf new file mode 100644 index 000000000..e754d1be0 --- /dev/null +++ b/go-ycsb/workloads/workloadf @@ -0,0 +1,37 @@ +# Copyright (c) 2010 Yahoo! Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. + +# Yahoo! Cloud System Benchmark +# Workload F: Read-modify-write workload +# Application example: user database, where user records are read and modified by the user or to record user activity. +# +# Read/read-modify-write ratio: 50/50 +# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) +# Request distribution: zipfian + +recordcount=1000 +operationcount=1000 +workload=core + +readallfields=true + +readproportion=0.5 +updateproportion=0 +scanproportion=0 +insertproportion=0 +readmodifywriteproportion=0.5 + +requestdistribution=uniform + From 44e0d894ca57b8937fb234549ff6879b318a44d7 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Wed, 21 Dec 2022 20:32:46 +0800 Subject: [PATCH 08/22] modify db_path --- kv/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kv/config/config.go b/kv/config/config.go index 676cdb824..dec968078 100644 --- a/kv/config/config.go +++ b/kv/config/config.go @@ -105,6 +105,6 @@ func NewTestConfig() *Config { SchedulerStoreHeartbeatTickInterval: 500 * time.Millisecond, RegionMaxSize: 144 * MB, RegionSplitSize: 96 * MB, - DBPath: "/tmp/badger", + DBPath: "./dbtest", } } From 7526da3f4597e7e3ee501aa87459e07b4f0524a9 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 14 Mar 2023 15:53:23 +0800 Subject: [PATCH 09/22] clean the implementation of tinykv project --- kv/config/config.go | 2 +- kv/server/raw_api.go | 103 +---------- .../standalone_storage/standalone_storage.go | 133 ++++++--------- .../standalone_storage.go | 42 +++++ .../standalone_storage.go | 160 ------------------ 5 files changed, 103 insertions(+), 337 deletions(-) create mode 100644 kv/storage/standalone_storage_ldb/standalone_storage.go delete mode 100644 kv/storage/standalone_storage_rdb/standalone_storage.go diff --git a/kv/config/config.go b/kv/config/config.go index dec968078..3301ca54d 100644 --- a/kv/config/config.go +++ b/kv/config/config.go @@ -105,6 +105,6 @@ func NewTestConfig() *Config { SchedulerStoreHeartbeatTickInterval: 500 * time.Millisecond, RegionMaxSize: 144 * MB, RegionSplitSize: 96 * MB, - DBPath: "./dbtest", + DBPath: "./dbtest_rdb", } } diff --git a/kv/server/raw_api.go b/kv/server/raw_api.go index 7b4574093..e9a9e6983 100644 --- a/kv/server/raw_api.go +++ b/kv/server/raw_api.go @@ -14,121 +14,26 @@ import ( // RawGet return the corresponding Get response based on RawGetRequest's CF and Key fields func (server *Server) RawGet(_ context.Context, req *kvrpcpb.RawGetRequest) (*kvrpcpb.RawGetResponse, error) { // Your Code Here (1). - resp := &kvrpcpb.RawGetResponse{} - reader, err := server.storage.Reader(req.Context) - if err != nil { - if regionErr, ok := err.(*raft_storage.RegionError); ok { - resp.RegionError = regionErr.RequestErr - return resp, nil - } - return nil, err - } - defer reader.Close() - val, err := reader.GetCF(req.Cf, req.Key) - if err != nil { - if regionErr, ok := err.(*raft_storage.RegionError); ok { - resp.RegionError = regionErr.RequestErr - return resp, nil - } - return nil, err - } - resp.Value = val - if val == nil { - resp.NotFound = true - } - return resp, nil + return nil, nil } // RawPut puts the target data into storage and returns the corresponding response func (server *Server) RawPut(_ context.Context, req *kvrpcpb.RawPutRequest) (*kvrpcpb.RawPutResponse, error) { // Your Code Here (1). // Hint: Consider using Storage.Modify to store data to be modified - resp := &kvrpcpb.RawPutResponse{} - batch := []storage.Modify{ - { - Data: storage.Put{ - Cf: req.Cf, - Key: req.Key, - Value: req.Value, - }, - }} - err := server.storage.Write(req.Context, batch) - if err != nil { - if regionErr, ok := err.(*raft_storage.RegionError); ok { - resp.RegionError = regionErr.RequestErr - return resp, nil - } - return nil, err - } - return resp, nil + return nil, nil } // RawDelete delete the target data from storage and returns the corresponding response func (server *Server) RawDelete(_ context.Context, req *kvrpcpb.RawDeleteRequest) (*kvrpcpb.RawDeleteResponse, error) { // Your Code Here (1). // Hint: Consider using Storage.Modify to store data to be deleted - resp := &kvrpcpb.RawDeleteResponse{} - batch := []storage.Modify{ - { - Data: storage.Delete{ - Cf: req.Cf, - Key: req.Key, - }, - }} - err := server.storage.Write(req.Context, batch) - if err != nil { - if regionErr, ok := err.(*raft_storage.RegionError); ok { - resp.RegionError = regionErr.RequestErr - return resp, nil - } - return nil, err - } - return resp, nil + return nil, nil } // RawScan scan the data starting from the start key up to limit. and return the corresponding result func (server *Server) RawScan(_ context.Context, req *kvrpcpb.RawScanRequest) (*kvrpcpb.RawScanResponse, error) { // Your Code Here (1). // Hint: Consider using reader.IterCF - resp := &kvrpcpb.RawScanResponse{} - if req.Limit == 0 { - return resp, nil - } - reader, err := server.storage.Reader(req.Context) - if err != nil { - if regionErr, ok := err.(*raft_storage.RegionError); ok { - resp.RegionError = regionErr.RequestErr - return resp, nil - } - return nil, err - } - defer reader.Close() - iter := reader.IterCF(req.Cf) - if err != nil { - if regionErr, ok := err.(*raft_storage.RegionError); ok { - resp.RegionError = regionErr.RequestErr - return resp, nil - } - return nil, err - } - defer iter.Close() - var pairs []*kvrpcpb.KvPair - n := req.Limit - for iter.Seek(req.StartKey); iter.Valid(); iter.Next() { - item := iter.Item() - val, err := item.ValueCopy(nil) - if err != nil { - return nil, err - } - pairs = append(pairs, &kvrpcpb.KvPair{ - Key: item.KeyCopy(nil), - Value: val, - }) - n-- - if n == 0 { - break - } - } - resp.Kvs = pairs - return resp, nil + return nil, nil } diff --git a/kv/storage/standalone_storage/standalone_storage.go b/kv/storage/standalone_storage/standalone_storage.go index 5936894b8..0c085f758 100644 --- a/kv/storage/standalone_storage/standalone_storage.go +++ b/kv/storage/standalone_storage/standalone_storage.go @@ -1,9 +1,11 @@ -package standalone_storage_ldb +package standalone_storage import ( - "fmt" + "errors" + "log" + + "github.com/tecbot/gorocksdb" - "github.com/jmhodges/levigo" "github.com/pingcap-incubator/tinykv/kv/config" "github.com/pingcap-incubator/tinykv/kv/storage" "github.com/pingcap-incubator/tinykv/kv/util/engine_util" @@ -14,32 +16,60 @@ import ( // communicate with other nodes and all data is stored locally. type StandAloneStorage struct { // Your Data Here (1). - db *levigo.DB - roptions *levigo.ReadOptions - woptions *levigo.WriteOptions } -func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { - // Your Code Here (1). - dbName := conf.DBPath - opt := levigo.NewOptions() - ropt := levigo.NewReadOptions() - wopt := levigo.NewWriteOptions() - opt.SetCreateIfMissing(true) - opt.SetWriteBufferSize(67108864) - policy := levigo.NewBloomFilter(10) - opt.SetFilterPolicy(policy) - db, err := levigo.Open(dbName, opt) +// opendb +func OpenDB(conf *config.Config) (*gorocksdb.DB, error) { + options := gorocksdb.NewDefaultOptions() + options.SetCreateIfMissing(true) + + bloomFilter := gorocksdb.NewBloomFilter(10) + + readOptions := gorocksdb.NewDefaultReadOptions() + readOptions.SetFillCache(false) + + rateLimiter := gorocksdb.NewRateLimiter(10000000, 10000, 10) + options.SetRateLimiter(rateLimiter) + options.SetCreateIfMissing(true) + options.EnableStatistics() + options.SetWriteBufferSize(8 * 1024) + options.SetMaxWriteBufferNumber(3) + options.SetMaxBackgroundCompactions(10) + // options.SetCompression(gorocksdb.SnappyCompression) + // options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle) + + options.SetHashSkipListRep(2000000, 4, 4) + + blockBasedTableOptions := gorocksdb.NewDefaultBlockBasedTableOptions() + blockBasedTableOptions.SetBlockCache(gorocksdb.NewLRUCache(64 * 1024)) + blockBasedTableOptions.SetFilterPolicy(bloomFilter) + blockBasedTableOptions.SetBlockSizeDeviation(5) + blockBasedTableOptions.SetBlockRestartInterval(10) + blockBasedTableOptions.SetBlockCacheCompressed(gorocksdb.NewLRUCache(64 * 1024)) + blockBasedTableOptions.SetCacheIndexAndFilterBlocks(true) + blockBasedTableOptions.SetIndexType(gorocksdb.KHashSearchIndexType) + + options.SetBlockBasedTableFactory(blockBasedTableOptions) + //log.Println(bloomFilter, readOptions) + options.SetPrefixExtractor(gorocksdb.NewFixedPrefixTransform(3)) + + options.SetAllowConcurrentMemtableWrites(false) + + db, err := gorocksdb.OpenDb(options, conf.DBPath) + if err != nil { - fmt.Printf("open leveldb error!\n") - return nil + log.Fatalln("OPEN DB error", db, err) + db.Close() + return nil, errors.New("fail to open db") + } else { + log.Println("OPEN DB success", db) } - s := &StandAloneStorage{ - db, - ropt, - wopt, - } - return s + return db, nil +} + +func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { + // Your Code Here (1). + return nil } func (s *StandAloneStorage) Start() error { @@ -49,66 +79,15 @@ func (s *StandAloneStorage) Start() error { func (s *StandAloneStorage) Stop() error { // Your Code Here (1). - s.db.Close() return nil } func (s *StandAloneStorage) Reader(ctx *kvrpcpb.Context) (storage.StorageReader, error) { // Your Code Here (1). - return &StandAloneStorageReader{ - s.db, - s.roptions, - }, nil + return nil, nil } func (s *StandAloneStorage) Write(ctx *kvrpcpb.Context, batch []storage.Modify) error { // Your Code Here (1). - for _, m := range batch { - switch m.Data.(type) { - case storage.Put: - put := m.Data.(storage.Put) - if err := s.db.Put(s.woptions, engine_util.KeyWithCF(put.Cf, put.Key), put.Value); err != nil { - return err - } - case storage.Delete: - del := m.Data.(storage.Delete) - if err := s.db.Delete(s.woptions, engine_util.KeyWithCF(del.Cf, del.Key)); err != nil { - return err - } - } - } return nil } - -type StandAloneStorageReader struct { - db *levigo.DB - roptions *levigo.ReadOptions -} - -func (sReader *StandAloneStorageReader) Close() { - return -} - -func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { - val, err := sReader.db.Get(sReader.roptions, engine_util.KeyWithCF(cf, key)) - - return val, err -} - -func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { - return engine_util.NewLDBIterator(cf, sReader.db, sReader.roptions) -} - -// func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { -// value, err := sReader.db.Get(sReader.) - -// v, e := engine_util.GetCF(sReader.db, cf, key) -// if e == badger.ErrKeyNotFound { -// return nil, nil -// } -// return v, e -// } -// func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { -// txn := sReader.db.NewTransaction(false) -// return engine_util.NewCFIterator(cf, txn) -// } diff --git a/kv/storage/standalone_storage_ldb/standalone_storage.go b/kv/storage/standalone_storage_ldb/standalone_storage.go new file mode 100644 index 000000000..09a62b77a --- /dev/null +++ b/kv/storage/standalone_storage_ldb/standalone_storage.go @@ -0,0 +1,42 @@ +package standalone_storage_ldb + +import ( + "fmt" + + "github.com/jmhodges/levigo" + "github.com/pingcap-incubator/tinykv/kv/config" + "github.com/pingcap-incubator/tinykv/kv/storage" + "github.com/pingcap-incubator/tinykv/kv/util/engine_util" + "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" +) + +// StandAloneStorage is an implementation of `Storage` for a single-node TinyKV instance. It does not +// communicate with other nodes and all data is stored locally. +type StandAloneStorage struct { + // Your Data Here (1). +} + +func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { + // Your Code Here (1). + return nil +} + +func (s *StandAloneStorage) Start() error { + // Your Code Here (1). + return nil +} + +func (s *StandAloneStorage) Stop() error { + // Your Code Here (1). + return nil +} + +func (s *StandAloneStorage) Reader(ctx *kvrpcpb.Context) (storage.StorageReader, error) { + // Your Code Here (1). + return nil, nil +} + +func (s *StandAloneStorage) Write(ctx *kvrpcpb.Context, batch []storage.Modify) error { + // Your Code Here (1). + return nil +} diff --git a/kv/storage/standalone_storage_rdb/standalone_storage.go b/kv/storage/standalone_storage_rdb/standalone_storage.go deleted file mode 100644 index 8ab8a42d2..000000000 --- a/kv/storage/standalone_storage_rdb/standalone_storage.go +++ /dev/null @@ -1,160 +0,0 @@ -package standalone_storage - -import ( - "errors" - "log" - - "github.com/tecbot/gorocksdb" - - "github.com/pingcap-incubator/tinykv/kv/config" - "github.com/pingcap-incubator/tinykv/kv/storage" - "github.com/pingcap-incubator/tinykv/kv/util/engine_util" - "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" -) - -// StandAloneStorage is an implementation of `Storage` for a single-node TinyKV instance. It does not -// communicate with other nodes and all data is stored locally. -type StandAloneStorage struct { - // Your Data Here (1). - db *gorocksdb.DB - roptions *gorocksdb.ReadOptions - woptions *gorocksdb.WriteOptions -} - -// opendb -func OpenDB(conf *config.Config) (*gorocksdb.DB, error) { - options := gorocksdb.NewDefaultOptions() - options.SetCreateIfMissing(true) - - bloomFilter := gorocksdb.NewBloomFilter(10) - - readOptions := gorocksdb.NewDefaultReadOptions() - readOptions.SetFillCache(false) - - rateLimiter := gorocksdb.NewRateLimiter(10000000, 10000, 10) - options.SetRateLimiter(rateLimiter) - options.SetCreateIfMissing(true) - options.EnableStatistics() - options.SetWriteBufferSize(8 * 1024) - options.SetMaxWriteBufferNumber(3) - options.SetMaxBackgroundCompactions(10) - // options.SetCompression(gorocksdb.SnappyCompression) - // options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle) - - options.SetHashSkipListRep(2000000, 4, 4) - - blockBasedTableOptions := gorocksdb.NewDefaultBlockBasedTableOptions() - blockBasedTableOptions.SetBlockCache(gorocksdb.NewLRUCache(64 * 1024)) - blockBasedTableOptions.SetFilterPolicy(bloomFilter) - blockBasedTableOptions.SetBlockSizeDeviation(5) - blockBasedTableOptions.SetBlockRestartInterval(10) - blockBasedTableOptions.SetBlockCacheCompressed(gorocksdb.NewLRUCache(64 * 1024)) - blockBasedTableOptions.SetCacheIndexAndFilterBlocks(true) - blockBasedTableOptions.SetIndexType(gorocksdb.KHashSearchIndexType) - - options.SetBlockBasedTableFactory(blockBasedTableOptions) - //log.Println(bloomFilter, readOptions) - options.SetPrefixExtractor(gorocksdb.NewFixedPrefixTransform(3)) - - options.SetAllowConcurrentMemtableWrites(false) - - db, err := gorocksdb.OpenDb(options, conf.DBPath) - - if err != nil { - log.Fatalln("OPEN DB error", db, err) - db.Close() - return nil, errors.New("fail to open db") - } else { - log.Println("OPEN DB success", db) - } - return db, nil -} - -func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { - // Your Code Here (1). - db, err := OpenDB(conf) - - if err != nil { - log.Println("fail to open db,", nil, db) - } - - ropt := gorocksdb.NewDefaultReadOptions() - wopt := gorocksdb.NewDefaultWriteOptions() - s := &StandAloneStorage{ - db, - ropt, - wopt, - } - return s -} - -func (s *StandAloneStorage) Start() error { - // Your Code Here (1). - return nil -} - -func (s *StandAloneStorage) Stop() error { - // Your Code Here (1). - s.db.Close() - return nil -} - -func (s *StandAloneStorage) Reader(ctx *kvrpcpb.Context) (storage.StorageReader, error) { - // Your Code Here (1). - return &StandAloneStorageReader{ - s.db, - s.roptions, - }, nil -} - -func (s *StandAloneStorage) Write(ctx *kvrpcpb.Context, batch []storage.Modify) error { - // Your Code Here (1). - for _, m := range batch { - switch m.Data.(type) { - case storage.Put: - put := m.Data.(storage.Put) - if err := s.db.Put(s.woptions, engine_util.KeyWithCF(put.Cf, put.Key), put.Value); err != nil { - return err - } - case storage.Delete: - del := m.Data.(storage.Delete) - if err := s.db.Delete(s.woptions, engine_util.KeyWithCF(del.Cf, del.Key)); err != nil { - return err - } - } - } - return nil -} - -type StandAloneStorageReader struct { - db *gorocksdb.DB - roptions *gorocksdb.ReadOptions -} - -func (sReader *StandAloneStorageReader) Close() { - return -} - -func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { - val, err := sReader.db.Get(sReader.roptions, engine_util.KeyWithCF(cf, key)) - - return val.Data(), err -} - -func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { - return engine_util.NewRDBIterator(cf, sReader.db, sReader.roptions) -} - -// func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { -// value, err := sReader.db.Get(sReader.) - -// v, e := engine_util.GetCF(sReader.db, cf, key) -// if e == badger.ErrKeyNotFound { -// return nil, nil -// } -// return v, e -// } -// func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { -// txn := sReader.db.NewTransaction(false) -// return engine_util.NewCFIterator(cf, txn) -// } From 181037da76c455795f0ce629c861202e3f067a36 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 14 Mar 2023 18:54:42 +0800 Subject: [PATCH 10/22] merge and add a tutorial to doc/tinydb --- doc/tinydb/Home.md | 69 +++++++++++++++++++ doc/tinydb/project1-KV Separation.md | 45 ++++++++++++ .../standalone_storage/standalone_storage.go | 53 +------------- .../standalone_storage.go | 42 ----------- 4 files changed, 115 insertions(+), 94 deletions(-) create mode 100644 doc/tinydb/Home.md create mode 100644 doc/tinydb/project1-KV Separation.md delete mode 100644 kv/storage/standalone_storage_ldb/standalone_storage.go diff --git a/doc/tinydb/Home.md b/doc/tinydb/Home.md new file mode 100644 index 000000000..dbb25562e --- /dev/null +++ b/doc/tinydb/Home.md @@ -0,0 +1,69 @@ +# Welcome to the TinyDB! +TinyDB is a course designed to help you quickly familiarize yourself with the underlying storage engine of [TiKV Project](https://github.com/tikv/tikv). + +After completing this course, you will have a better understanding of LSM-tree based KV stores with high throughput, high scalability and high space utilization. + +## Course Introduction +TinyDB is forked from open source [TinyKV](https://github.com/talent-plan/tinykv), a key-value storage system with the Raft consensus algorithm. TinyKV focuses on the storage layer of a distributed database system, which uses [badger](https://github.com/dgraph-io/badger), a Go library to store keys and values, as its storage engine. In order to get closer to the actual implementation of TiKV, TinyDB plans to replace the original storage engine badger with [LevelDB](https://github.com/google/leveldb)/[RocksDB](https://github.com/facebook/rocksdb) wrapped by Golang. Therefore, please modify your implementation of project1 to use the interface of levigo(a wrapper of LevelDB) or gorocksdb(a wrapper of RocksDB) rather than badger. + +In this course, you need to implement an existing optimization method on LevelDB/RocksDB. We provide several projects in this folder, which introduce some classic and generally accepted optimization ideas presented in recent famous paper. Please choose one and implement it. + +After completing the implementation, you need to test and evaluate your optimization. We provide go-ycsb, which can be used to evaluate database performance. If you successfully implement a project, you will get better performance in reading or writing or some other dimension. Finally, you need to chart your evaluation results and submit a report and source code. The experts will give you an appropriate score depending on your optimization results and report. + +### LevelDB/RocksDB +[LevelDB](https://github.com/google/leveldb)/[RocksDB](https://github.com/facebook/rocksdb) is a storage engine for server workloads on various storage media, with the initial focus on fast storage (especially Flash storage). It is a C++ library to store key-value pairs. It supports both point lookups and range scans, and provides different types of ACID guarantees. + +RocksDB borrows significant code from the open source [leveldb](https://code.google.com/google/leveldb/) project and does a lot of performance optimization. It performs better than LevelDB under many real workloads and TiKV uses it as storage engine. However, RocksDB has a higher amount of code and is more difficult to learn. As a beginner's course for KV storage, TinyDB requires you to complete the course with LevelDB. Besides, if you already have the knowledge of LSM-tree or enough coding ability, TinyDB also provide code based on the RocksDB. + +## Evaluation & Report +After finishing the project, you need to present evaluation results that demonstrate the benefits of your optimization and write a report. After submitting, you will receive a score based on the optimization results and your report. + +### Go-ycsb +Go-ycsb is a Go port of YCSB. It fully supports all YCSB generators and the Core workload so we can do the basic CRUD benchmarks with Go. Please follow [go-ycsb](https://github.com/pingcap/go-ycsb/blob/master/README.md) to understand it. There are different workloads can be used in the go-ycsb/workloads folder. The specific parameters can be changed if needed. + +## Build TinyDB from Source + +### Prerequisites + +* `git`: The source code of TinyDB is hosted on GitHub as a git repository. To work with git repository, please [install `git`](https://git-scm.com/downloads). +* `go`: TinyDB is a Go project. To build TinyDB from source, please [install `go`](https://golang.org/doc/install) with version greater or equal to 1.13. +* `leveldb`: LevelDB is a storage engine of TinyDB, please [install `leveldb`](https://github.com/google/leveldb) with version greater or equal to 1.7. +* `rocksdb`: RocksDB is also a storage engine of TinyDB, please [install `rocksdb`](https://github.com/facebook/rocksdb) with version greater or equal to 5.16. + +### Clone + +Clone the source code to your development machine. + +```bash +git clone https://github.com/QingyangZ/tinydb +``` + +### Build + +Build TinyDB from the source code. + +```bash +cd tinydb +make +``` + +### Go-ycsb Test + +Build go-ycsb and test TinyDB + +```bash +cd go-ycsb +make +``` + +#### Load + +```bash +./bin/go-ycsb load tinydb -P workloads/workloada +``` + +#### Run +```bash +./bin/go-ycsb run tinydb -P workloads/workloada +``` + \ No newline at end of file diff --git a/doc/tinydb/project1-KV Separation.md b/doc/tinydb/project1-KV Separation.md new file mode 100644 index 000000000..a25132269 --- /dev/null +++ b/doc/tinydb/project1-KV Separation.md @@ -0,0 +1,45 @@ +# Project 1: KV Separation + +KV Separation is inspired by the paper WiscKey published in the FAST Conference, whose main idea is to separate the keys from values to minimize the I/O amplification. It is the first project you need to implement. +![kv separation](http://catkang.github.io/assets/img/lsm_upon_ssd/kv_split.png) + +## Features +* WiscKey writes faster than LevelDB because the KV separation reduces write magnification and makes the compaction much more efficient, especially when KV size is large. +* The read performance of WiscKey may be a little poor than LevelDB, because KV Separation has the disadvantage of requiring one more operation to read value from disk. However, KV Separation allows the SST file to store more KV records, which greatly reduces the number of SST files. As a result, more KV records can be stored by the tablecache and blockcache. In conclusion, the read performance of KV Separation is not as bad as expected. +* The range operations become inefficient, because it changes from a sequential read to a sequential read plus multiple random reads. With the parallel IO capability of SSD, this loss can be offset as much as possible, which is due to the strong random access performance of SSD. + +## Code Guidance +Here are some steps to implement this project. (You can also complete this project in your own way) + +### Prerequisites +`leveldb`: the basic storage engine of TinyDB. Please learn the architecture of LSM-Tree by its wiki or readme text. In terms of code, you should at least know the implemetation of Read(function `DBImpl::Get` in `leveldb/db/db_impl.cc`), Write(function `DBImpl::Write` in `leveldb/db/db_impl.cc`) and Compaction (function `DBImpl::BackgroundCompaction` in `leveldb/db/db_impl.cc`). + +`rocksdb`: if you already have a good understanding of leveldb and plan to implement the project on rocksdb, please learn the implemetation of its Read(function `DBImpl::Get` in `rocksdb/db/db_impl/db_impl.cc`), Write(function `DBImpl::WriteImpl` in `rocksdb/db/db_impl/db_impl_write.cc`) and Compaction(function `DBImpl::BackgroundCompaction` in `rocksdb/db/db_impl/db_impl_compaction_flush.cc`). + +### Initialization +First, you need to initialize a vLog file to store values. For convenience, the original log file structure of leveldb and rocksdb can be use for reference. Take leveldb as an example. The original log file is initialized when the database is opened in the function `DB::Open` (`leveldb/db/db_impl.cc`). Please implement a function to initialize a new vLog file. + +### Write +Unlike LevelDB, the KV pairs are first written to the vLog file. The record written to the memtable and SST files is actually . The address points to the offset of the KV pairs in the vLog file. Therefore, the function `DBImpl::Write` (`leveldb/db/db_impl.cc`) needs to be modified. First, write the value into the vLog file and get the address. Then write key, address and log_number into LSM tree. + +### Read +The result read from a memtable or SST file is actually an address pointing to a location in the vLog file. As a result, the function `DBImpl::Get` (`leveldb/db/db_impl.cc`) alse needs to be modified. After getting the address and log_number from LSM tree, the real value needs to be read from the vLog file pointed from the address. + +### Crash Consistency +In LevelDB's implementation, when immemtbale is flushed into the SST files successfully and the change is already logged in the manifest file (VersionSet::LogAndApply succeeds), the LOG file will be deleted. Otherwise, the database will replay the log file during recovery. + +However, vLog file cannot be deleted in KV Separation, because the contents of each value are still stored here. So what to do when recovering from failure? It is impossible to replay the vLog file from beginning to end. WiscKey's approach is to add `HeadInfo:pos` to VersionEdit before immemtable being flushed. `HeadInfo:pos` means the checkpoint in vLog, which means that the KV record before `HeadInfo:pos` has been successfully written to the SST files. When recovering, `VersionSet::Recover` (`leveldb/db/version_set.cc`) will replay the edit record, get the HeadInfo. After getting the `HeadInfo:pos`, it will replay the vLog file from the pos location. + +### Garbage Collection +Key-value stores based on standard LSM-trees do not immediately reclaim free space when a key-value pair is deleted or overwritten. During compaction, if data relating to a deleted or overwritten key-value pair is found, the data is discarded and space is reclaimed. In WiscKey, only invalid keys are reclaimed by the LSM tree compaction. Since WiscKey does not compact values, it needs a special garbage collector to reclaim free space in the vLog. + +![gc](https://github.com/joker-qi/WiscKey/raw/master/images/garbage.png) + +WiscKey targets a lightweight and online garbage collector. To make this possible, WiscKey introduces a new data layout as shown in the figure above: the tuple (key size, value size, key, value) is stored in the vLog. + +During garbage collection, WiscKey first reads a chunk of key-value pairs (e.g., several MBs) from the tail of the vLog, then finds which of those values are valid (not yet overwritten or deleted) by querying the LSM-tree. WiscKey then appends valid values back to the head of the vLog. Finally, it frees the space occupied previously by the chunk, and updates the tail accordingly. + +Please implement a garbage collection function which should be called when appropriate. For example, you can record `drop_count` which means the number of invalid keys during compactions (function `DBImpl::BackgroundCompaction` in `leveldb/db/db_impl.cc`). When `drop_count` reaches a certain threshold, garbage collection is triggered. + +## Optimizations +The above implementation still has many shortcomings. The paper also mentions some optimizations you can implement. Moreover, it is better if you have other optimization ideas and implement them, which will get you a higher score. \ No newline at end of file diff --git a/kv/storage/standalone_storage/standalone_storage.go b/kv/storage/standalone_storage/standalone_storage.go index 0c085f758..80c793f9a 100644 --- a/kv/storage/standalone_storage/standalone_storage.go +++ b/kv/storage/standalone_storage/standalone_storage.go @@ -1,9 +1,7 @@ package standalone_storage import ( - "errors" - "log" - + "github.com/jmhodges/levigo" "github.com/tecbot/gorocksdb" "github.com/pingcap-incubator/tinykv/kv/config" @@ -18,55 +16,6 @@ type StandAloneStorage struct { // Your Data Here (1). } -// opendb -func OpenDB(conf *config.Config) (*gorocksdb.DB, error) { - options := gorocksdb.NewDefaultOptions() - options.SetCreateIfMissing(true) - - bloomFilter := gorocksdb.NewBloomFilter(10) - - readOptions := gorocksdb.NewDefaultReadOptions() - readOptions.SetFillCache(false) - - rateLimiter := gorocksdb.NewRateLimiter(10000000, 10000, 10) - options.SetRateLimiter(rateLimiter) - options.SetCreateIfMissing(true) - options.EnableStatistics() - options.SetWriteBufferSize(8 * 1024) - options.SetMaxWriteBufferNumber(3) - options.SetMaxBackgroundCompactions(10) - // options.SetCompression(gorocksdb.SnappyCompression) - // options.SetCompactionStyle(gorocksdb.UniversalCompactionStyle) - - options.SetHashSkipListRep(2000000, 4, 4) - - blockBasedTableOptions := gorocksdb.NewDefaultBlockBasedTableOptions() - blockBasedTableOptions.SetBlockCache(gorocksdb.NewLRUCache(64 * 1024)) - blockBasedTableOptions.SetFilterPolicy(bloomFilter) - blockBasedTableOptions.SetBlockSizeDeviation(5) - blockBasedTableOptions.SetBlockRestartInterval(10) - blockBasedTableOptions.SetBlockCacheCompressed(gorocksdb.NewLRUCache(64 * 1024)) - blockBasedTableOptions.SetCacheIndexAndFilterBlocks(true) - blockBasedTableOptions.SetIndexType(gorocksdb.KHashSearchIndexType) - - options.SetBlockBasedTableFactory(blockBasedTableOptions) - //log.Println(bloomFilter, readOptions) - options.SetPrefixExtractor(gorocksdb.NewFixedPrefixTransform(3)) - - options.SetAllowConcurrentMemtableWrites(false) - - db, err := gorocksdb.OpenDb(options, conf.DBPath) - - if err != nil { - log.Fatalln("OPEN DB error", db, err) - db.Close() - return nil, errors.New("fail to open db") - } else { - log.Println("OPEN DB success", db) - } - return db, nil -} - func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { // Your Code Here (1). return nil diff --git a/kv/storage/standalone_storage_ldb/standalone_storage.go b/kv/storage/standalone_storage_ldb/standalone_storage.go deleted file mode 100644 index 09a62b77a..000000000 --- a/kv/storage/standalone_storage_ldb/standalone_storage.go +++ /dev/null @@ -1,42 +0,0 @@ -package standalone_storage_ldb - -import ( - "fmt" - - "github.com/jmhodges/levigo" - "github.com/pingcap-incubator/tinykv/kv/config" - "github.com/pingcap-incubator/tinykv/kv/storage" - "github.com/pingcap-incubator/tinykv/kv/util/engine_util" - "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" -) - -// StandAloneStorage is an implementation of `Storage` for a single-node TinyKV instance. It does not -// communicate with other nodes and all data is stored locally. -type StandAloneStorage struct { - // Your Data Here (1). -} - -func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { - // Your Code Here (1). - return nil -} - -func (s *StandAloneStorage) Start() error { - // Your Code Here (1). - return nil -} - -func (s *StandAloneStorage) Stop() error { - // Your Code Here (1). - return nil -} - -func (s *StandAloneStorage) Reader(ctx *kvrpcpb.Context) (storage.StorageReader, error) { - // Your Code Here (1). - return nil, nil -} - -func (s *StandAloneStorage) Write(ctx *kvrpcpb.Context, batch []storage.Modify) error { - // Your Code Here (1). - return nil -} From 1e59452e535f3c1f831cbb8475f8f1c48c26dba5 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Wed, 22 Mar 2023 22:14:06 +0800 Subject: [PATCH 11/22] remove go-ycsb --- go-ycsb/.dockerignore | 3 - go-ycsb/.github/workflows/docker.yml | 19 - .../workflows/github-release-publish.yml | 26 - go-ycsb/.github/workflows/go.yml | 56 - go-ycsb/.gitignore | 9 - go-ycsb/Dockerfile | 32 - go-ycsb/LICENSE | 201 --- go-ycsb/Makefile | 41 - go-ycsb/README.md | 341 ----- go-ycsb/YCSB-LICENSE | 202 --- go-ycsb/cmd/go-ycsb/client.go | 111 -- go-ycsb/cmd/go-ycsb/main.go | 200 --- go-ycsb/cmd/go-ycsb/shell.go | 247 ---- go-ycsb/db/aerospike/db.go | 176 --- go-ycsb/db/badger/db.go | 263 ---- go-ycsb/db/basic/db.go | 272 ---- go-ycsb/db/boltdb/db.go | 244 ---- go-ycsb/db/cassandra/db.go | 268 ---- go-ycsb/db/dynamodb/db.go | 295 ---- go-ycsb/db/elasticsearch/db.go | 346 ----- go-ycsb/db/etcd/db.go | 163 --- go-ycsb/db/etcd/doc.go | 3 - go-ycsb/db/foundationdb/db.go | 204 --- go-ycsb/db/foundationdb/doc.go | 3 - go-ycsb/db/minio/db.go | 128 -- go-ycsb/db/mongodb/db.go | 203 --- go-ycsb/db/mysql/db.go | 512 ------- go-ycsb/db/pegasus/db.go | 176 --- go-ycsb/db/pg/db.go | 361 ----- go-ycsb/db/redis/db.go | 358 ----- go-ycsb/db/rocksdb/db.go | 265 ---- go-ycsb/db/rocksdb/doc.go | 3 - go-ycsb/db/spanner/db.go | 358 ----- go-ycsb/db/sqlite/db.go | 400 ------ go-ycsb/db/sqlite/doc.go | 3 - go-ycsb/db/tinydb/db.go | 160 --- go-ycsb/fdb-dockerfile | 37 - go-ycsb/go.mod | 176 --- go-ycsb/go.sum | 1234 ----------------- go-ycsb/pkg/client/client.go | 227 --- go-ycsb/pkg/client/dbwrapper.go | 174 --- go-ycsb/pkg/generator/acknowledged_counter.go | 110 -- go-ycsb/pkg/generator/constant.go | 53 - go-ycsb/pkg/generator/counter.go | 58 - go-ycsb/pkg/generator/discrete.go | 77 - go-ycsb/pkg/generator/exponential.go | 64 - go-ycsb/pkg/generator/histogram.go | 134 -- go-ycsb/pkg/generator/hotspot.go | 87 -- go-ycsb/pkg/generator/number.go | 46 - go-ycsb/pkg/generator/scrambled_zipfian.go | 76 - go-ycsb/pkg/generator/sequential.go | 63 - go-ycsb/pkg/generator/skewedlatest.go | 68 - go-ycsb/pkg/generator/uniform.go | 57 - go-ycsb/pkg/generator/zipfian.go | 170 --- go-ycsb/pkg/measurement/csv.go | 51 - go-ycsb/pkg/measurement/histogram.go | 97 -- go-ycsb/pkg/measurement/histograms.go | 76 - go-ycsb/pkg/measurement/measurement.go | 126 -- go-ycsb/pkg/prop/prop.go | 119 -- go-ycsb/pkg/util/concurrent_map.go | 319 ----- go-ycsb/pkg/util/core.go | 134 -- go-ycsb/pkg/util/core_test.go | 24 - go-ycsb/pkg/util/hack.go | 43 - go-ycsb/pkg/util/hash.go | 46 - go-ycsb/pkg/util/output.go | 77 - go-ycsb/pkg/util/row.go | 115 -- go-ycsb/pkg/util/row_test.go | 38 - go-ycsb/pkg/util/spinlock.go | 50 - go-ycsb/pkg/util/tls.go | 59 - go-ycsb/pkg/util/util.go | 73 - go-ycsb/pkg/workload/core.go | 705 ---------- go-ycsb/pkg/ycsb/db.go | 120 -- go-ycsb/pkg/ycsb/generator.go | 28 - go-ycsb/pkg/ycsb/measurement.go | 31 - go-ycsb/pkg/ycsb/workload.go | 71 - go-ycsb/tool/binary/bench.sh | 59 - go-ycsb/tool/binary/property/badger | 16 - go-ycsb/tool/binary/property/rocksdb | 27 - go-ycsb/tool/docker/bench.sh | 103 -- go-ycsb/tool/docker/cassandra.yml | 24 - go-ycsb/tool/docker/clear.sh | 7 - go-ycsb/tool/docker/cockroach.yml | 26 - go-ycsb/tool/docker/config/init_cassandra.sh | 14 - go-ycsb/tool/docker/config/init_cockroach.sh | 8 - go-ycsb/tool/docker/config/pd.toml | 91 -- go-ycsb/tool/docker/config/tidb.toml | 260 ---- go-ycsb/tool/docker/config/tikv.toml | 689 --------- go-ycsb/tool/docker/mariadb.yml | 17 - go-ycsb/tool/docker/mysql.yml | 17 - go-ycsb/tool/docker/mysql8.yml | 17 - go-ycsb/tool/docker/pg.yml | 18 - go-ycsb/tool/docker/raw.yml | 44 - go-ycsb/tool/docker/run_bench.sh | 10 - go-ycsb/tool/docker/scylla.yml | 24 - go-ycsb/tool/docker/sqlite.yml | 8 - go-ycsb/tool/docker/tidb.yml | 60 - go-ycsb/tool/docker/tikv.yml | 44 - go-ycsb/tool/report.go | 245 ---- go-ycsb/workloads/minio | 4 - go-ycsb/workloads/workload_template | 206 --- go-ycsb/workloads/workloada | 37 - go-ycsb/workloads/workloadb | 36 - go-ycsb/workloads/workloadc | 38 - go-ycsb/workloads/workloadd | 41 - go-ycsb/workloads/workloade | 46 - go-ycsb/workloads/workloadf | 37 - 106 files changed, 14238 deletions(-) delete mode 100644 go-ycsb/.dockerignore delete mode 100644 go-ycsb/.github/workflows/docker.yml delete mode 100644 go-ycsb/.github/workflows/github-release-publish.yml delete mode 100644 go-ycsb/.github/workflows/go.yml delete mode 100644 go-ycsb/.gitignore delete mode 100644 go-ycsb/Dockerfile delete mode 100644 go-ycsb/LICENSE delete mode 100644 go-ycsb/Makefile delete mode 100644 go-ycsb/README.md delete mode 100644 go-ycsb/YCSB-LICENSE delete mode 100644 go-ycsb/cmd/go-ycsb/client.go delete mode 100644 go-ycsb/cmd/go-ycsb/main.go delete mode 100644 go-ycsb/cmd/go-ycsb/shell.go delete mode 100644 go-ycsb/db/aerospike/db.go delete mode 100644 go-ycsb/db/badger/db.go delete mode 100644 go-ycsb/db/basic/db.go delete mode 100644 go-ycsb/db/boltdb/db.go delete mode 100644 go-ycsb/db/cassandra/db.go delete mode 100644 go-ycsb/db/dynamodb/db.go delete mode 100644 go-ycsb/db/elasticsearch/db.go delete mode 100644 go-ycsb/db/etcd/db.go delete mode 100644 go-ycsb/db/etcd/doc.go delete mode 100644 go-ycsb/db/foundationdb/db.go delete mode 100644 go-ycsb/db/foundationdb/doc.go delete mode 100644 go-ycsb/db/minio/db.go delete mode 100644 go-ycsb/db/mongodb/db.go delete mode 100644 go-ycsb/db/mysql/db.go delete mode 100644 go-ycsb/db/pegasus/db.go delete mode 100644 go-ycsb/db/pg/db.go delete mode 100644 go-ycsb/db/redis/db.go delete mode 100644 go-ycsb/db/rocksdb/db.go delete mode 100644 go-ycsb/db/rocksdb/doc.go delete mode 100644 go-ycsb/db/spanner/db.go delete mode 100644 go-ycsb/db/sqlite/db.go delete mode 100644 go-ycsb/db/sqlite/doc.go delete mode 100644 go-ycsb/db/tinydb/db.go delete mode 100644 go-ycsb/fdb-dockerfile delete mode 100644 go-ycsb/go.mod delete mode 100644 go-ycsb/go.sum delete mode 100644 go-ycsb/pkg/client/client.go delete mode 100644 go-ycsb/pkg/client/dbwrapper.go delete mode 100644 go-ycsb/pkg/generator/acknowledged_counter.go delete mode 100644 go-ycsb/pkg/generator/constant.go delete mode 100644 go-ycsb/pkg/generator/counter.go delete mode 100644 go-ycsb/pkg/generator/discrete.go delete mode 100644 go-ycsb/pkg/generator/exponential.go delete mode 100644 go-ycsb/pkg/generator/histogram.go delete mode 100644 go-ycsb/pkg/generator/hotspot.go delete mode 100644 go-ycsb/pkg/generator/number.go delete mode 100644 go-ycsb/pkg/generator/scrambled_zipfian.go delete mode 100644 go-ycsb/pkg/generator/sequential.go delete mode 100644 go-ycsb/pkg/generator/skewedlatest.go delete mode 100644 go-ycsb/pkg/generator/uniform.go delete mode 100644 go-ycsb/pkg/generator/zipfian.go delete mode 100644 go-ycsb/pkg/measurement/csv.go delete mode 100644 go-ycsb/pkg/measurement/histogram.go delete mode 100644 go-ycsb/pkg/measurement/histograms.go delete mode 100644 go-ycsb/pkg/measurement/measurement.go delete mode 100644 go-ycsb/pkg/prop/prop.go delete mode 100644 go-ycsb/pkg/util/concurrent_map.go delete mode 100644 go-ycsb/pkg/util/core.go delete mode 100644 go-ycsb/pkg/util/core_test.go delete mode 100644 go-ycsb/pkg/util/hack.go delete mode 100644 go-ycsb/pkg/util/hash.go delete mode 100644 go-ycsb/pkg/util/output.go delete mode 100644 go-ycsb/pkg/util/row.go delete mode 100644 go-ycsb/pkg/util/row_test.go delete mode 100644 go-ycsb/pkg/util/spinlock.go delete mode 100644 go-ycsb/pkg/util/tls.go delete mode 100644 go-ycsb/pkg/util/util.go delete mode 100644 go-ycsb/pkg/workload/core.go delete mode 100644 go-ycsb/pkg/ycsb/db.go delete mode 100644 go-ycsb/pkg/ycsb/generator.go delete mode 100644 go-ycsb/pkg/ycsb/measurement.go delete mode 100644 go-ycsb/pkg/ycsb/workload.go delete mode 100755 go-ycsb/tool/binary/bench.sh delete mode 100644 go-ycsb/tool/binary/property/badger delete mode 100644 go-ycsb/tool/binary/property/rocksdb delete mode 100755 go-ycsb/tool/docker/bench.sh delete mode 100644 go-ycsb/tool/docker/cassandra.yml delete mode 100755 go-ycsb/tool/docker/clear.sh delete mode 100644 go-ycsb/tool/docker/cockroach.yml delete mode 100755 go-ycsb/tool/docker/config/init_cassandra.sh delete mode 100755 go-ycsb/tool/docker/config/init_cockroach.sh delete mode 100644 go-ycsb/tool/docker/config/pd.toml delete mode 100644 go-ycsb/tool/docker/config/tidb.toml delete mode 100644 go-ycsb/tool/docker/config/tikv.toml delete mode 100644 go-ycsb/tool/docker/mariadb.yml delete mode 100644 go-ycsb/tool/docker/mysql.yml delete mode 100644 go-ycsb/tool/docker/mysql8.yml delete mode 100644 go-ycsb/tool/docker/pg.yml delete mode 100644 go-ycsb/tool/docker/raw.yml delete mode 100755 go-ycsb/tool/docker/run_bench.sh delete mode 100644 go-ycsb/tool/docker/scylla.yml delete mode 100644 go-ycsb/tool/docker/sqlite.yml delete mode 100644 go-ycsb/tool/docker/tidb.yml delete mode 100644 go-ycsb/tool/docker/tikv.yml delete mode 100644 go-ycsb/tool/report.go delete mode 100644 go-ycsb/workloads/minio delete mode 100644 go-ycsb/workloads/workload_template delete mode 100644 go-ycsb/workloads/workloada delete mode 100644 go-ycsb/workloads/workloadb delete mode 100644 go-ycsb/workloads/workloadc delete mode 100644 go-ycsb/workloads/workloadd delete mode 100644 go-ycsb/workloads/workloade delete mode 100644 go-ycsb/workloads/workloadf diff --git a/go-ycsb/.dockerignore b/go-ycsb/.dockerignore deleted file mode 100644 index 8effa3eae..000000000 --- a/go-ycsb/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.git -LICENSE -PATENTS diff --git a/go-ycsb/.github/workflows/docker.yml b/go-ycsb/.github/workflows/docker.yml deleted file mode 100644 index d6a4ca036..000000000 --- a/go-ycsb/.github/workflows/docker.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Build the Docker image - run: docker build . --file Dockerfile --tag my-image-name:$(date +%s) - diff --git a/go-ycsb/.github/workflows/github-release-publish.yml b/go-ycsb/.github/workflows/github-release-publish.yml deleted file mode 100644 index 436b2616c..000000000 --- a/go-ycsb/.github/workflows/github-release-publish.yml +++ /dev/null @@ -1,26 +0,0 @@ -# .github/workflows/github-release-publish.yml -name: Publish artifacts to github release - -on: - release: - types: [published] - -jobs: - releases-matrix: - name: Release Go Binary - runs-on: ubuntu-latest - strategy: - matrix: - goos: [linux, darwin] - goarch: [amd64, arm64] - steps: - - uses: actions/checkout@v3 - - uses: wangyoucao577/go-release-action@v1.28 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - binary_name: "./bin/go-ycsb" - sha256sum: true - asset_name: go-ycsb-${{ matrix.goos }}-${{ matrix.goarch }} - build_command: "make" diff --git a/go-ycsb/.github/workflows/go.yml b/go-ycsb/.github/workflows/go.yml deleted file mode 100644 index 37c973ed3..000000000 --- a/go-ycsb/.github/workflows/go.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Go - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - - build: - runs-on: ubuntu-latest - strategy: - matrix: - os: ["linux", "darwin"] - arch: ["amd64", "arm64"] - steps: - - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.18 - - - name: Build for ${{ matrix.os }}-${{ matrix.arch }} - env: - GOOS: ${{ matrix.os }} - GOARCH: ${{ matrix.arch }} - run: | - make - tar -C bin -zcf go-ycsb-$GOOS-$GOARCH.tar.gz go-ycsb - - - name: Release latest build - uses: softprops/action-gh-release@v1 - if: github.event_name == 'push' - with: - name: Latest Build - tag_name: latest-${{ github.sha }} - files: | - *.tar.gz - - name: Clean legacy latest releases - uses: actions/github-script@v6 - if: github.event_name == 'push' - with: - script: | - const { owner, repo } = context.repo; - const releases = (await github.rest.repos.listReleases({ owner, repo })).data.filter(r => r.draft && r.tag_name.startsWith('latest')); - for (const r of releases) { await github.rest.repos.deleteRelease({ owner, repo, release_id: r.id }).catch(_ => {}); } - - name: Clean legacy latest tags - if: github.event_name == 'push' - run: | - git tag -l | grep latest | grep -v latest-${{ github.sha }} | xargs -I{} git push -d origin {} || true diff --git a/go-ycsb/.gitignore b/go-ycsb/.gitignore deleted file mode 100644 index 6746da662..000000000 --- a/go-ycsb/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -bin -data -logs -log -.idea -.vscode -vendor -tool/tool -.DS_Store \ No newline at end of file diff --git a/go-ycsb/Dockerfile b/go-ycsb/Dockerfile deleted file mode 100644 index 689451513..000000000 --- a/go-ycsb/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM golang:1.18.4-alpine3.16 - -ENV GOPATH /go - -RUN apk update && apk upgrade && \ - apk add --no-cache git build-base wget - -RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 \ - && chmod +x /usr/local/bin/dumb-init - -RUN mkdir -p /go/src/github.com/pingcap/go-ycsb -WORKDIR /go/src/github.com/pingcap/go-ycsb - -COPY go.mod . -COPY go.sum . - -RUN GO111MODULE=on go mod download - -COPY . . - -RUN GO111MODULE=on go build -o /go-ycsb ./cmd/* - -FROM alpine:3.8 - -COPY --from=0 /go-ycsb /go-ycsb -COPY --from=0 /usr/local/bin/dumb-init /usr/local/bin/dumb-init - -ADD workloads /workloads - -EXPOSE 6060 - -ENTRYPOINT [ "/usr/local/bin/dumb-init", "/go-ycsb" ] diff --git a/go-ycsb/LICENSE b/go-ycsb/LICENSE deleted file mode 100644 index 261eeb9e9..000000000 --- a/go-ycsb/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/go-ycsb/Makefile b/go-ycsb/Makefile deleted file mode 100644 index 478f60a51..000000000 --- a/go-ycsb/Makefile +++ /dev/null @@ -1,41 +0,0 @@ -FDB_CHECK := $(shell command -v fdbcli 2> /dev/null) -ROCKSDB_CHECK := $(shell echo "int main() { return 0; }" | gcc -lrocksdb -x c++ -o /dev/null - 2>/dev/null; echo $$?) -SQLITE_CHECK := $(shell echo "int main() { return 0; }" | gcc -lsqlite3 -x c++ -o /dev/null - 2>/dev/null; echo $$?) - -TAGS = - -ifdef FDB_CHECK - TAGS += foundationdb -endif - -ifneq ($(shell go env GOOS), $(shell go env GOHOSTOS)) - CROSS_COMPILE := 1 -endif -ifneq ($(shell go env GOARCH), $(shell go env GOHOSTARCH)) - CROSS_COMPILE := 1 -endif - -ifndef CROSS_COMPILE - -ifeq ($(SQLITE_CHECK), 0) - TAGS += libsqlite3 -endif - -ifeq ($(ROCKSDB_CHECK), 0) - TAGS += rocksdb - CGO_CXXFLAGS := "${CGO_CXXFLAGS} -std=c++11" - CGO_FLAGS += CGO_CXXFLAGS=$(CGO_CXXFLAGS) -endif - -endif - -default: build - -build: export GO111MODULE=on -build: - $(CGO_FLAGS) go build -o bin/go-ycsb cmd/go-ycsb/* - -check: - golint -set_exit_status db/... cmd/... pkg/... - - diff --git a/go-ycsb/README.md b/go-ycsb/README.md deleted file mode 100644 index 9b4c1f094..000000000 --- a/go-ycsb/README.md +++ /dev/null @@ -1,341 +0,0 @@ -# go-ycsb - -go-ycsb is a Go port of [YCSB](https://github.com/brianfrankcooper/YCSB). It fully supports all YCSB generators and the Core workload so we can do the basic CRUD benchmarks with Go. - -## Why another Go YCSB? - -+ We want to build a standard benchmark tool in Go. -+ We are not familiar with Java. - -## Getting Started - -### Download - -https://github.com/pingcap/go-ycsb/releases/latest - -**Linux** -``` -wget -c https://github.com/pingcap/go-ycsb/releases/latest/download/go-ycsb-linux-amd64.tar.gz -O - | tar -xz - -# give it a try -./go-ycsb --help -``` - -**OSX** -``` -wget -c https://github.com/pingcap/go-ycsb/releases/latest/download/go-ycsb-darwin-amd64.tar.gz -O - | tar -xz - -# give it a try -./go-ycsb --help -``` - -### Building from source - -```bash -git clone https://github.com/pingcap/go-ycsb.git -cd go-ycsb -make - -# give it a try -./bin/go-ycsb --help -``` - -Notice: - -+ Minimum supported go version is 1.16. -+ To use FoundationDB, you must install [client](https://www.foundationdb.org/download/) library at first, now the supported version is 6.2.11. -+ To use RocksDB, you must follow [INSTALL](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) to install RocksDB at first. - -## Usage - -Mostly, we can start from the official document [Running-a-Workload](https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload). - -### Shell - -```basic -./bin/go-ycsb shell basic -» help -YCSB shell command - -Usage: - shell [command] - -Available Commands: - delete Delete a record - help Help about any command - insert Insert a record - read Read a record - scan Scan starting at key - table Get or [set] the name of the table - update Update a record -``` - -### Load - -```bash -./bin/go-ycsb load basic -P workloads/workloada -``` - -### Run - -```bash -./bin/go-ycsb run basic -P workloads/workloada -``` - -## Supported Database - -- MySQL / TiDB -- TiKV -- FoundationDB -- Aerospike -- Badger -- Cassandra / ScyllaDB -- Pegasus -- PostgreSQL / CockroachDB / AlloyDB / Yugabyte -- RocksDB -- Spanner -- Sqlite -- MongoDB -- Redis and Redis Cluster -- BoltDB -- etcd -- DynamoDB - -## Output configuration - -|field|default value|description| -|-|-|-| -|measurementtype|"histogram"|The mechanism for recording measurements, one of `histogram`, `raw` or `csv`| -|measurement.output_file|""|File to write output to, default writes to stdout| - -## Database Configuration - -You can pass the database configurations through `-p field=value` in the command line directly. - -Common configurations: - -|field|default value|description| -|-|-|-| -|dropdata|false|Whether to remove all data before test| -|verbose|false|Output the execution query| -|debug.pprof|":6060"|Go debug profile address| - -### MySQL & TiDB - -|field|default value|description| -|-|-|-| -|mysql.host|"127.0.0.1"|MySQL Host| -|mysql.port|3306|MySQL Port| -|mysql.user|"root"|MySQL User| -|mysql.password||MySQL Password| -|mysql.db|"test"|MySQL Database| -|tidb.cluster_index|true|Whether to use cluster index, for TiDB only| -|tidb.instances|""|Comma-seperated address list of tidb instances (eg: `tidb-0:4000,tidb-1:4000`)| - - -### TiKV - -|field|default value|description| -|-|-|-| -|tikv.pd|"127.0.0.1:2379"|PD endpoints, seperated by comma| -|tikv.type|"raw"|TiKV mode, "raw", "txn", or "coprocessor"| -|tikv.conncount|128|gRPC connection count| -|tikv.batchsize|128|Request batch size| -|tikv.async_commit|true|Enalbe async commit or not| -|tikv.one_pc|true|Enable one phase or not| -|tikv.apiversion|"V1"|[api-version](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#api-version-new-in-v610) of tikv server, "V1" or "V2"| - -### FoundationDB - -|field|default value|description| -|-|-|-| -|fdb.cluster|""|The cluster file used for FoundationDB, if not set, will use the [default](https://apple.github.io/foundationdb/administration.html#default-cluster-file)| -|fdb.dbname|"DB"|The cluster database name| -|fdb.apiversion|510|API version, now only 5.1 is supported| - -### PostgreSQL & CockroachDB & AlloyDB & Yugabyte - -|field|default value|description| -|-|-|-| -|pg.host|"127.0.0.1"|PostgreSQL Host| -|pg.port|5432|PostgreSQL Port| -|pg.user|"root"|PostgreSQL User| -|pg.password||PostgreSQL Password| -|pg.db|"test"|PostgreSQL Database| -|pg.sslmode|"disable|PostgreSQL ssl mode| - -### Aerospike - -|field|default value|description| -|-|-|-| -|aerospike.host|"localhost"|The port of the Aerospike service| -|aerospike.port|3000|The port of the Aerospike service| -|aerospike.ns|"test"|The namespace to use| - -### Badger - -|field|default value|description| -|-|-|-| -|badger.dir|"/tmp/badger"|The directory to save data| -|badger.valuedir|"/tmp/badger"|The directory to save value, if not set, use badger.dir| -|badger.sync_writes|false|Sync all writes to disk| -|badger.num_versions_to_keep|1|How many versions to keep per key| -|badger.max_table_size|64MB|Each table (or file) is at most this size| -|badger.level_size_multiplier|10|Equals SizeOf(Li+1)/SizeOf(Li)| -|badger.max_levels|7|Maximum number of levels of compaction| -|badger.value_threshold|32|If value size >= this threshold, only store value offsets in tree| -|badger.num_memtables|5|Maximum number of tables to keep in memory, before stalling| -|badger.num_level0_tables|5|Maximum number of Level 0 tables before we start compacting| -|badger.num_level0_tables_stall|10|If we hit this number of Level 0 tables, we will stall until L0 is compacted away| -|badger.level_one_size|256MB|Maximum total size for L1| -|badger.value_log_file_size|1GB|Size of single value log file| -|badger.value_log_max_entries|1000000|Max number of entries a value log file can hold (approximately). A value log file would be determined by the smaller of its file size and max entries| -|badger.num_compactors|3|Number of compaction workers to run concurrently| -|badger.do_not_compact|false|Stops LSM tree from compactions| -|badger.table_loading_mode|options.LoadToRAM|How should LSM tree be accessed| -|badger.value_log_loading_mode|options.MemoryMap|How should value log be accessed| - -### RocksDB - -|field|default value|description| -|-|-|-| -|rocksdb.dir|"/tmp/rocksdb"|The directory to save data| -|rocksdb.allow_concurrent_memtable_writes|true|Sets whether to allow concurrent memtable writes| -|rocksdb.allow_mmap_reads|false|Enable/Disable mmap reads for reading sst tables| -|rocksdb.allow_mmap_writes|false|Enable/Disable mmap writes for writing sst tables| -|rocksdb.arena_block_size|0(write_buffer_size / 8)|Sets the size of one block in arena memory allocation| -|rocksdb.db_write_buffer_size|0(disable)|Sets the amount of data to build up in memtables across all column families before writing to disk| -|rocksdb.hard_pending_compaction_bytes_limit|256GB|Sets the bytes threshold at which all writes are stopped if estimated bytes needed to be compaction exceed this threshold| -|rocksdb.level0_file_num_compaction_trigger|4|Sets the number of files to trigger level-0 compaction| -|rocksdb.level0_slowdown_writes_trigger|20|Sets the soft limit on number of level-0 files| -|rocksdb.level0_stop_writes_trigger|36|Sets the maximum number of level-0 files. We stop writes at this point| -|rocksdb.max_bytes_for_level_base|256MB|Sets the maximum total data size for base level| -|rocksdb.max_bytes_for_level_multiplier|10|Sets the max Bytes for level multiplier| -|rocksdb.max_total_wal_size|0(\[sum of all write_buffer_size * max_write_buffer_number\] * 4)|Sets the maximum total wal size in bytes. Once write-ahead logs exceed this size, we will start forcing the flush of column families whose memtables are backed by the oldest live WAL file (i.e. the ones that are causing all the space amplification)| -|rocksdb.memtable_huge_page_size|0|Sets the page size for huge page for arena used by the memtable| -|rocksdb.num_levels|7|Sets the number of levels for this database| -|rocksdb.use_direct_reads|false|Enable/Disable direct I/O mode (O_DIRECT) for reads| -|rocksdb.use_fsync|false|Enable/Disable fsync| -|rocksdb.write_buffer_size|64MB|Sets the amount of data to build up in memory (backed by an unsorted log on disk) before converting to a sorted on-disk file| -|rocksdb.max_write_buffer_number|2|Sets the maximum number of write buffers that are built up in memory| -|rocksdb.max_background_jobs|2|Sets maximum number of concurrent background jobs (compactions and flushes)| -|rocksdb.block_size|4KB|Sets the approximate size of user data packed per block. Note that the block size specified here corresponds opts uncompressed data. The actual size of the unit read from disk may be smaller if compression is enabled| -|rocksdb.block_size_deviation|10|Sets the block size deviation. This is used opts close a block before it reaches the configured 'block_size'. If the percentage of free space in the current block is less than this specified number and adding a new record opts the block will exceed the configured block size, then this block will be closed and the new record will be written opts the next block| -|rocksdb.cache_index_and_filter_blocks|false|Indicating if we'd put index/filter blocks to the block cache. If not specified, each "table reader" object will pre-load index/filter block during table initialization| -|rocksdb.no_block_cache|false|Specify whether block cache should be used or not| -|rocksdb.pin_l0_filter_and_index_blocks_in_cache|false|Sets cache_index_and_filter_blocks. If is true and the below is true (hash_index_allow_collision), then filter and index blocks are stored in the cache, but a reference is held in the "table reader" object so the blocks are pinned and only evicted from cache when the table reader is freed| -|rocksdb.whole_key_filtering|true|Specify if whole keys in the filter (not just prefixes) should be placed. This must generally be true for gets opts be efficient| -|rocksdb.block_restart_interval|16|Sets the number of keys between restart points for delta encoding of keys. This parameter can be changed dynamically| -|rocksdb.filter_policy|nil|Sets the filter policy opts reduce disk reads. Many applications will benefit from passing the result of NewBloomFilterPolicy() here| -|rocksdb.index_type|kBinarySearch|Sets the index type used for this table. __kBinarySearch__: A space efficient index block that is optimized for binary-search-based index. __kHashSearch__: The hash index, if enabled, will do the hash lookup when `Options.prefix_extractor` is provided. __kTwoLevelIndexSearch__: A two-level index implementation. Both levels are binary search indexes| -|rocksdb.block_align|false|Enable/Disable align data blocks on lesser of page size and block size| - -### Spanner - -|field|default value|description| -|-|-|-| -|spanner.db|""|Spanner Database| -|spanner.credentials|"~/.spanner/credentials.json"|Google application credentials for Spanner| - -### Sqlite - -|field|default value|description| -|-|-|-| -|sqlite.db|"/tmp/sqlite.db"|Database path| -|sqlite.mode|"rwc"|Open Mode: ro, rc, rwc, memory| -|sqlite.journalmode|"DELETE"|Journal mode: DELETE, TRUNCSTE, PERSIST, MEMORY, WAL, OFF| -|sqlite.cache|"Shared"|Cache: shared, private| - -### Cassandra - -|field|default value|description| -|-|-|-| -|cassandra.cluster|"127.0.0.1:9042"|Cassandra cluster| -|cassandra.keyspace|"test"|Keyspace| -|cassandra.connections|2|Number of connections per host| -|cassandra.username|cassandra|Username| -|cassandra.password|cassandra|Password| - -### MongoDB - -|field|default value|description| -|-|-|-| -|mongodb.url|"mongodb://127.0.0.1:27017"|MongoDB URI| -|mongodb.tls_skip_verify|false|Enable/disable server ca certificate verification| -|mongodb.tls_ca_file|""|Path to mongodb server ca certificate file| -|mongodb.namespace|"ycsb.ycsb"|Namespace to use| -|mongodb.authdb|"admin"|Authentication database| -|mongodb.username|N/A|Username for authentication| -|mongodb.password|N/A|Password for authentication| - -### Redis -|field|default value|description| -|-|-|-| -|redis.datatype|hash|"hash", "string" or "json" ("json" requires [RedisJSON](https://redis.io/docs/stack/json/) available)| -|redis.mode|single|"single" or "cluster"| -|redis.network|tcp|"tcp" or "unix"| -|redis.addr||Redis server address(es) in "host:port" form, can be semi-colon `;` separated in cluster mode| -|redis.password||Redis server password| -|redis.db|0|Redis server target db| -|redis.max_redirects|8|The maximum number of retries before giving up (only for cluster mode)| -|redis.read_only|false|Enables read-only commands on slave nodes (only for cluster mode)| -|redis.route_by_latency|false|Allows routing read-only commands to the closest master or slave node (only for cluster mode)| -|redis.route_randomly|false|Allows routing read-only commands to the random master or slave node (only for cluster mode)| -|redis.max_retries||Max retries before giving up connection| -|redis.min_retry_backoff|8ms|Minimum backoff between each retry| -|redis.max_retry_backoff|512ms|Maximum backoff between each retry| -|redis.dial_timeout|5s|Dial timeout for establishing new connection| -|redis.read_timeout|3s|Timeout for socket reads| -|redis.write_timeout|3s|Timeout for socket writes| -|redis.pool_size|10|Maximum number of socket connections| -|redis.min_idle_conns|0|Minimum number of idle connections| -|redis.max_conn_age|0|Connection age at which client closes the connection| -|redis.pool_timeout|4s|Amount of time client waits for connections are busy before returning an error| -|redis.idle_timeout|5m|Amount of time after which client closes idle connections. Should be less than server timeout| -|redis.idle_check_frequency|1m|Frequency of idle checks made by idle connections reaper| -|redis.tls_ca||Path to CA file| -|redis.tls_cert||Path to cert file| -|redis.tls_key||Path to key file| -|redis.tls_insecure_skip_verify|false|Controls whether a client verifies the server's certificate chain and host name| - -### BoltDB - -|field|default value|description| -|-|-|-| -|bolt.path|"/tmp/boltdb"|The database file path. If the file does not exists then it will be created automatically| -|bolt.timeout|0|The amount of time to wait to obtain a file lock. When set to zero it will wait indefinitely. This option is only available on Darwin and Linux| -|bolt.no_grow_sync|false|Sets DB.NoGrowSync flag before memory mapping the file| -|bolt.read_only|false|Open the database in read-only mode| -|bolt.mmap_flags|0|Set the DB.MmapFlags flag before memory mapping the file| -|bolt.initial_mmap_size|0|The initial mmap size of the database in bytes. If <= 0, the initial map size is 0. If the size is smaller than the previous database, it takes no effect| - -### etcd - -|field|default value|description| -|-|-|-| -|etcd.endpoints|"localhost:2379"|The etcd endpoint(s), multiple endpoints can be passed separated by comma.| -|etcd.dial_timeout|"2s"|The dial timeout duration passed into the client config.| -|etcd.cert_file|""|When using secure etcd, this should point to the crt file.| -|etcd.key_file|""|When using secure etcd, this should point to the pem file.| -|etcd.cacert_file|""|When using secure etcd, this should point to the ca file.| - -### DynamoDB - -|field|default value|description| -|-|-|-| -|dynamodb.tablename|"ycsb"|The database tablename| -|dynamodb.primarykey|"_key"|The table primary key fieldname| -|dynamodb.rc.units|10|Read request units throughput| -|dynamodb.wc.units|10|Write request units throughput| -|dynamodb.ensure.clean.table|true|On load mode ensure that the table is clean at the begining. In case of true and if the table previously exists it will be deleted and recreated| -|dynamodb.endpoint|""|Used endpoint for connection. If empty will use the default loaded configs| -|dynamodb.region|""|Used region for connection ( should match endpoint ). If empty will use the default loaded configs| -|dynamodb.consistent.reads|false|Reads on DynamoDB provide an eventually consistent read by default. If your benchmark/use-case requires a strongly consistent read, set this option to true| -|dynamodb.delete.after.run.stage|false|Detele the database table after the run stage| - - - -## TODO - -- [ ] Support more measurement, like HdrHistogram -- [ ] Add tests for generators diff --git a/go-ycsb/YCSB-LICENSE b/go-ycsb/YCSB-LICENSE deleted file mode 100644 index 7a4a3ea24..000000000 --- a/go-ycsb/YCSB-LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/go-ycsb/cmd/go-ycsb/client.go b/go-ycsb/cmd/go-ycsb/client.go deleted file mode 100644 index 22abb5d36..000000000 --- a/go-ycsb/cmd/go-ycsb/client.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "strconv" - "time" - - "github.com/pingcap/go-ycsb/pkg/client" - "github.com/pingcap/go-ycsb/pkg/measurement" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/spf13/cobra" -) - -func runClientCommandFunc(cmd *cobra.Command, args []string, doTransactions bool, command string) { - dbName := args[0] - - initialGlobal(dbName, func() { - doTransFlag := "true" - if !doTransactions { - doTransFlag = "false" - } - globalProps.Set(prop.DoTransactions, doTransFlag) - globalProps.Set(prop.Command, command) - - if cmd.Flags().Changed("threads") { - // We set the threadArg via command line. - globalProps.Set(prop.ThreadCount, strconv.Itoa(threadsArg)) - } - - if cmd.Flags().Changed("target") { - globalProps.Set(prop.Target, strconv.Itoa(targetArg)) - } - - if cmd.Flags().Changed("interval") { - globalProps.Set(prop.LogInterval, strconv.Itoa(reportInterval)) - } - }) - - fmt.Println("***************** properties *****************") - for key, value := range globalProps.Map() { - fmt.Printf("\"%s\"=\"%s\"\n", key, value) - } - fmt.Println("**********************************************") - - c := client.NewClient(globalProps, globalWorkload, globalDB) - start := time.Now() - c.Run(globalContext) - - fmt.Printf("Run finished, takes %s\n", time.Now().Sub(start)) - measurement.Output() -} - -func runLoadCommandFunc(cmd *cobra.Command, args []string) { - runClientCommandFunc(cmd, args, false, "load") -} - -func runTransCommandFunc(cmd *cobra.Command, args []string) { - runClientCommandFunc(cmd, args, true, "run") -} - -var ( - threadsArg int - targetArg int - reportInterval int -) - -func initClientCommand(m *cobra.Command) { - m.Flags().StringSliceVarP(&propertyFiles, "property_file", "P", nil, "Spefify a property file") - m.Flags().StringArrayVarP(&propertyValues, "prop", "p", nil, "Specify a property value with name=value") - m.Flags().StringVar(&tableName, "table", "", "Use the table name instead of the default \""+prop.TableNameDefault+"\"") - m.Flags().IntVar(&threadsArg, "threads", 1, "Execute using n threads - can also be specified as the \"threadcount\" property") - m.Flags().IntVar(&targetArg, "target", 0, "Attempt to do n operations per second (default: unlimited) - can also be specified as the \"target\" property") - m.Flags().IntVar(&reportInterval, "interval", 10, "Interval of outputting measurements in seconds") -} - -func newLoadCommand() *cobra.Command { - m := &cobra.Command{ - Use: "load db", - Short: "YCSB load benchmark", - Args: cobra.MinimumNArgs(1), - Run: runLoadCommandFunc, - } - - initClientCommand(m) - return m -} - -func newRunCommand() *cobra.Command { - m := &cobra.Command{ - Use: "run db", - Short: "YCSB run benchmark", - Args: cobra.MinimumNArgs(1), - Run: runTransCommandFunc, - } - - initClientCommand(m) - return m -} diff --git a/go-ycsb/cmd/go-ycsb/main.go b/go-ycsb/cmd/go-ycsb/main.go deleted file mode 100644 index 1bc3aa275..000000000 --- a/go-ycsb/cmd/go-ycsb/main.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - "log" - "net/http" - _ "net/http/pprof" - "os" - "os/signal" - "strings" - "syscall" - "time" - - "github.com/magiconair/properties" - - // Register workload - - "github.com/spf13/cobra" - - "github.com/pingcap/go-ycsb/pkg/client" - "github.com/pingcap/go-ycsb/pkg/measurement" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - _ "github.com/pingcap/go-ycsb/pkg/workload" - "github.com/pingcap/go-ycsb/pkg/ycsb" - - // Register basic database - _ "github.com/pingcap/go-ycsb/db/basic" - // Register MySQL database - _ "github.com/pingcap/go-ycsb/db/mysql" - // Register TiKV database - //_ "github.com/pingcap/go-ycsb/db/tikv" - // Register PostgreSQL database - _ "github.com/pingcap/go-ycsb/db/pg" - // Register Aerospike database - _ "github.com/pingcap/go-ycsb/db/aerospike" - // Register Badger database - //_ "github.com/pingcap/go-ycsb/db/badger" - // Register FoundationDB database - _ "github.com/pingcap/go-ycsb/db/foundationdb" - // Register RocksDB database - _ "github.com/pingcap/go-ycsb/db/rocksdb" - // Register Spanner database - //_ "github.com/pingcap/go-ycsb/db/spanner" - // Register pegasus database - _ "github.com/pingcap/go-ycsb/db/pegasus" - // Register sqlite database - _ "github.com/pingcap/go-ycsb/db/sqlite" - // Register cassandra database - _ "github.com/pingcap/go-ycsb/db/cassandra" - // Register mongodb database - _ "github.com/pingcap/go-ycsb/db/mongodb" - // Register redis database - _ "github.com/pingcap/go-ycsb/db/redis" - // Register boltdb database - _ "github.com/pingcap/go-ycsb/db/boltdb" - // Register minio - _ "github.com/pingcap/go-ycsb/db/minio" - // Register elastic - _ "github.com/pingcap/go-ycsb/db/elasticsearch" - // Register etcd - //_ "github.com/pingcap/go-ycsb/db/etcd" - // Register tinydb - _ "github.com/pingcap/go-ycsb/db/tinydb" - // Register dynamodb - _ "github.com/pingcap/go-ycsb/db/dynamodb" -) - -var ( - propertyFiles []string - propertyValues []string - dbName string - tableName string - - globalContext context.Context - globalCancel context.CancelFunc - - globalDB ycsb.DB - globalWorkload ycsb.Workload - globalProps *properties.Properties -) - -func initialGlobal(dbName string, onProperties func()) { - globalProps = properties.NewProperties() - if len(propertyFiles) > 0 { - globalProps = properties.MustLoadFiles(propertyFiles, properties.UTF8, false) - } - - for _, prop := range propertyValues { - seps := strings.SplitN(prop, "=", 2) - if len(seps) != 2 { - log.Fatalf("bad property: `%s`, expected format `name=value`", prop) - } - globalProps.Set(seps[0], seps[1]) - } - - if onProperties != nil { - onProperties() - } - - addr := globalProps.GetString(prop.DebugPprof, prop.DebugPprofDefault) - go func() { - http.ListenAndServe(addr, nil) - }() - - measurement.InitMeasure(globalProps) - - if len(tableName) == 0 { - tableName = globalProps.GetString(prop.TableName, prop.TableNameDefault) - } - - workloadName := globalProps.GetString(prop.Workload, "core") - workloadCreator := ycsb.GetWorkloadCreator(workloadName) - - var err error - if globalWorkload, err = workloadCreator.Create(globalProps); err != nil { - util.Fatalf("create workload %s failed %v", workloadName, err) - } - - dbCreator := ycsb.GetDBCreator(dbName) - if dbCreator == nil { - util.Fatalf("%s is not registered", dbName) - } - if globalDB, err = dbCreator.Create(globalProps); err != nil { - util.Fatalf("create db %s failed %v", dbName, err) - } - globalDB = client.DbWrapper{globalDB} -} - -func main() { - globalContext, globalCancel = context.WithCancel(context.Background()) - - sc := make(chan os.Signal, 1) - signal.Notify(sc, - syscall.SIGHUP, - syscall.SIGINT, - syscall.SIGTERM, - syscall.SIGQUIT) - - closeDone := make(chan struct{}, 1) - go func() { - sig := <-sc - fmt.Printf("\nGot signal [%v] to exit.\n", sig) - globalCancel() - - select { - case <-sc: - // send signal again, return directly - fmt.Printf("\nGot signal [%v] again to exit.\n", sig) - os.Exit(1) - case <-time.After(10 * time.Second): - fmt.Print("\nWait 10s for closed, force exit\n") - os.Exit(1) - case <-closeDone: - return - } - }() - - rootCmd := &cobra.Command{ - Use: "go-ycsb", - Short: "Go YCSB", - } - - rootCmd.AddCommand( - newShellCommand(), - newLoadCommand(), - newRunCommand(), - ) - - cobra.EnablePrefixMatching = true - - if err := rootCmd.Execute(); err != nil { - fmt.Println(rootCmd.UsageString()) - } - - globalCancel() - if globalDB != nil { - globalDB.Close() - } - - if globalWorkload != nil { - globalWorkload.Close() - } - - closeDone <- struct{}{} -} diff --git a/go-ycsb/cmd/go-ycsb/shell.go b/go-ycsb/cmd/go-ycsb/shell.go deleted file mode 100644 index ef1fe2a50..000000000 --- a/go-ycsb/cmd/go-ycsb/shell.go +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - "io" - "strconv" - "strings" - - "github.com/chzyer/readline" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - - "github.com/spf13/cobra" -) - -func newShellCommand() *cobra.Command { - m := &cobra.Command{ - Use: "shell db", - Short: "YCSB Command Line Client", - Args: cobra.MinimumNArgs(1), - Run: runShellCommandFunc, - } - m.Flags().StringSliceVarP(&propertyFiles, "property_file", "P", nil, "Spefify a property file") - m.Flags().StringSliceVarP(&propertyValues, "prop", "p", nil, "Specify a property value with name=value") - m.Flags().StringVar(&tableName, "table", "", "Use the table name instead of the default \""+prop.TableNameDefault+"\"") - return m -} - -var shellContext context.Context - -func runShellCommandFunc(cmd *cobra.Command, args []string) { - dbName := args[0] - initialGlobal(dbName, nil) - - shellContext = globalWorkload.InitThread(globalContext, 0, 1) - shellContext = globalDB.InitThread(shellContext, 0, 1) - - shellLoop() - - globalDB.CleanupThread(shellContext) - globalWorkload.CleanupThread(shellContext) -} - -func runShellCommand(args []string) { - cmd := &cobra.Command{ - Use: "shell", - Short: "YCSB shell command", - } - - cmd.SetArgs(args) - cmd.ParseFlags(args) - - cmd.AddCommand( - &cobra.Command{ - Use: "read key [field0 field1 field2 ...]", - Short: "Read a record", - Args: cobra.MinimumNArgs(1), - Run: runShellReadCommand, - DisableFlagsInUseLine: true, - }, - &cobra.Command{ - Use: "scan key recordcount [field0 field1 field2 ...]", - Short: "Scan starting at key", - Args: cobra.MinimumNArgs(2), - Run: runShellScanCommand, - DisableFlagsInUseLine: true, - }, - &cobra.Command{ - Use: "insert key field0=value0 [field1=value1 ...]", - Short: "Insert a record", - Args: cobra.MinimumNArgs(2), - Run: runShellInsertCommand, - DisableFlagsInUseLine: true, - }, - &cobra.Command{ - Use: "update key field0=value0 [field1=value1 ...]", - Short: "Update a record", - Args: cobra.MinimumNArgs(2), - Run: runShellUpdateCommand, - DisableFlagsInUseLine: true, - }, - &cobra.Command{ - Use: "delete key", - Short: "Delete a record", - Args: cobra.MinimumNArgs(1), - Run: runShellDeleteCommand, - DisableFlagsInUseLine: true, - }, - &cobra.Command{ - Use: "table [tablename]", - Short: "Get or [set] the name of the table", - Args: cobra.MaximumNArgs(1), - Run: runShellTableCommand, - DisableFlagsInUseLine: true, - }, - ) - - if err := cmd.Execute(); err != nil { - fmt.Println(cmd.UsageString()) - } -} - -func runShellReadCommand(cmd *cobra.Command, args []string) { - key := args[0] - fields := args[1:] - row, err := globalDB.Read(shellContext, tableName, key, fields) - if err != nil { - fmt.Printf("Read %s failed %v\n", key, err) - return - } - - if row == nil { - fmt.Printf("Read empty for %s\n", key) - return - } - - fmt.Printf("Read %s ok\n", key) - for key, value := range row { - fmt.Printf("%s=%q\n", key, value) - } -} - -func runShellScanCommand(cmd *cobra.Command, args []string) { - key := args[0] - recordCount, err := strconv.Atoi(args[1]) - if err != nil { - fmt.Printf("invalid record count %s for scan\n", args[1]) - return - } - fields := args[2:] - - rows, err := globalDB.Scan(shellContext, tableName, key, recordCount, fields) - if err != nil { - fmt.Printf("Scan from %s with %d failed %v\n", key, recordCount, err) - return - } - - if len(rows) == 0 { - fmt.Println("0 records") - return - } - - fmt.Println("--------------------------------") - for i, row := range rows { - fmt.Printf("Record %d\n", i+1) - for key, value := range row { - fmt.Printf("%s=%q\n", key, value) - } - } - fmt.Println("--------------------------------") -} - -func runShellInsertCommand(cmd *cobra.Command, args []string) { - key := args[0] - values := make(map[string][]byte, len(args[1:])) - - for _, arg := range args[1:] { - sep := strings.SplitN(arg, "=", 2) - values[sep[0]] = []byte(sep[1]) - } - - if err := globalDB.Insert(shellContext, tableName, key, values); err != nil { - fmt.Printf("Insert %s failed %v\n", key, err) - return - } - - fmt.Printf("Insert %s ok\n", key) -} - -func runShellUpdateCommand(cmd *cobra.Command, args []string) { - key := args[0] - values := make(map[string][]byte, len(args[1:])) - - for _, arg := range args[1:] { - sep := strings.SplitN(arg, "=", 2) - values[sep[0]] = []byte(sep[1]) - } - - if err := globalDB.Update(shellContext, tableName, key, values); err != nil { - fmt.Printf("Update %s failed %v\n", key, err) - return - } - - fmt.Printf("Update %s ok\n", key) -} - -func runShellDeleteCommand(cmd *cobra.Command, args []string) { - key := args[0] - if err := globalDB.Delete(shellContext, tableName, key); err != nil { - fmt.Printf("Delete %s failed %v\n", key, err) - return - } - - fmt.Printf("Delete %s ok\n", key) -} - -func runShellTableCommand(cmd *cobra.Command, args []string) { - if len(args) == 1 { - tableName = args[0] - } - fmt.Printf("Using table %s\n", tableName) -} - -func shellLoop() { - l, err := readline.NewEx(&readline.Config{ - Prompt: "\033[31m»\033[0m ", - HistoryFile: "/tmp/readline.tmp", - InterruptPrompt: "^C", - EOFPrompt: "^D", - HistorySearchFold: true, - }) - if err != nil { - util.Fatal(err) - } - defer l.Close() - - for { - line, err := l.Readline() - if err != nil { - if err == readline.ErrInterrupt { - return - } else if err == io.EOF { - return - } - continue - } - if line == "exit" { - return - } - args := strings.Split(strings.TrimSpace(line), " ") - runShellCommand(args) - } -} diff --git a/go-ycsb/db/aerospike/db.go b/go-ycsb/db/aerospike/db.go deleted file mode 100644 index cc8cd4e5f..000000000 --- a/go-ycsb/db/aerospike/db.go +++ /dev/null @@ -1,176 +0,0 @@ -package aerospike - -import ( - "context" - "errors" - - as "github.com/aerospike/aerospike-client-go" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -const ( - asNs = "aerospike.ns" - asHost = "aerospike.host" - asPort = "aerospike.port" -) - -type aerospikedb struct { - client *as.Client - ns string -} - -// Close closes the database layer. -func (adb *aerospikedb) Close() error { - adb.client.Close() - return nil -} - -// InitThread initializes the state associated to the goroutine worker. -// The Returned context will be passed to the following usage. -func (adb *aerospikedb) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { - return ctx -} - -// CleanupThread cleans up the state when the worker finished. -func (adb *aerospikedb) CleanupThread(ctx context.Context) { -} - -// Read reads a record from the database and returns a map of each field/value pair. -// table: The name of the table. -// key: The record key of the record to read. -// fileds: The list of fields to read, nil|empty for reading all. -func (adb *aerospikedb) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - asKey, err := as.NewKey(adb.ns, table, key) - record, err := adb.client.Get(nil, asKey) - if err != nil { - return nil, err - } - if record == nil { - return map[string][]byte{}, nil - } - res := make(map[string][]byte, len(record.Bins)) - var ok bool - for k, v := range record.Bins { - res[k], ok = v.([]byte) - if !ok { - return nil, errors.New("couldn't convert to byte array") - } - } - return res, nil -} - -// Scan scans records from the database. -// table: The name of the table. -// startKey: The first record key to read. -// count: The number of records to read. -// fields: The list of fields to read, nil|empty for reading all. -func (adb *aerospikedb) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - policy := as.NewScanPolicy() - policy.ConcurrentNodes = true - recordset, err := adb.client.ScanAll(policy, adb.ns, table) - if err != nil { - return nil, err - } - filter := make(map[string]bool, len(fields)) - for _, field := range fields { - filter[field] = true - } - scanRes := make([]map[string][]byte, 0) - var ok bool - nRead := 0 - for res := range recordset.Results() { - if res.Err != nil { - recordset.Close() - return nil, res.Err - } - vals := make(map[string][]byte, len(res.Record.Bins)) - for k, v := range res.Record.Bins { - if !filter[k] { - continue - } - vals[k], ok = v.([]byte) - if !ok { - return nil, errors.New("couldn't convert to byte array") - } - } - scanRes = append(scanRes, vals) - nRead++ - if nRead == count { - break - } - } - return scanRes, nil -} - -// Update updates a record in the database. Any field/value pairs will be written into the -// database or overwritten the existing values with the same field name. -// table: The name of the table. -// key: The record key of the record to update. -// values: A map of field/value pairs to update in the record. -func (adb *aerospikedb) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - asKey, err := as.NewKey(adb.ns, table, key) - if err != nil { - return err - } - record, err := adb.client.Get(nil, asKey) - if err != nil { - return err - } - bins := as.BinMap{} - var policy *as.WritePolicy - if record != nil { - bins = record.Bins - policy := as.NewWritePolicy(record.Generation, 0) - policy.GenerationPolicy = as.EXPECT_GEN_EQUAL - } - for k, v := range values { - bins[k] = v - } - return adb.client.Put(policy, asKey, bins) -} - -// Insert inserts a record in the database. Any field/value pairs will be written into the -// database. -// table: The name of the table. -// key: The record key of the record to insert. -// values: A map of field/value pairs to insert in the record. -func (adb *aerospikedb) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - asKey, err := as.NewKey(adb.ns, table, key) - if err != nil { - return err - } - bins := make([]*as.Bin, len(values)) - i := 0 - for k, v := range values { - bins[i] = as.NewBin(k, v) - i++ - } - return adb.client.PutBins(nil, asKey, bins...) -} - -// Delete deletes a record from the database. -// table: The name of the table. -// key: The record key of the record to delete. -func (adb *aerospikedb) Delete(ctx context.Context, table string, key string) error { - asKey, err := as.NewKey(adb.ns, table, key) - if err != nil { - return err - } - _, err = adb.client.Delete(nil, asKey) - return err -} - -type aerospikeCreator struct{} - -func (a aerospikeCreator) Create(p *properties.Properties) (ycsb.DB, error) { - adb := &aerospikedb{} - adb.ns = p.GetString(asNs, "test") - var err error - adb.client, err = as.NewClient(p.GetString(asHost, "localhost"), p.GetInt(asPort, 3000)) - return adb, err -} - -func init() { - ycsb.RegisterDBCreator("aerospike", aerospikeCreator{}) -} diff --git a/go-ycsb/db/badger/db.go b/go-ycsb/db/badger/db.go deleted file mode 100644 index 6599141af..000000000 --- a/go-ycsb/db/badger/db.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package badger - -import ( - "context" - "fmt" - "os" - - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/options" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// properties -const ( - badgerDir = "badger.dir" - badgerValueDir = "badger.valuedir" - badgerSyncWrites = "badger.sync_writes" - badgerNumVersionsToKeep = "badger.num_versions_to_keep" - badgerMaxTableSize = "badger.max_table_size" - badgerLevelSizeMultiplier = "badger.level_size_multiplier" - badgerMaxLevels = "badger.max_levels" - badgerValueThreshold = "badger.value_threshold" - badgerNumMemtables = "badger.num_memtables" - badgerNumLevelZeroTables = "badger.num_level0_tables" - badgerNumLevelZeroTablesStall = "badger.num_level0_tables_stall" - badgerLevelOneSize = "badger.level_one_size" - badgerValueLogFileSize = "badger.value_log_file_size" - badgerValueLogMaxEntries = "badger.value_log_max_entries" - badgerNumCompactors = "badger.num_compactors" - badgerDoNotCompact = "badger.do_not_compact" - badgerTableLoadingMode = "badger.table_loading_mode" - badgerValueLogLoadingMode = "badger.value_log_loading_mode" - // TODO: add more configurations -) - -type badgerCreator struct { -} - -type badgerDB struct { - p *properties.Properties - - db *badger.DB - - r *util.RowCodec - bufPool *util.BufPool -} - -type contextKey string - -const stateKey = contextKey("badgerDB") - -type badgerState struct { -} - -func (c badgerCreator) Create(p *properties.Properties) (ycsb.DB, error) { - opts := getOptions(p) - - if p.GetBool(prop.DropData, prop.DropDataDefault) { - os.RemoveAll(opts.Dir) - os.RemoveAll(opts.ValueDir) - } - - db, err := badger.Open(opts) - if err != nil { - return nil, err - } - - return &badgerDB{ - p: p, - db: db, - r: util.NewRowCodec(p), - bufPool: util.NewBufPool(), - }, nil -} - -func getOptions(p *properties.Properties) badger.Options { - opts := badger.DefaultOptions - opts.Dir = p.GetString(badgerDir, "/tmp/badger") - opts.ValueDir = p.GetString(badgerValueDir, opts.Dir) - - opts.SyncWrites = p.GetBool(badgerSyncWrites, false) - opts.NumVersionsToKeep = p.GetInt(badgerNumVersionsToKeep, 1) - opts.MaxTableSize = p.GetInt64(badgerMaxTableSize, 64<<20) - opts.LevelSizeMultiplier = p.GetInt(badgerLevelSizeMultiplier, 10) - opts.MaxLevels = p.GetInt(badgerMaxLevels, 7) - opts.ValueThreshold = p.GetInt(badgerValueThreshold, 32) - opts.NumMemtables = p.GetInt(badgerNumMemtables, 5) - opts.NumLevelZeroTables = p.GetInt(badgerNumLevelZeroTables, 5) - opts.NumLevelZeroTablesStall = p.GetInt(badgerNumLevelZeroTablesStall, 10) - opts.LevelOneSize = p.GetInt64(badgerLevelOneSize, 256<<20) - opts.ValueLogFileSize = p.GetInt64(badgerValueLogFileSize, 1<<30) - opts.ValueLogMaxEntries = uint32(p.GetUint64(badgerValueLogMaxEntries, 1000000)) - opts.NumCompactors = p.GetInt(badgerNumCompactors, 3) - opts.DoNotCompact = p.GetBool(badgerDoNotCompact, false) - if b := p.GetString(badgerTableLoadingMode, "LoadToRAM"); len(b) > 0 { - if b == "FileIO" { - opts.TableLoadingMode = options.FileIO - } else if b == "LoadToRAM" { - opts.TableLoadingMode = options.LoadToRAM - } else if b == "MemoryMap" { - opts.TableLoadingMode = options.MemoryMap - } - } - if b := p.GetString(badgerValueLogLoadingMode, "MemoryMap"); len(b) > 0 { - if b == "FileIO" { - opts.ValueLogLoadingMode = options.FileIO - } else if b == "LoadToRAM" { - opts.ValueLogLoadingMode = options.LoadToRAM - } else if b == "MemoryMap" { - opts.ValueLogLoadingMode = options.MemoryMap - } - } - - return opts -} - -func (db *badgerDB) Close() error { - return db.db.Close() -} - -func (db *badgerDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (db *badgerDB) CleanupThread(_ context.Context) { -} - -func (db *badgerDB) getRowKey(table string, key string) []byte { - return util.Slice(fmt.Sprintf("%s:%s", table, key)) -} - -func (db *badgerDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - var m map[string][]byte - err := db.db.View(func(txn *badger.Txn) error { - rowKey := db.getRowKey(table, key) - item, err := txn.Get(rowKey) - if err != nil { - return err - } - row, err := item.Value() - if err != nil { - return err - } - - m, err = db.r.Decode(row, fields) - return err - }) - - return m, err -} - -func (db *badgerDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - res := make([]map[string][]byte, count) - err := db.db.View(func(txn *badger.Txn) error { - rowStartKey := db.getRowKey(table, startKey) - it := txn.NewIterator(badger.DefaultIteratorOptions) - defer it.Close() - - i := 0 - for it.Seek(rowStartKey); it.Valid() && i < count; it.Next() { - item := it.Item() - value, err := item.ValueCopy(nil) - if err != nil { - return err - } - - m, err := db.r.Decode(value, fields) - if err != nil { - return err - } - - res[i] = m - i++ - } - - return nil - }) - - return res, err -} - -func (db *badgerDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - err := db.db.Update(func(txn *badger.Txn) error { - rowKey := db.getRowKey(table, key) - item, err := txn.Get(rowKey) - if err != nil { - return err - } - - value, err := item.Value() - if err != nil { - return err - } - - data, err := db.r.Decode(value, nil) - if err != nil { - return err - } - - for field, value := range values { - data[field] = value - } - - buf := db.bufPool.Get() - defer func() { - db.bufPool.Put(buf) - }() - - buf, err = db.r.Encode(buf, data) - if err != nil { - return err - } - return txn.Set(rowKey, buf) - }) - return err -} - -func (db *badgerDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - err := db.db.Update(func(txn *badger.Txn) error { - rowKey := db.getRowKey(table, key) - - buf := db.bufPool.Get() - defer func() { - db.bufPool.Put(buf) - }() - - buf, err := db.r.Encode(buf, values) - if err != nil { - return err - } - return txn.Set(rowKey, buf) - }) - - return err -} - -func (db *badgerDB) Delete(ctx context.Context, table string, key string) error { - err := db.db.Update(func(txn *badger.Txn) error { - return txn.Delete(db.getRowKey(table, key)) - }) - - return err -} - -func init() { - ycsb.RegisterDBCreator("badger", badgerCreator{}) -} diff --git a/go-ycsb/db/basic/db.go b/go-ycsb/db/basic/db.go deleted file mode 100644 index 9d1ec192f..000000000 --- a/go-ycsb/db/basic/db.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package basic - -import ( - "bytes" - "context" - "fmt" - "math/rand" - "time" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -const ( - simulateDelay = "basicdb.simulatedelay" - simulateDelayDefault = int64(0) - randomizeDelay = "basicdb.randomizedelay" - randomizeDelayDefault = true -) - -type contextKey string - -const stateKey = contextKey("basicDB") - -type basicState struct { - r *rand.Rand - - buf *bytes.Buffer -} - -// BasicDB just prints out the requested operations, instead of doing them against a database -type basicDB struct { - verbose bool - randomizeDelay bool - toDelay int64 -} - -func (db *basicDB) delay(ctx context.Context, state *basicState) { - if db.toDelay == 0 { - return - } - - r := state.r - delayTime := time.Duration(db.toDelay) * time.Millisecond - if db.randomizeDelay { - delayTime = time.Duration(r.Int63n(db.toDelay)) * time.Millisecond - if delayTime == 0 { - return - } - } - - select { - case <-time.After(delayTime): - case <-ctx.Done(): - } -} - -func (db *basicDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - state := new(basicState) - state.r = rand.New(rand.NewSource(time.Now().UnixNano())) - state.buf = new(bytes.Buffer) - - return context.WithValue(ctx, stateKey, state) -} - -func (db *basicDB) CleanupThread(_ context.Context) { - -} - -func (db *basicDB) Close() error { - return nil -} - -func (db *basicDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - state := ctx.Value(stateKey).(*basicState) - - db.delay(ctx, state) - - if !db.verbose { - return nil, nil - } - - buf := state.buf - s := fmt.Sprintf("READ %s %s [ ", table, key) - buf.WriteString(s) - - if len(fields) > 0 { - for _, f := range fields { - buf.WriteString(f) - buf.WriteByte(' ') - } - } else { - buf.WriteString(" ") - } - buf.WriteByte(']') - fmt.Println(buf.String()) - buf.Reset() - return nil, nil -} - -func (db *basicDB) BatchRead(ctx context.Context, table string, keys []string, fields []string) ([]map[string][]byte, error) { - panic("The basicDB has not implemented the batch operation") -} - -func (db *basicDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - state := ctx.Value(stateKey).(*basicState) - - db.delay(ctx, state) - - if !db.verbose { - return nil, nil - } - - buf := state.buf - s := fmt.Sprintf("SCAN %s %s %d [ ", table, startKey, count) - buf.WriteString(s) - - if len(fields) > 0 { - for _, f := range fields { - buf.WriteString(f) - buf.WriteByte(' ') - } - } else { - buf.WriteString(" ") - } - buf.WriteByte(']') - fmt.Println(buf.String()) - buf.Reset() - return nil, nil -} - -func (db *basicDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - state := ctx.Value(stateKey).(*basicState) - - db.delay(ctx, state) - - if !db.verbose { - return nil - } - - buf := state.buf - s := fmt.Sprintf("UPDATE %s %s [ ", table, key) - buf.WriteString(s) - - for key, value := range values { - buf.WriteString(key) - buf.WriteByte('=') - buf.Write(value) - buf.WriteByte(' ') - } - - buf.WriteByte(']') - fmt.Println(buf.String()) - buf.Reset() - return nil -} - -func (db *basicDB) BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) error { - panic("The basicDB has not implemented the batch operation") -} - -func (db *basicDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - state := ctx.Value(stateKey).(*basicState) - - db.delay(ctx, state) - - if !db.verbose { - return nil - } - - buf := state.buf - insertRecord(buf, table, key, values) - return nil -} - -func (db *basicDB) BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) error { - state := ctx.Value(stateKey).(*basicState) - - db.delay(ctx, state) - - if !db.verbose { - return nil - } - buf := state.buf - for i, key := range keys { - insertRecord(buf, table, key, values[i]) - } - return nil -} - -func insertRecord(buf *bytes.Buffer, table string, key string, values map[string][]byte) { - s := fmt.Sprintf("INSERT %s %s [ ", table, key) - buf.WriteString(s) - for valueKey, value := range values { - buf.WriteString(valueKey) - buf.WriteByte('=') - buf.Write(value) - buf.WriteByte(' ') - } - buf.WriteByte(']') - fmt.Println(buf.String()) - buf.Reset() -} - -func (db *basicDB) Delete(ctx context.Context, table string, key string) error { - state := ctx.Value(stateKey).(*basicState) - - db.delay(ctx, state) - if !db.verbose { - return nil - } - - buf := state.buf - s := fmt.Sprintf("DELETE %s %s", table, key) - buf.WriteString(s) - - fmt.Println(buf.String()) - buf.Reset() - return nil -} - -func (db *basicDB) BatchDelete(ctx context.Context, table string, keys []string) error { - panic("The basicDB has not implemented the batch operation") -} - -type basicDBCreator struct{} - -func (basicDBCreator) Create(p *properties.Properties) (ycsb.DB, error) { - db := new(basicDB) - - db.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) - db.randomizeDelay = p.GetBool(randomizeDelay, randomizeDelayDefault) - db.toDelay = p.GetInt64(simulateDelay, simulateDelayDefault) - - return db, nil -} - -func init() { - fmt.Println("basic ok") - ycsb.RegisterDBCreator("basic", basicDBCreator{}) -} diff --git a/go-ycsb/db/boltdb/db.go b/go-ycsb/db/boltdb/db.go deleted file mode 100644 index 278261226..000000000 --- a/go-ycsb/db/boltdb/db.go +++ /dev/null @@ -1,244 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package boltdb - -import ( - "context" - "fmt" - "os" - - "github.com/boltdb/bolt" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// properties -const ( - boltPath = "bolt.path" - boltTimeout = "bolt.timeout" - boltNoGrowSync = "bolt.no_grow_sync" - boltReadOnly = "bolt.read_only" - boltMmapFlags = "bolt.mmap_flags" - boltInitialMmapSize = "bolt.initial_mmap_size" -) - -type boltCreator struct { -} - -type boltOptions struct { - Path string - FileMode os.FileMode - DBOptions *bolt.Options -} - -type boltDB struct { - p *properties.Properties - - db *bolt.DB - - r *util.RowCodec - bufPool *util.BufPool -} - -func (c boltCreator) Create(p *properties.Properties) (ycsb.DB, error) { - opts := getOptions(p) - - if p.GetBool(prop.DropData, prop.DropDataDefault) { - os.RemoveAll(opts.Path) - } - - db, err := bolt.Open(opts.Path, opts.FileMode, opts.DBOptions) - if err != nil { - return nil, err - } - - return &boltDB{ - p: p, - db: db, - r: util.NewRowCodec(p), - bufPool: util.NewBufPool(), - }, nil -} - -func getOptions(p *properties.Properties) boltOptions { - path := p.GetString(boltPath, "/tmp/boltdb") - - opts := bolt.DefaultOptions - opts.Timeout = p.GetDuration(boltTimeout, 0) - opts.NoGrowSync = p.GetBool(boltNoGrowSync, false) - opts.ReadOnly = p.GetBool(boltReadOnly, false) - opts.MmapFlags = p.GetInt(boltMmapFlags, 0) - opts.InitialMmapSize = p.GetInt(boltInitialMmapSize, 0) - - return boltOptions{ - Path: path, - FileMode: 0600, - DBOptions: opts, - } -} - -func (db *boltDB) Close() error { - return db.db.Close() -} - -func (db *boltDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (db *boltDB) CleanupThread(_ context.Context) { -} - -func (db *boltDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - var m map[string][]byte - err := db.db.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(table)) - if bucket == nil { - return fmt.Errorf("table not found: %s", table) - } - - row := bucket.Get([]byte(key)) - if row == nil { - return fmt.Errorf("key not found: %s.%s", table, key) - } - - var err error - m, err = db.r.Decode(row, fields) - return err - }) - return m, err -} - -func (db *boltDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - res := make([]map[string][]byte, count) - err := db.db.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(table)) - if bucket == nil { - return fmt.Errorf("table not found: %s", table) - } - - cursor := bucket.Cursor() - key, value := cursor.Seek([]byte(startKey)) - for i := 0; key != nil && i < count; i++ { - m, err := db.r.Decode(value, fields) - if err != nil { - return err - } - - res[i] = m - key, value = cursor.Next() - } - - return nil - }) - return res, err -} - -func (db *boltDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - err := db.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(table)) - if bucket == nil { - return fmt.Errorf("table not found: %s", table) - } - - value := bucket.Get([]byte(key)) - if value == nil { - return fmt.Errorf("key not found: %s.%s", table, key) - } - - data, err := db.r.Decode(value, nil) - if err != nil { - return err - } - - for field, value := range values { - data[field] = value - } - - buf := db.bufPool.Get() - defer func() { - db.bufPool.Put(buf) - }() - - buf, err = db.r.Encode(buf, data) - if err != nil { - return err - } - - return bucket.Put([]byte(key), buf) - }) - return err -} - -func (db *boltDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - err := db.db.Update(func(tx *bolt.Tx) error { - bucket, err := tx.CreateBucketIfNotExists([]byte(table)) - if err != nil { - return err - } - - buf := db.bufPool.Get() - defer func() { - db.bufPool.Put(buf) - }() - - buf, err = db.r.Encode(buf, values) - if err != nil { - return err - } - - return bucket.Put([]byte(key), buf) - }) - return err -} - -func (db *boltDB) Delete(ctx context.Context, table string, key string) error { - err := db.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(table)) - if bucket == nil { - return nil - } - - err := bucket.Delete([]byte(key)) - if err != nil { - return err - } - - if bucket.Stats().KeyN == 0 { - _ = tx.DeleteBucket([]byte(table)) - } - return nil - }) - return err -} - -func init() { - ycsb.RegisterDBCreator("boltdb", boltCreator{}) -} diff --git a/go-ycsb/db/cassandra/db.go b/go-ycsb/db/cassandra/db.go deleted file mode 100644 index cc9cd6352..000000000 --- a/go-ycsb/db/cassandra/db.go +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package cassandra - -import ( - "bytes" - "context" - "fmt" - "strings" - "time" - - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - - "github.com/gocql/gocql" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// cassandra properties -const ( - cassandraCluster = "cassandra.cluster" - cassandraKeyspace = "cassandra.keyspace" - cassandraConnections = "cassandra.connections" - cassandraUsername = "cassandra.username" - cassandraPassword = "cassandra.password" - - cassandraUsernameDefault = "cassandra" - cassandraPasswordDefault = "cassandra" - cassandraClusterDefault = "127.0.0.1:9042" - cassandraKeyspaceDefault = "test" - cassandraConnectionsDefault = 2 // refer to https://github.com/gocql/gocql/blob/master/cluster.go#L52 -) - -type cassandraCreator struct { -} - -type cassandraDB struct { - p *properties.Properties - session *gocql.Session - verbose bool - - bufPool *util.BufPool - keySpace string - - fieldNames []string -} - -type contextKey string - -const stateKey = contextKey("cassandraDB") - -type cassandraState struct { -} - -func (c cassandraCreator) Create(p *properties.Properties) (ycsb.DB, error) { - d := new(cassandraDB) - d.p = p - - hosts := strings.Split(p.GetString(cassandraCluster, cassandraClusterDefault), ",") - - cluster := gocql.NewCluster(hosts...) - cluster.Keyspace = p.GetString(cassandraKeyspace, cassandraKeyspaceDefault) - d.keySpace = cluster.Keyspace - - cluster.NumConns = p.GetInt(cassandraConnections, cassandraConnectionsDefault) - cluster.Timeout = 30 * time.Second - cluster.Consistency = gocql.Quorum - - username := p.GetString(cassandraUsername, cassandraUsernameDefault) - password := p.GetString(cassandraPassword, cassandraPasswordDefault) - cluster.Authenticator = gocql.PasswordAuthenticator{Username: username, Password: password} - - session, err := cluster.CreateSession() - if err != nil { - return nil, err - } - - d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) - d.session = session - - d.bufPool = util.NewBufPool() - - if err := d.createTable(); err != nil { - return nil, err - } - - return d, nil -} - -func (db *cassandraDB) createTable() error { - tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) - - if db.p.GetBool(prop.DropData, prop.DropDataDefault) { - if err := db.session.Query(fmt.Sprintf("DROP TABLE IF EXISTS %s.%s", db.keySpace, tableName)).Exec(); err != nil { - return err - } - } - - fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) - - db.fieldNames = make([]string, fieldCount) - for i := int64(0); i < fieldCount; i++ { - db.fieldNames[i] = fmt.Sprintf("field%d", i) - } - - buf := new(bytes.Buffer) - s := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s.%s (YCSB_KEY VARCHAR PRIMARY KEY", db.keySpace, tableName) - buf.WriteString(s) - - for i := int64(0); i < fieldCount; i++ { - buf.WriteString(fmt.Sprintf(", FIELD%d VARCHAR", i)) - } - - buf.WriteString(");") - - if db.verbose { - fmt.Println(buf.String()) - } - - err := db.session.Query(buf.String()).Exec() - return err -} - -func (db *cassandraDB) Close() error { - if db.session == nil { - return nil - } - - db.session.Close() - return nil -} - -func (db *cassandraDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (db *cassandraDB) CleanupThread(_ctx context.Context) { - -} - -func (db *cassandraDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - var query string - if len(fields) == 0 { - fields = db.fieldNames - } - - query = fmt.Sprintf(`SELECT %s FROM %s.%s WHERE YCSB_KEY = ?`, strings.Join(fields, ","), db.keySpace, table) - - if db.verbose { - fmt.Printf("%s\n", query) - } - - m := make(map[string][]byte, len(fields)) - dest := make([]interface{}, len(fields)) - for i := 0; i < len(fields); i++ { - v := new([]byte) - dest[i] = v - } - - err := db.session.Query(query, key).WithContext(ctx).Scan(dest...) - if err == gocql.ErrNotFound { - return nil, nil - } else if err != nil { - return nil, err - } - - for i, v := range dest { - m[fields[i]] = *v.(*[]byte) - } - - return m, nil -} - -func (db *cassandraDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - return nil, fmt.Errorf("scan is not supported") -} - -func (db *cassandraDB) execQuery(ctx context.Context, query string, args ...interface{}) error { - if db.verbose { - fmt.Printf("%s %v\n", query, args) - } - - err := db.session.Query(query, args...).WithContext(ctx).Exec() - return err -} - -func (db *cassandraDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - buf := bytes.NewBuffer(db.bufPool.Get()) - defer func() { - db.bufPool.Put(buf.Bytes()) - }() - - buf.WriteString("UPDATE ") - buf.WriteString(fmt.Sprintf("%s.%s", db.keySpace, table)) - buf.WriteString(" SET ") - firstField := true - pairs := util.NewFieldPairs(values) - args := make([]interface{}, 0, len(values)+1) - for _, p := range pairs { - if firstField { - firstField = false - } else { - buf.WriteString(", ") - } - - buf.WriteString(p.Field) - buf.WriteString(`= ?`) - args = append(args, p.Value) - } - buf.WriteString(" WHERE YCSB_KEY = ?") - - args = append(args, key) - - return db.execQuery(ctx, buf.String(), args...) -} - -func (db *cassandraDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - args := make([]interface{}, 0, 1+len(values)) - args = append(args, key) - - buf := bytes.NewBuffer(db.bufPool.Get()) - defer func() { - db.bufPool.Put(buf.Bytes()) - }() - - buf.WriteString("INSERT INTO ") - buf.WriteString(fmt.Sprintf("%s.%s", db.keySpace, table)) - buf.WriteString(" (YCSB_KEY") - - pairs := util.NewFieldPairs(values) - for _, p := range pairs { - args = append(args, p.Value) - buf.WriteString(" ,") - buf.WriteString(p.Field) - } - buf.WriteString(") VALUES (?") - - for i := 0; i < len(pairs); i++ { - buf.WriteString(" ,?") - } - - buf.WriteByte(')') - - return db.execQuery(ctx, buf.String(), args...) -} - -func (db *cassandraDB) Delete(ctx context.Context, table string, key string) error { - query := fmt.Sprintf(`DELETE FROM %s.%s WHERE YCSB_KEY = ?`, db.keySpace, table) - - return db.execQuery(ctx, query, key) -} - -func init() { - ycsb.RegisterDBCreator("cassandra", cassandraCreator{}) - ycsb.RegisterDBCreator("scylla", cassandraCreator{}) -} diff --git a/go-ycsb/db/dynamodb/db.go b/go-ycsb/db/dynamodb/db.go deleted file mode 100644 index 16e604bc2..000000000 --- a/go-ycsb/db/dynamodb/db.go +++ /dev/null @@ -1,295 +0,0 @@ -package dynamodb - -import ( - "context" - "errors" - "fmt" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/ycsb" - "log" - "strings" - "time" -) - -type dynamodbWrapper struct { - client *dynamodb.Client - tablename *string - primarykey string - primarykeyPtr *string - readCapacityUnits int64 - writeCapacityUnits int64 - consistentRead bool - deleteAfterRun bool - command string -} - -func (r *dynamodbWrapper) Close() error { - var err error = nil - if strings.Compare("run", r.command) == 0 { - log.Printf("Ensuring that the table is deleted after the run stage...\n") - if r.deleteAfterRun { - err = r.deleteTable() - if err != nil { - log.Printf("Couldn't delete table after run. Here's why: %v\n", err) - } - } - } - return err -} - -func (r *dynamodbWrapper) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (r *dynamodbWrapper) CleanupThread(_ context.Context) { -} - -func (r *dynamodbWrapper) Read(ctx context.Context, table string, key string, fields []string) (data map[string][]byte, err error) { - data = make(map[string][]byte, len(fields)) - - response, err := r.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - Key: r.GetKey(key), - TableName: r.tablename, - ConsistentRead: aws.Bool(r.consistentRead), - }) - if err != nil { - log.Printf("Couldn't get info about %v. Here's why: %v\n", key, err) - } else { - err = attributevalue.UnmarshalMap(response.Item, &data) - if err != nil { - log.Printf("Couldn't unmarshal response. Here's why: %v\n", err) - } - } - return - -} - -// GetKey returns the composite primary key of the document in a format that can be -// sent to DynamoDB. -func (r *dynamodbWrapper) GetKey(key string) map[string]types.AttributeValue { - return map[string]types.AttributeValue{ - r.primarykey: &types.AttributeValueMemberB{Value: []byte(key)}, - } -} - -func (r *dynamodbWrapper) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - return nil, fmt.Errorf("scan is not supported") -} - -func (r *dynamodbWrapper) Update(ctx context.Context, table string, key string, values map[string][]byte) (err error) { - var upd = expression.UpdateBuilder{} - for name, value := range values { - upd = upd.Set(expression.Name(name), expression.Value(&types.AttributeValueMemberB{Value: value})) - } - expr, err := expression.NewBuilder().WithUpdate(upd).Build() - - _, err = r.client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - Key: r.GetKey(key), - TableName: r.tablename, - UpdateExpression: expr.Update(), - ExpressionAttributeNames: expr.Names(), - ExpressionAttributeValues: expr.Values(), - }) - if err != nil { - log.Printf("Couldn't update item to table. Here's why: %v\nUpdateExpression:%s\nExpressionAttributeNames:%s\n", err, *expr.Update(), expr.Names()) - } - return -} - -func (r *dynamodbWrapper) Insert(ctx context.Context, table string, key string, values map[string][]byte) (err error) { - values[r.primarykey] = []byte(key) - item, err := attributevalue.MarshalMap(values) - if err != nil { - panic(err) - } - _, err = r.client.PutItem(context.TODO(), - &dynamodb.PutItemInput{ - TableName: r.tablename, Item: item, - }) - if err != nil { - log.Printf("Couldn't add item to table. Here's why: %v\n", err) - } - return -} - -func (r *dynamodbWrapper) Delete(ctx context.Context, table string, key string) error { - _, err := r.client.DeleteItem(context.TODO(), &dynamodb.DeleteItemInput{ - TableName: r.tablename, - Key: r.GetKey(key), - }) - return err -} - -type dynamoDbCreator struct{} - -// TableExists determines whether a DynamoDB table exists. -func (r *dynamodbWrapper) tableExists() (bool, error) { - exists := true - _, err := r.client.DescribeTable( - context.TODO(), &dynamodb.DescribeTableInput{TableName: r.tablename}, - ) - if err != nil { - var notFoundEx *types.ResourceNotFoundException - if errors.As(err, ¬FoundEx) { - log.Printf("Table %v does not exist.\n", *r.tablename) - err = nil - } else { - log.Printf("Couldn't determine existence of table %v. Here's why: %v\n", *r.tablename, err) - } - exists = false - } - return exists, err -} - -// This function uses NewTableExistsWaiter to wait for the table to be created by -// DynamoDB before it returns. -func (r *dynamodbWrapper) createTable() (*types.TableDescription, error) { - var tableDesc *types.TableDescription - table, err := r.client.CreateTable(context.TODO(), &dynamodb.CreateTableInput{ - AttributeDefinitions: []types.AttributeDefinition{{ - AttributeName: r.primarykeyPtr, - AttributeType: types.ScalarAttributeTypeB, - }}, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: r.primarykeyPtr, - KeyType: types.KeyTypeHash, - }, - }, - TableName: r.tablename, - ProvisionedThroughput: &types.ProvisionedThroughput{ - ReadCapacityUnits: aws.Int64(r.readCapacityUnits), - WriteCapacityUnits: aws.Int64(r.writeCapacityUnits), - }, - }) - if err != nil { - log.Printf("Couldn't create table %v. Here's why: %v\n", *r.tablename, err) - } else { - log.Printf("Waiting for table to be available.\n") - waiter := dynamodb.NewTableExistsWaiter(r.client) - err = waiter.Wait(context.TODO(), &dynamodb.DescribeTableInput{ - TableName: r.tablename}, 5*time.Minute) - if err != nil { - log.Printf("Wait for table exists failed. Here's why: %v\n", err) - } - tableDesc = table.TableDescription - } - return tableDesc, err -} - -func (r dynamoDbCreator) Create(p *properties.Properties) (ycsb.DB, error) { - rds := &dynamodbWrapper{} - - rds.tablename = aws.String(p.GetString(tablename, tablenameDefault)) - // other than the primary key, you do not need to define - // any extra attributes or data types when you create a table. - rds.primarykey = p.GetString(primaryKeyFieldName, primaryKeyFieldNameDefault) - rds.primarykeyPtr = aws.String(rds.primarykey) - rds.readCapacityUnits = p.GetInt64(readCapacityUnitsFieldName, readCapacityUnitsFieldNameDefault) - rds.writeCapacityUnits = p.GetInt64(writeCapacityUnitsFieldName, writeCapacityUnitsFieldNameDefault) - rds.consistentRead = p.GetBool(consistentReadFieldName, consistentReadFieldNameDefault) - rds.deleteAfterRun = p.GetBool(deleteTableAfterRunFieldName, deleteTableAfterRunFieldNameDefault) - endpoint := p.GetString(endpointField, endpointFieldDefault) - region := p.GetString(regionField, regionFieldDefault) - rds.command, _ = p.Get(prop.Command) - var err error = nil - var cfg aws.Config - if strings.Contains(endpoint, "localhost") && strings.Compare(region, "localhost") != 0 { - log.Printf("given you're using dynamodb local endpoint you need to specify -p %s='localhost'. Ignoring %s and enforcing -p %s='localhost'\n", regionField, region, regionField) - region = "localhost" - } - if strings.Compare(endpoint, endpointFieldDefault) == 0 { - if strings.Compare(region, regionFieldDefault) != 0 { - // if endpoint is default but we have region - cfg, err = config.LoadDefaultConfig(context.TODO(), config.WithRegion(region)) - } else { - // if both endpoint and region are default - cfg, err = config.LoadDefaultConfig(context.TODO()) - } - } else { - cfg, err = config.LoadDefaultConfig(context.TODO(), - config.WithRegion(region), - config.WithEndpointResolver(aws.EndpointResolverFunc( - func(service, region string) (aws.Endpoint, error) { - return aws.Endpoint{URL: endpoint, SigningRegion: region}, nil - })), - ) - } - if err != nil { - log.Fatalf("unable to load SDK config, %v", err) - } - // Create DynamoDB client - rds.client = dynamodb.NewFromConfig(cfg) - exists, err := rds.tableExists() - - if strings.Compare("load", rds.command) == 0 { - if !exists { - _, err = rds.createTable() - } else { - ensureCleanTable := p.GetBool(ensureCleanTableFieldName, ensureCleanTableFieldNameDefault) - if ensureCleanTable { - log.Printf("dynamo table named %s already existed. Deleting it...\n", *rds.tablename) - _ = rds.deleteTable() - _, err = rds.createTable() - } else { - log.Printf("dynamo table named %s already existed. Skipping table creation.\n", *rds.tablename) - } - } - } else { - if !exists { - log.Fatalf("dynamo table named %s does not exist. You need to run the load stage previous than '%s'...\n", *rds.tablename, "run") - } - } - return rds, err -} - -func (rds *dynamodbWrapper) deleteTable() error { - _, err := rds.client.DeleteTable(context.TODO(), &dynamodb.DeleteTableInput{ - TableName: rds.tablename, - }) - if err != nil { - log.Fatalf("Unable to delete table, %v", err) - } - waiter := dynamodb.NewTableNotExistsWaiter(rds.client) - err = waiter.Wait(context.TODO(), &dynamodb.DescribeTableInput{ - TableName: rds.tablename}, 5*time.Minute) - if err != nil { - log.Fatalf("Wait for table deletion failed. Here's why: %v", err) - } - return err -} - -const ( - tablename = "dynamodb.tablename" - tablenameDefault = "ycsb" - primaryKeyFieldName = "dynamodb.primarykey" - primaryKeyFieldNameDefault = "_key" - readCapacityUnitsFieldName = "dynamodb.rc.units" - readCapacityUnitsFieldNameDefault = 10 - writeCapacityUnitsFieldName = "dynamodb.wc.units" - writeCapacityUnitsFieldNameDefault = 10 - ensureCleanTableFieldName = "dynamodb.ensure.clean.table" - ensureCleanTableFieldNameDefault = true - endpointField = "dynamodb.endpoint" - endpointFieldDefault = "" - regionField = "dynamodb.region" - regionFieldDefault = "" - // GetItem provides an eventually consistent read by default. - // If your application requires a strongly consistent read, set ConsistentRead to true. - // Although a strongly consistent read might take more time than an eventually consistent read, it always returns the last updated value. - consistentReadFieldName = "dynamodb.consistent.reads" - consistentReadFieldNameDefault = false - deleteTableAfterRunFieldName = "dynamodb.delete.after.run.stage" - deleteTableAfterRunFieldNameDefault = false -) - -func init() { - ycsb.RegisterDBCreator("dynamodb", dynamoDbCreator{}) -} diff --git a/go-ycsb/db/elasticsearch/db.go b/go-ycsb/db/elasticsearch/db.go deleted file mode 100644 index a896b8494..000000000 --- a/go-ycsb/db/elasticsearch/db.go +++ /dev/null @@ -1,346 +0,0 @@ -package elastic - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "fmt" - "github.com/cenkalti/backoff/v4" - "github.com/elastic/go-elasticsearch/v8" - "github.com/elastic/go-elasticsearch/v8/esapi" - "github.com/elastic/go-elasticsearch/v8/esutil" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/ycsb" - "net" - "net/http" - "runtime" - "strings" - "time" -) - -const ( - elasticUrl = "es.hosts.list" - elasticUrlDefault = "http://127.0.0.1:9200" - elasticInsecureSSLProp = "es.insecure.ssl" - elasticInsecureSSLPropDefault = false - elasticExitOnIndexCreateFailureProp = "es.exit.on.index.create.fail" - elasticExitOnIndexCreateFailurePropDefault = false - elasticShardCountProp = "es.number_of_shards" - elasticShardCountPropDefault = 1 - elasticReplicaCountProp = "es.number_of_replicas" - elasticReplicaCountPropDefault = 0 - elasticUsername = "es.username" - elasticUsernameDefault = "elastic" - elasticPassword = "es.password" - elasticPasswordPropDefault = "" - elasticFlushInterval = "es.flush_interval" - bulkIndexerNumberOfWorkers = "es.bulk.num_workers" - elasticMaxRetriesProp = "es.max_retires" - elasticMaxRetriesPropDefault = 10 - bulkIndexerFlushBytesProp = "es.bulk.flush_bytes" - bulkIndexerFlushBytesDefault = 5e+6 - bulkIndexerFlushIntervalSecondsProp = "es.bulk.flush_interval_secs" - bulkIndexerFlushIntervalSecondsPropDefault = 30 - elasticIndexNameDefault = "ycsb" - elasticIndexName = "es.index" -) - -type elastic struct { - cli *elasticsearch.Client - bi esutil.BulkIndexer - indexName string - verbose bool -} - -func (m *elastic) Close() error { - return nil -} - -func (m *elastic) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { - return ctx -} - -func (m *elastic) CleanupThread(ctx context.Context) { - m.bi.Close(context.Background()) -} - -// Read a document. -func (m *elastic) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - res, err := m.cli.Get(m.indexName, key) - if err != nil { - if m.verbose { - fmt.Println("Cannot read document %d: %s", key, err) - } - return nil, err - } - var r map[string][]byte - json.NewDecoder(res.Body).Decode(&r) - return r, nil -} - -// Scan documents. -func (m *elastic) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - return nil, fmt.Errorf("scan is not supported") - -} - -// Insert a document. -func (m *elastic) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - data, err := json.Marshal(values) - if err != nil { - if m.verbose { - fmt.Println("Cannot encode document %d: %s", key, err) - } - return err - } - // Add an item to the BulkIndexer - err = m.bi.Add( - context.Background(), - esutil.BulkIndexerItem{ - // Action field configures the operation to perform (index, create, delete, update) - Action: "index", - - // DocumentID is the (optional) document ID - DocumentID: key, - - // Body is an `io.Reader` with the payload - Body: bytes.NewReader(data), - - // OnSuccess is called for each successful operation - OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) { - }, - // OnFailure is called for each failed operation - OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { - if err != nil { - fmt.Printf("ERROR BULK INSERT: %s", err) - } else { - fmt.Printf("ERROR BULK INSERT: %s: %s", res.Error.Type, res.Error.Reason) - } - }, - }, - ) - if err != nil { - if m.verbose { - fmt.Println("Unexpected error while bulk inserting: %s", err) - } - return err - } - return nil -} - -// Update a document. -func (m *elastic) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - data, err := json.Marshal(values) - if err != nil { - if m.verbose { - fmt.Println("Cannot encode document %d: %s", key, err) - } - return err - } - // Add an item to the BulkIndexer - err = m.bi.Add( - context.Background(), - esutil.BulkIndexerItem{ - // Action field configures the operation to perform (index, create, delete, update) - Action: "update", - - // DocumentID is the (optional) document ID - DocumentID: key, - - // Body is an `io.Reader` with the payload - Body: bytes.NewReader(data), - - // OnSuccess is called for each successful operation - OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) { - }, - // OnFailure is called for each failed operation - OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { - if err != nil { - fmt.Printf("ERROR BULK UPDATING: %s", err) - } else { - fmt.Printf("ERROR BULK UPDATING: %s: %s", res.Error.Type, res.Error.Reason) - } - }, - }, - ) - if err != nil { - if m.verbose { - fmt.Println("Unexpected error while bulk updating: %s", err) - } - return err - } - return nil -} - -// Delete a document. -func (m *elastic) Delete(ctx context.Context, table string, key string) error { - // Add an delete to the BulkIndexer - err := m.bi.Add( - context.Background(), - esutil.BulkIndexerItem{ - // Action field configures the operation to perform (index, create, delete, update) - Action: "delete", - - // DocumentID is the (optional) document ID - DocumentID: key, - - // OnSuccess is called for each successful operation - OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) { - }, - // OnFailure is called for each failed operation - OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { - if err != nil { - fmt.Printf("ERROR BULK UPDATING: %s", err) - } else { - fmt.Printf("ERROR BULK UPDATING: %s: %s", res.Error.Type, res.Error.Reason) - } - }, - }, - ) - if err != nil { - if m.verbose { - fmt.Println("Unexpected error while bulk deleting: %s", err) - } - return err - } - return nil -} - -type elasticCreator struct { -} - -func (c elasticCreator) Create(p *properties.Properties) (ycsb.DB, error) { - defaultNumCpus := runtime.NumCPU() - bulkIndexerRefresh := "false" - batchSize := p.GetInt(prop.BatchSize, prop.DefaultBatchSize) - if batchSize <= 1 { - bulkIndexerRefresh = "wait_for" - fmt.Printf("Bulk API is disable given the property `%s`=1. For optimal indexing speed please increase this value property\n", prop.BatchSize) - } - bulkIndexerNumCpus := p.GetInt(bulkIndexerNumberOfWorkers, defaultNumCpus) - elasticMaxRetries := p.GetInt(elasticMaxRetriesProp, elasticMaxRetriesPropDefault) - bulkIndexerFlushBytes := p.GetInt(bulkIndexerFlushBytesProp, bulkIndexerFlushBytesDefault) - flushIntervalSeconds := p.GetInt(bulkIndexerFlushIntervalSecondsProp, bulkIndexerFlushIntervalSecondsPropDefault) - elasticReplicaCount := p.GetInt(elasticReplicaCountProp, elasticReplicaCountPropDefault) - elasticShardCount := p.GetInt(elasticShardCountProp, elasticShardCountPropDefault) - - addressesS := p.GetString(elasticUrl, elasticUrlDefault) - insecureSSL := p.GetBool(elasticInsecureSSLProp, elasticInsecureSSLPropDefault) - failOnCreate := p.GetBool(elasticExitOnIndexCreateFailureProp, elasticExitOnIndexCreateFailurePropDefault) - esUser := p.GetString(elasticUsername, elasticUsernameDefault) - esPass := p.GetString(elasticPassword, elasticPasswordPropDefault) - command, _ := p.Get(prop.Command) - verbose := p.GetBool(prop.Verbose, prop.VerboseDefault) - iname := p.GetString(elasticIndexName, elasticIndexNameDefault) - addresses := strings.Split(addressesS, ",") - - retryBackoff := backoff.NewExponentialBackOff() - // - //// Get the SystemCertPool, continue with an empty pool on error - //rootCAs, _ := x509.SystemCertPool() - //if rootCAs == nil { - // rootCAs = x509.NewCertPool() - //} - - cfg := elasticsearch.Config{ - Addresses: addresses, - // Retry on 429 TooManyRequests statuses - RetryOnStatus: []int{502, 503, 504, 429}, - - // Configure the backoff function - RetryBackoff: func(i int) time.Duration { - if i == 1 { - retryBackoff.Reset() - } - return retryBackoff.NextBackOff() - }, - MaxRetries: elasticMaxRetries, - Username: esUser, - Password: esPass, - // Transport / SSL - Transport: &http.Transport{ - MaxIdleConnsPerHost: 10, - ResponseHeaderTimeout: time.Second, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: insecureSSL, - }, - }, - } - es, err := elasticsearch.NewClient(cfg) - if err != nil { - fmt.Println(fmt.Sprintf("Error creating the elastic client: %s", err)) - return nil, err - } - fmt.Println("Connected to elastic!") - if verbose { - fmt.Println(es.Info()) - } - // Create the BulkIndexer - var flushIntervalTime = flushIntervalSeconds * int(time.Second) - - bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ - Index: iname, // The default index name - Client: es, // The Elasticsearch client - NumWorkers: bulkIndexerNumCpus, // The number of worker goroutines - FlushBytes: bulkIndexerFlushBytes, // The flush threshold in bytes - FlushInterval: time.Duration(flushIntervalTime), // The periodic flush interval - // If true, Elasticsearch refreshes the affected - // shards to make this operation visible to search - // if wait_for then wait for a refresh to make this operation visible to search, - // if false do nothing with refreshes. Valid values: true, false, wait_for. Default: false. - Refresh: bulkIndexerRefresh, - }) - if err != nil { - fmt.Println("Error creating the elastic indexer: %s", err) - return nil, err - } - - if strings.Compare("load", command) == 0 { - fmt.Println("Ensuring that if the index exists we recreat it") - // Re-create the index - var res *esapi.Response - if res, err = es.Indices.Delete([]string{iname}, es.Indices.Delete.WithIgnoreUnavailable(true)); err != nil || res.IsError() { - fmt.Println(fmt.Sprintf("Cannot delete index: %s", err)) - return nil, err - } - res.Body.Close() - - // Define index mapping. - mapping := map[string]interface{}{"settings": map[string]interface{}{"index": map[string]interface{}{"number_of_shards": elasticShardCount, "number_of_replicas": elasticReplicaCount}}} - data, err := json.Marshal(mapping) - if err != nil { - if verbose { - fmt.Println(fmt.Sprintf("Cannot encode index mapping %v: %s", mapping, err)) - } - return nil, err - } - res, err = es.Indices.Create(iname, es.Indices.Create.WithBody(strings.NewReader(string(data)))) - if err != nil && failOnCreate { - fmt.Println(fmt.Sprintf("Cannot create index: %s", err)) - return nil, err - } - if res.IsError() && failOnCreate { - fmt.Println(fmt.Sprintf("Cannot create index: %s", res)) - return nil, fmt.Errorf("Cannot create index: %s", res) - } - res.Body.Close() - } - - m := &elastic{ - cli: es, - bi: bi, - indexName: iname, - verbose: verbose, - } - return m, nil -} - -func init() { - ycsb.RegisterDBCreator("elastic", elasticCreator{}) -} diff --git a/go-ycsb/db/etcd/db.go b/go-ycsb/db/etcd/db.go deleted file mode 100644 index 10580a849..000000000 --- a/go-ycsb/db/etcd/db.go +++ /dev/null @@ -1,163 +0,0 @@ -package etcd - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "fmt" - "strings" - "time" - - clientv3 "go.etcd.io/etcd/client/v3" - - "github.com/magiconair/properties" - "go.etcd.io/etcd/client/pkg/v3/transport" - - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// properties -const ( - etcdEndpoints = "etcd.endpoints" - etcdDialTimeout = "etcd.dial_timeout" - etcdCertFile = "etcd.cert_file" - etcdKeyFile = "etcd.key_file" - etcdCaFile = "etcd.cacert_file" -) - -type etcdCreator struct{} - -type etcdDB struct { - p *properties.Properties - client *clientv3.Client -} - -func init() { - ycsb.RegisterDBCreator("etcd", etcdCreator{}) -} - -func (c etcdCreator) Create(p *properties.Properties) (ycsb.DB, error) { - cfg, err := getClientConfig(p) - if err != nil { - return nil, err - } - - client, err := clientv3.New(*cfg) - if err != nil { - return nil, err - } - - return &etcdDB{ - p: p, - client: client, - }, nil -} - -func getClientConfig(p *properties.Properties) (*clientv3.Config, error) { - endpoints := p.GetString(etcdEndpoints, "localhost:2379") - dialTimeout := p.GetDuration(etcdDialTimeout, 2*time.Second) - - var tlsConfig *tls.Config - if strings.Contains(endpoints, "https") { - tlsInfo := transport.TLSInfo{ - CertFile: p.MustGetString(etcdCertFile), - KeyFile: p.MustGetString(etcdKeyFile), - TrustedCAFile: p.MustGetString(etcdCaFile), - } - c, err := tlsInfo.ClientConfig() - if err != nil { - return nil, err - } - tlsConfig = c - } - - return &clientv3.Config{ - Endpoints: strings.Split(endpoints, ","), - DialTimeout: dialTimeout, - TLS: tlsConfig, - }, nil -} - -func (db *etcdDB) Close() error { - return db.client.Close() -} - -func (db *etcdDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (db *etcdDB) CleanupThread(_ context.Context) { -} - -func getRowKey(table string, key string) string { - return fmt.Sprintf("%s:%s", table, key) -} - -func (db *etcdDB) Read(ctx context.Context, table string, key string, _ []string) (map[string][]byte, error) { - rkey := getRowKey(table, key) - value, err := db.client.Get(ctx, rkey) - if err != nil { - return nil, err - } - - if value.Count == 0 { - return nil, fmt.Errorf("could not find value for key [%s]", rkey) - } - - var r map[string][]byte - err = json.NewDecoder(bytes.NewReader(value.Kvs[0].Value)).Decode(&r) - if err != nil { - return nil, err - } - return r, nil -} - -func (db *etcdDB) Scan(ctx context.Context, table string, startKey string, count int, _ []string) ([]map[string][]byte, error) { - res := make([]map[string][]byte, count) - rkey := getRowKey(table, startKey) - values, err := db.client.Get(ctx, rkey, clientv3.WithFromKey(), clientv3.WithLimit(int64(count))) - if err != nil { - return nil, err - } - - if values.Count != int64(count) { - return nil, fmt.Errorf("unexpected number of result for key [%s], expected %d but was %d", rkey, count, values.Count) - } - - for _, v := range values.Kvs { - var r map[string][]byte - err = json.NewDecoder(bytes.NewReader(v.Value)).Decode(&r) - if err != nil { - return nil, err - } - res = append(res, r) - } - return res, nil -} - -func (db *etcdDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - rkey := getRowKey(table, key) - data, err := json.Marshal(values) - if err != nil { - return err - } - _, err = db.client.Put(ctx, rkey, string(data)) - if err != nil { - return err - } - - return nil -} - -func (db *etcdDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - return db.Update(ctx, table, key, values) -} - -func (db *etcdDB) Delete(ctx context.Context, table string, key string) error { - _, err := db.client.Delete(ctx, getRowKey(table, key)) - if err != nil { - return err - } - return nil -} diff --git a/go-ycsb/db/etcd/doc.go b/go-ycsb/db/etcd/doc.go deleted file mode 100644 index 3ae594e7d..000000000 --- a/go-ycsb/db/etcd/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -package etcd - -// If you want to use etcd, please follow the [Getting Started](https://github.com/etcd-io/etcd#getting-etcd) guide to install it. diff --git a/go-ycsb/db/foundationdb/db.go b/go-ycsb/db/foundationdb/db.go deleted file mode 100644 index 962f81d21..000000000 --- a/go-ycsb/db/foundationdb/db.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build foundationdb - -package foundationdb - -import ( - "context" - "fmt" - - "github.com/apple/foundationdb/bindings/go/src/fdb" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/util" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -const ( - fdbClusterFile = "fdb.cluster" - fdbDatabase = "fdb.dbname" - fdbAPIVersion = "fdb.apiversion" -) - -type fDB struct { - db fdb.Database - r *util.RowCodec - bufPool *util.BufPool -} - -func createDB(p *properties.Properties) (ycsb.DB, error) { - clusterFile := p.GetString(fdbClusterFile, "") - database := p.GetString(fdbDatabase, "DB") - apiVersion := p.GetInt(fdbAPIVersion, 510) - - fdb.MustAPIVersion(apiVersion) - - db, err := fdb.Open(clusterFile, []byte(database)) - if err != nil { - return nil, err - } - - bufPool := util.NewBufPool() - - return &fDB{ - db: db, - r: util.NewRowCodec(p), - bufPool: bufPool, - }, nil -} - -func (db *fDB) Close() error { - return nil -} - -func (db *fDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (db *fDB) CleanupThread(ctx context.Context) { -} - -func (db *fDB) getRowKey(table string, key string) []byte { - return util.Slice(fmt.Sprintf("%s:%s", table, key)) -} - -func (db *fDB) getEndRowKey(table string) []byte { - // ';' is ':' + 1 in the ASCII - return util.Slice(fmt.Sprintf("%s;", table)) -} - -func (db *fDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - rowKey := db.getRowKey(table, key) - row, err := db.db.Transact(func(tr fdb.Transaction) (interface{}, error) { - f := tr.Get(fdb.Key(rowKey)) - return f.Get() - }) - - if err != nil { - return nil, err - } else if row == nil { - return nil, nil - } - - return db.r.Decode(row.([]byte), fields) -} - -func (db *fDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - rowKey := db.getRowKey(table, startKey) - res, err := db.db.Transact(func(tr fdb.Transaction) (interface{}, error) { - r := fdb.KeyRange{ - Begin: fdb.Key(rowKey), - End: fdb.Key(db.getEndRowKey(table)), - } - ri := tr.GetRange(r, fdb.RangeOptions{Limit: count}).Iterator() - res := make([]map[string][]byte, 0, count) - for ri.Advance() { - kv, err := ri.Get() - if err != nil { - return nil, err - } - - if kv.Value == nil { - res = append(res, nil) - } else { - v, err := db.r.Decode(kv.Value, fields) - if err != nil { - return nil, err - } - res = append(res, v) - } - - } - - return res, nil - }) - if err != nil { - return nil, err - } - return res.([]map[string][]byte), nil -} - -func (db *fDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - rowKey := db.getRowKey(table, key) - _, err := db.db.Transact(func(tr fdb.Transaction) (ret interface{}, e error) { - f := tr.Get(fdb.Key(rowKey)) - row, err := f.Get() - if err != nil { - return nil, err - } else if row == nil { - return nil, nil - } - - data, err := db.r.Decode(row, nil) - if err != nil { - return nil, err - } - - for field, value := range values { - data[field] = value - } - - buf := db.bufPool.Get() - defer db.bufPool.Put(buf) - - buf, err = db.r.Encode(buf, data) - if err != nil { - return nil, err - } - - tr.Set(fdb.Key(rowKey), buf) - return - }) - - return err -} - -func (db *fDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - // Simulate TiDB data - buf := db.bufPool.Get() - defer db.bufPool.Put(buf) - - buf, err := db.r.Encode(buf, values) - if err != nil { - return err - } - - rowKey := db.getRowKey(table, key) - _, err = db.db.Transact(func(tr fdb.Transaction) (ret interface{}, e error) { - tr.Set(fdb.Key(rowKey), buf) - return - }) - return err -} - -func (db *fDB) Delete(ctx context.Context, table string, key string) error { - rowKey := db.getRowKey(table, key) - _, err := db.db.Transact(func(tr fdb.Transaction) (ret interface{}, e error) { - tr.Clear(fdb.Key(rowKey)) - return - }) - return err -} - -type fdbCreator struct { -} - -func (c fdbCreator) Create(p *properties.Properties) (ycsb.DB, error) { - return createDB(p) -} - -func init() { - ycsb.RegisterDBCreator("fdb", fdbCreator{}) - ycsb.RegisterDBCreator("foundationdb", fdbCreator{}) -} diff --git a/go-ycsb/db/foundationdb/doc.go b/go-ycsb/db/foundationdb/doc.go deleted file mode 100644 index 42dbd4777..000000000 --- a/go-ycsb/db/foundationdb/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -package foundationdb - -// If you want to use FoundationDB, you must install [client](https://www.foundationdb.org/download/) libraray. diff --git a/go-ycsb/db/minio/db.go b/go-ycsb/db/minio/db.go deleted file mode 100644 index d946e52cf..000000000 --- a/go-ycsb/db/minio/db.go +++ /dev/null @@ -1,128 +0,0 @@ -package minio - -import ( - "bytes" - "context" - "io/ioutil" - - "github.com/magiconair/properties" - "github.com/minio/minio-go" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -const ( - minioAccessKey = "minio.access-key" - minioSecretKey = "minio.secret-key" - minioEndpoint = "minio.endpoint" - minioSecure = "minio.secure" -) - -type minioCreator struct{} - -func (c minioCreator) Create(p *properties.Properties) (ycsb.DB, error) { - accessKeyID := p.GetString(minioAccessKey, "minio") - secretAccessKey := p.GetString(minioSecretKey, "myminio") - endpoint := p.GetString(minioEndpoint, "http://127.0.0.1:9000") - secure := p.GetBool(minioSecure, false) - client, err := minio.New(endpoint, accessKeyID, secretAccessKey, secure) - if err != nil { - return nil, err - } - return &minioDB{ - db: client, - }, nil -} - -type minioDB struct { - db *minio.Client -} - -// Close closes the database layer. -func (db *minioDB) Close() error { - return nil -} - -// InitThread initializes the state associated to the goroutine worker. -// The Returned context will be passed to the following usage. -func (db *minioDB) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { - return ctx -} - -// CleanupThread cleans up the state when the worker finished. -func (db *minioDB) CleanupThread(ctx context.Context) { -} - -// Read reads a record from the database and returns a map of each field/value pair. -// table: The name of the table. -// key: The record key of the record to read. -// fields: The list of fields to read, nil|empty for reading all. -func (db *minioDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - obj, err := db.db.GetObjectWithContext(ctx, table, key, minio.GetObjectOptions{}) - if err != nil { - return nil, err - } - defer obj.Close() - bs, err := ioutil.ReadAll(obj) - if err != nil { - return nil, err - } - return map[string][]byte{"field0": bs}, nil -} - -// Scan scans records from the database. -// table: The name of the table. -// startKey: The first record key to read. -// count: The number of records to read. -// fields: The list of fields to read, nil|empty for reading all. -func (db *minioDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - res := make([]map[string][]byte, count) - done := make(chan struct{}) - defer close(done) - ch := db.db.ListObjectsV2(table, startKey, true, done) - - for i := 0; i < count; i++ { - obj, ok := <-ch - if !ok { - break - } - res[i] = map[string][]byte{obj.Key: nil} - } - return res, nil -} - -// Update updates a record in the database. Any field/value pairs will be written into the -// database or overwritten the existing values with the same field name. -// table: The name of the table. -// key: The record key of the record to update. -// values: A map of field/value pairs to update in the record. -func (db *minioDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - var bs []byte - for _, v := range values { - bs = v - break - } - reader := bytes.NewBuffer(bs) - size := int64(len(bs)) - _, err := db.db.PutObjectWithContext(ctx, table, key, reader, size, minio.PutObjectOptions{}) - return err -} - -// Insert inserts a record in the database. Any field/value pairs will be written into the -// database. -// table: The name of the table. -// key: The record key of the record to insert. -// values: A map of field/value pairs to insert in the record. -func (db *minioDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - return db.Update(ctx, table, key, values) -} - -// Delete deletes a record from the database. -// table: The name of the table. -// key: The record key of the record to delete. -func (db *minioDB) Delete(ctx context.Context, table string, key string) error { - return db.db.RemoveObject(table, key) -} - -func init() { - ycsb.RegisterDBCreator("minio", minioCreator{}) -} diff --git a/go-ycsb/db/mongodb/db.go b/go-ycsb/db/mongodb/db.go deleted file mode 100644 index 913c6bbf7..000000000 --- a/go-ycsb/db/mongodb/db.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright (c) 2020 Daimler TSS GmbH TLS support - -package mongodb - -import ( - "context" - "crypto/x509" - "errors" - "fmt" - "io/ioutil" - "log" - "strings" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/ycsb" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - "go.mongodb.org/mongo-driver/x/mongo/driver/connstring" -) - -const ( - mongodbUrl = "mongodb.url" - mongodbAuthdb = "mongodb.authdb" - mongodbUsername = "mongodb.username" - mongodbPassword = "mongodb.password" - - // see https://github.com/brianfrankcooper/YCSB/tree/master/mongodb#mongodb-configuration-parameters - mongodbUrlDefault = "mongodb://127.0.0.1:27017/ycsb?w=1" - mongodbDatabaseDefault = "ycsb" - mongodbAuthdbDefault = "admin" - mongodbTLSSkipVerify = "mongodb.tls_skip_verify" - mongodbTLSCAFile = "mongodb.tls_ca_file" -) - -type mongoDB struct { - cli *mongo.Client - db *mongo.Database -} - -func (m *mongoDB) Close() error { - return m.cli.Disconnect(context.Background()) -} - -func (m *mongoDB) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { - return ctx -} - -func (m *mongoDB) CleanupThread(ctx context.Context) { -} - -// Read a document. -func (m *mongoDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - projection := map[string]bool{"_id": false} - for _, field := range fields { - projection[field] = true - } - opt := &options.FindOneOptions{Projection: projection} - var doc map[string][]byte - if err := m.db.Collection(table).FindOne(ctx, bson.M{"_id": key}, opt).Decode(&doc); err != nil { - return nil, fmt.Errorf("Read error: %s", err.Error()) - } - return doc, nil -} - -// Scan documents. -func (m *mongoDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - projection := map[string]bool{"_id": false} - for _, field := range fields { - projection[field] = true - } - limit := int64(count) - opt := &options.FindOptions{Projection: projection, Sort: bson.M{"_id": 1}, Limit: &limit} - cursor, err := m.db.Collection(table).Find(ctx, bson.M{"_id": bson.M{"$gte": startKey}}, opt) - if err != nil { - return nil, fmt.Errorf("Scan error: %s", err.Error()) - } - defer cursor.Close(ctx) - var docs []map[string][]byte - for cursor.Next(ctx) { - var doc map[string][]byte - if err := cursor.Decode(&doc); err != nil { - return docs, fmt.Errorf("Scan error: %s", err.Error()) - } - docs = append(docs, doc) - } - return docs, nil -} - -// Insert a document. -func (m *mongoDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - doc := bson.M{"_id": key} - for k, v := range values { - doc[k] = v - } - if _, err := m.db.Collection(table).InsertOne(ctx, doc); err != nil { - fmt.Println(err) - return fmt.Errorf("Insert error: %s", err.Error()) - } - return nil -} - -// Update a document. -func (m *mongoDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - res, err := m.db.Collection(table).UpdateOne(ctx, bson.M{"_id": key}, bson.M{"$set": values}) - if err != nil { - return fmt.Errorf("Update error: %s", err.Error()) - } - if res.MatchedCount != 1 { - return fmt.Errorf("Update error: %s not found", key) - } - return nil -} - -// Delete a document. -func (m *mongoDB) Delete(ctx context.Context, table string, key string) error { - res, err := m.db.Collection(table).DeleteOne(ctx, bson.M{"_id": key}) - if err != nil { - return fmt.Errorf("Delete error: %s", err.Error()) - } - if res.DeletedCount != 1 { - return fmt.Errorf("Delete error: %s not found", key) - } - return nil -} - -type mongodbCreator struct{} - -func (c mongodbCreator) Create(p *properties.Properties) (ycsb.DB, error) { - uri := p.GetString(mongodbUrl, mongodbUrlDefault) - authdb := p.GetString(mongodbAuthdb, mongodbAuthdbDefault) - tlsSkipVerify := p.GetBool(mongodbTLSSkipVerify, false) - caFile := p.GetString(mongodbTLSCAFile, "") - - connString, err := connstring.Parse(uri) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cliOpts := options.Client().ApplyURI(uri) - if cliOpts.TLSConfig != nil { - if len(connString.Hosts) > 0 { - servername := strings.Split(connString.Hosts[0], ":")[0] - log.Printf("using server name for tls: %s\n", servername) - cliOpts.TLSConfig.ServerName = servername - } - if tlsSkipVerify { - log.Println("skipping tls cert validation") - cliOpts.TLSConfig.InsecureSkipVerify = true - } - - if caFile != "" { - // Load CA cert - caCert, err := ioutil.ReadFile(caFile) - if err != nil { - log.Fatal(err) - } - caCertPool := x509.NewCertPool() - if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { - log.Fatalf("certifacte %s could not be parsed", caFile) - } - - cliOpts.TLSConfig.RootCAs = caCertPool - } - } - - username, usrExist := p.Get(mongodbUsername) - password, pwdExist := p.Get(mongodbPassword) - if usrExist && pwdExist { - cliOpts.SetAuth(options.Credential{AuthSource: authdb, Username: username, Password: password}) - } else if usrExist { - return nil, errors.New("mongodb.username is set, but mongodb.password is missing") - } else if pwdExist { - return nil, errors.New("mongodb.password is set, but mongodb.username is missing") - } - - cli, err := mongo.Connect(ctx, cliOpts) - if err != nil { - return nil, err - } - if err := cli.Ping(ctx, nil); err != nil { - return nil, err - } - // check if auth passed - if _, err := cli.ListDatabaseNames(ctx, map[string]string{}); err != nil { - return nil, errors.New("auth failed") - } - - fmt.Println("Connected to MongoDB!") - - m := &mongoDB{ - cli: cli, - db: cli.Database(mongodbDatabaseDefault), - } - return m, nil -} - -func init() { - ycsb.RegisterDBCreator("mongodb", mongodbCreator{}) -} diff --git a/go-ycsb/db/mysql/db.go b/go-ycsb/db/mysql/db.go deleted file mode 100644 index 4c4887662..000000000 --- a/go-ycsb/db/mysql/db.go +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package mysql - -import ( - "bytes" - "context" - "crypto/sha1" - "database/sql" - "database/sql/driver" - "encoding/hex" - "fmt" - "strings" - "sync/atomic" - - "github.com/go-sql-driver/mysql" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// mysql properties -const ( - mysqlHost = "mysql.host" - mysqlPort = "mysql.port" - mysqlUser = "mysql.user" - mysqlPassword = "mysql.password" - mysqlDBName = "mysql.db" - mysqlForceIndex = "mysql.force_index" - // TODO: support batch and auto commit - - tidbClusterIndex = "tidb.cluster_index" - tidbInstances = "tidb.instances" -) - -type muxDriver struct { - cursor uint64 - instances []string - internal driver.Driver -} - -func (drv *muxDriver) Open(name string) (driver.Conn, error) { - k := atomic.AddUint64(&drv.cursor, 1) - return drv.internal.Open(drv.instances[int(k)%len(drv.instances)]) -} - -func openTiDBInstances(addrs []string, user string, pass string, db string) (*sql.DB, error) { - instances := make([]string, len(addrs)) - hash := sha1.New() - for i, addr := range addrs { - hash.Write([]byte("+" + addr)) - instances[i] = fmt.Sprintf("%s:%s@tcp(%s)/%s", user, pass, addr, db) - } - digest := hash.Sum(nil) - driver := "tidb:" + hex.EncodeToString(digest[:]) - for _, n := range sql.Drivers() { - if n == driver { - return sql.Open(driver, "") - } - } - sql.Register(driver, &muxDriver{instances: instances, internal: &mysql.MySQLDriver{}}) - return sql.Open(driver, "") -} - -type mysqlCreator struct { - name string -} - -type mysqlDB struct { - p *properties.Properties - db *sql.DB - verbose bool - forceIndexKeyword string - - bufPool *util.BufPool -} - -type contextKey string - -const stateKey = contextKey("mysqlDB") - -type mysqlState struct { - // Do we need a LRU cache here? - stmtCache map[string]*sql.Stmt - - conn *sql.Conn -} - -func (c mysqlCreator) Create(p *properties.Properties) (ycsb.DB, error) { - d := new(mysqlDB) - d.p = p - - host := p.GetString(mysqlHost, "127.0.0.1") - port := p.GetInt(mysqlPort, 3306) - user := p.GetString(mysqlUser, "root") - password := p.GetString(mysqlPassword, "") - dbName := p.GetString(mysqlDBName, "test") - tidbList := p.GetString(tidbInstances, "") - - var ( - db *sql.DB - err error - tidbs []string - ) - for _, tidb := range strings.Split(tidbList, ",") { - tidb = strings.TrimSpace(tidb) - if len(tidb) > 0 { - tidbs = append(tidbs, tidb) - } - } - if len(tidbs) > 0 { - db, err = openTiDBInstances(tidbs, user, password, dbName) - } else { - dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", user, password, host, port, dbName) - db, err = sql.Open("mysql", dsn) - } - if err != nil { - return nil, err - } - - threadCount := int(p.GetInt64(prop.ThreadCount, prop.ThreadCountDefault)) - db.SetMaxIdleConns(threadCount + 1) - db.SetMaxOpenConns(threadCount * 2) - - d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) - if p.GetBool(mysqlForceIndex, true) { - d.forceIndexKeyword = "FORCE INDEX(`PRIMARY`)" - } - d.db = db - - d.bufPool = util.NewBufPool() - - if err := d.createTable(c.name); err != nil { - return nil, err - } - - return d, nil -} - -func (db *mysqlDB) createTable(driverName string) error { - tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) - if db.p.GetBool(prop.DropData, prop.DropDataDefault) && - !db.p.GetBool(prop.DoTransactions, true) { - if _, err := db.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)); err != nil { - return err - } - } - - fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) - fieldLength := db.p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) - - buf := new(bytes.Buffer) - s := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (YCSB_KEY VARCHAR(64) PRIMARY KEY", tableName) - buf.WriteString(s) - - if (driverName == "tidb" || driverName == "mysql") && db.p.GetBool(tidbClusterIndex, true) { - buf.WriteString(" /*T![clustered_index] CLUSTERED */") - } - - for i := int64(0); i < fieldCount; i++ { - buf.WriteString(fmt.Sprintf(", FIELD%d VARCHAR(%d)", i, fieldLength)) - } - - buf.WriteString(");") - - _, err := db.db.Exec(buf.String()) - return err -} - -func (db *mysqlDB) Close() error { - if db.db == nil { - return nil - } - - return db.db.Close() -} - -func (db *mysqlDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - conn, err := db.db.Conn(ctx) - if err != nil { - panic(fmt.Sprintf("failed to create db conn %v", err)) - } - - state := &mysqlState{ - stmtCache: make(map[string]*sql.Stmt), - conn: conn, - } - - return context.WithValue(ctx, stateKey, state) -} - -func (db *mysqlDB) CleanupThread(ctx context.Context) { - state := ctx.Value(stateKey).(*mysqlState) - - for _, stmt := range state.stmtCache { - stmt.Close() - } - state.conn.Close() -} - -func (db *mysqlDB) getAndCacheStmt(ctx context.Context, query string) (*sql.Stmt, error) { - state := ctx.Value(stateKey).(*mysqlState) - - if stmt, ok := state.stmtCache[query]; ok { - return stmt, nil - } - - stmt, err := state.conn.PrepareContext(ctx, query) - if err == sql.ErrConnDone { - // Try build the connection and prepare again - if state.conn, err = db.db.Conn(ctx); err == nil { - stmt, err = state.conn.PrepareContext(ctx, query) - } - } - - if err != nil { - return nil, err - } - - state.stmtCache[query] = stmt - return stmt, nil -} - -func (db *mysqlDB) clearCacheIfFailed(ctx context.Context, query string, err error) { - if err == nil { - return - } - - state := ctx.Value(stateKey).(*mysqlState) - if stmt, ok := state.stmtCache[query]; ok { - stmt.Close() - } - delete(state.stmtCache, query) -} - -func (db *mysqlDB) queryRows(ctx context.Context, query string, count int, args ...interface{}) ([]map[string][]byte, error) { - if db.verbose { - fmt.Printf("%s %v\n", query, args) - } - - stmt, err := db.getAndCacheStmt(ctx, query) - if err != nil { - return nil, err - } - rows, err := stmt.QueryContext(ctx, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - cols, err := rows.Columns() - if err != nil { - return nil, err - } - - vs := make([]map[string][]byte, 0, count) - for rows.Next() { - m := make(map[string][]byte, len(cols)) - dest := make([]interface{}, len(cols)) - for i := 0; i < len(cols); i++ { - v := new([]byte) - dest[i] = v - } - if err = rows.Scan(dest...); err != nil { - return nil, err - } - - for i, v := range dest { - m[cols[i]] = *v.(*[]byte) - } - - vs = append(vs, m) - } - - return vs, rows.Err() -} - -func (db *mysqlDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - var query string - if len(fields) == 0 { - query = fmt.Sprintf(`SELECT * FROM %s %s WHERE YCSB_KEY = ?`, table, db.forceIndexKeyword) - } else { - query = fmt.Sprintf(`SELECT %s FROM %s %s WHERE YCSB_KEY = ?`, strings.Join(fields, ","), table, db.forceIndexKeyword) - } - - rows, err := db.queryRows(ctx, query, 1, key) - db.clearCacheIfFailed(ctx, query, err) - - if err != nil { - return nil, err - } else if len(rows) == 0 { - return nil, nil - } - - return rows[0], nil -} - -func (db *mysqlDB) BatchRead(ctx context.Context, table string, keys []string, fields []string) ([]map[string][]byte, error) { - args := make([]interface{}, 0, len(keys)) - buf := db.bufPool.Get() - defer db.bufPool.Put(buf) - if len(fields) == 0 { - buf = append(buf, fmt.Sprintf(`SELECT * FROM %s %s WHERE YCSB_KEY IN (`, table, db.forceIndexKeyword)...) - } else { - buf = append(buf, fmt.Sprintf(`SELECT %s FROM %s %s WHERE YCSB_KEY IN (`, strings.Join(fields, ","), table, db.forceIndexKeyword)...) - } - for i, key := range keys { - buf = append(buf, '?') - if i < len(keys)-1 { - buf = append(buf, ',') - } - args = append(args, key) - } - buf = append(buf, ')') - - query := string(buf[:]) - rows, err := db.queryRows(ctx, query, len(keys), args...) - db.clearCacheIfFailed(ctx, query, err) - - if err != nil { - return nil, err - } else if len(rows) == 0 { - return nil, nil - } - - return rows, nil -} - -func (db *mysqlDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - var query string - if len(fields) == 0 { - query = fmt.Sprintf(`SELECT * FROM %s %s WHERE YCSB_KEY >= ? LIMIT ?`, table, db.forceIndexKeyword) - } else { - query = fmt.Sprintf(`SELECT %s FROM %s %s WHERE YCSB_KEY >= ? LIMIT ?`, strings.Join(fields, ","), table, db.forceIndexKeyword) - } - - rows, err := db.queryRows(ctx, query, count, startKey, count) - db.clearCacheIfFailed(ctx, query, err) - - return rows, err -} - -func (db *mysqlDB) execQuery(ctx context.Context, query string, args ...interface{}) error { - if db.verbose { - fmt.Printf("%s %v\n", query, args) - } - - stmt, err := db.getAndCacheStmt(ctx, query) - if err != nil { - return err - } - - _, err = stmt.ExecContext(ctx, args...) - db.clearCacheIfFailed(ctx, query, err) - return err -} - -func (db *mysqlDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - buf := bytes.NewBuffer(db.bufPool.Get()) - defer func() { - db.bufPool.Put(buf.Bytes()) - }() - - buf.WriteString("UPDATE ") - buf.WriteString(table) - buf.WriteString(" SET ") - firstField := true - pairs := util.NewFieldPairs(values) - args := make([]interface{}, 0, len(values)+1) - for _, p := range pairs { - if firstField { - firstField = false - } else { - buf.WriteString(", ") - } - - buf.WriteString(p.Field) - buf.WriteString(`= ?`) - args = append(args, p.Value) - } - buf.WriteString(" WHERE YCSB_KEY = ?") - - args = append(args, key) - - return db.execQuery(ctx, buf.String(), args...) -} - -func (db *mysqlDB) BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) error { - // mysql does not support BatchUpdate, fallback to Update like dbwrapper.go - for i := range keys { - err := db.Update(ctx, table, keys[i], values[i]) - if err != nil { - return err - } - } - return nil -} - -func (db *mysqlDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - args := make([]interface{}, 0, 1+len(values)) - args = append(args, key) - - buf := bytes.NewBuffer(db.bufPool.Get()) - defer func() { - db.bufPool.Put(buf.Bytes()) - }() - - buf.WriteString("INSERT IGNORE INTO ") - buf.WriteString(table) - buf.WriteString(" (YCSB_KEY") - - pairs := util.NewFieldPairs(values) - for _, p := range pairs { - args = append(args, p.Value) - buf.WriteString(" ,") - buf.WriteString(p.Field) - } - buf.WriteString(") VALUES (?") - - for i := 0; i < len(pairs); i++ { - buf.WriteString(" ,?") - } - - buf.WriteByte(')') - - return db.execQuery(ctx, buf.String(), args...) -} - -func (db *mysqlDB) BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) error { - args := make([]interface{}, 0, (1+len(values))*len(keys)) - buf := db.bufPool.Get() - defer db.bufPool.Put(buf) - buf = append(buf, "INSERT IGNORE INTO "...) - buf = append(buf, table...) - buf = append(buf, " (YCSB_KEY"...) - - valueString := strings.Builder{} - valueString.WriteString("(?") - pairs := util.NewFieldPairs(values[0]) - for _, p := range pairs { - buf = append(buf, " ,"...) - buf = append(buf, p.Field...) - - valueString.WriteString(" ,?") - } - // Example: INSERT IGNORE INTO table ([columns]) VALUES - buf = append(buf, ") VALUES "...) - // Example: (?, ?, ?, ....) - valueString.WriteByte(')') - valueStrings := make([]string, 0, len(keys)) - for range keys { - valueStrings = append(valueStrings, valueString.String()) - } - // Example: INSERT IGNORE INTO table ([columns]) VALUES (?, ?, ?...), (?, ?, ?), ... - buf = append(buf, strings.Join(valueStrings, ",")...) - - for i, key := range keys { - args = append(args, key) - pairs := util.NewFieldPairs(values[i]) - for _, p := range pairs { - args = append(args, p.Value) - } - } - - return db.execQuery(ctx, string(buf[:]), args...) -} - -func (db *mysqlDB) Delete(ctx context.Context, table string, key string) error { - query := fmt.Sprintf(`DELETE FROM %s WHERE YCSB_KEY = ?`, table) - - return db.execQuery(ctx, query, key) -} - -func (db *mysqlDB) BatchDelete(ctx context.Context, table string, keys []string) error { - args := make([]interface{}, 0, len(keys)) - buf := db.bufPool.Get() - defer db.bufPool.Put(buf) - buf = append(buf, fmt.Sprintf("DELETE FROM %s WHERE YCSB_KEY IN (", table)...) - for i, key := range keys { - buf = append(buf, '?') - if i < len(keys)-1 { - buf = append(buf, ',') - } - args = append(args, key) - } - buf = append(buf, ')') - - return db.execQuery(ctx, string(buf[:]), args...) -} - -func (db *mysqlDB) Analyze(ctx context.Context, table string) error { - _, err := db.db.Exec(fmt.Sprintf(`ANALYZE TABLE %s`, table)) - return err -} - -func init() { - ycsb.RegisterDBCreator("mysql", mysqlCreator{name: "mysql"}) - ycsb.RegisterDBCreator("tidb", mysqlCreator{name: "tidb"}) - ycsb.RegisterDBCreator("mariadb", mysqlCreator{name: "mariadb"}) -} diff --git a/go-ycsb/db/pegasus/db.go b/go-ycsb/db/pegasus/db.go deleted file mode 100644 index 033dabfea..000000000 --- a/go-ycsb/db/pegasus/db.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) 2017, Xiaomi, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package pegasus - -import ( - "context" - "encoding/json" - _ "net/http/pprof" - "strings" - "time" - - "github.com/XiaoMi/pegasus-go-client/pegalog" - "github.com/XiaoMi/pegasus-go-client/pegasus" - "github.com/XiaoMi/pegasus-go-client/pegasus2" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -var ( - RequestTimeout = 3 * time.Second -) - -type pegasusDB struct { - client *pegasus2.Client - sessions []pegasus.TableConnector -} - -func (db *pegasusDB) InitThread(ctx context.Context, threadId int, _ int) context.Context { - return context.WithValue(ctx, "tid", threadId) -} - -func (db *pegasusDB) CleanupThread(_ context.Context) { -} - -func (db *pegasusDB) Close() error { - for _, s := range db.sessions { - if err := s.Close(); err != nil { - return err - } - } - return db.client.Close() -} - -func (db *pegasusDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - timeoutCtx, _ := context.WithTimeout(ctx, RequestTimeout) - s := db.sessions[ctx.Value("tid").(int)] - - rawValue, err := s.Get(timeoutCtx, []byte(key), []byte("")) - if err == nil { - var value map[string][]byte - json.Unmarshal(rawValue, value) - - result := make(map[string][]byte) - for _, field := range fields { - if v, ok := value[field]; ok { - result[field] = v - } - } - - return result, nil - } else { - pegalog.GetLogger().Println(err) - return nil, err - } -} - -func (db *pegasusDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - return nil, nil -} - -func (db *pegasusDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - timeoutCtx, _ := context.WithTimeout(ctx, RequestTimeout) - s := db.sessions[ctx.Value("tid").(int)] - - value, _ := json.Marshal(values) - err := s.Set(timeoutCtx, []byte(key), nil, value) - if err != nil { - pegalog.GetLogger().Println(err) - } - return err -} - -func (db *pegasusDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - timeoutCtx, _ := context.WithTimeout(ctx, RequestTimeout) - s := db.sessions[ctx.Value("tid").(int)] - - value, _ := json.Marshal(values) - err := s.Set(timeoutCtx, []byte(key), []byte(""), value) - if err != nil { - pegalog.GetLogger().Println(err) - } - return err -} - -func (db *pegasusDB) Delete(ctx context.Context, table string, key string) error { - timeoutCtx, _ := context.WithTimeout(ctx, RequestTimeout) - s := db.sessions[ctx.Value("tid").(int)] - - err := s.Del(timeoutCtx, []byte(key), []byte("")) - if err != nil { - pegalog.GetLogger().Println(err) - } - return err -} - -type pegasusCreator struct{} - -func (pegasusCreator) Create(p *properties.Properties) (ycsb.DB, error) { - conf := p.MustGetString("meta_servers") - metaServers := strings.Split(conf, ",") - tbName := p.MustGetString("table") - threadCount := p.MustGetInt("threadcount") - - cfg := pegasus.Config{MetaServers: metaServers} - db := &pegasusDB{} - db.sessions = make([]pegasus.TableConnector, threadCount) - c := pegasus2.NewClient(cfg) - for i := 0; i < threadCount; i++ { - - var err error - timeoutCtx, _ := context.WithTimeout(context.Background(), RequestTimeout) - tb, err := c.OpenTable(timeoutCtx, tbName) - if err != nil { - pegalog.GetLogger().Println("failed to open table: ", err) - return nil, err - } - db.sessions[i] = tb - } - db.client = c - return db, nil -} - -func init() { - ycsb.RegisterDBCreator("pegasus", pegasusCreator{}) -} diff --git a/go-ycsb/db/pg/db.go b/go-ycsb/db/pg/db.go deleted file mode 100644 index 81a4a0bda..000000000 --- a/go-ycsb/db/pg/db.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package pg - -import ( - "bytes" - "context" - "database/sql" - "fmt" - "strings" - - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - - // pg package - _ "github.com/lib/pq" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// pg properties -const ( - pgHost = "pg.host" - pgPort = "pg.port" - pgUser = "pg.user" - pgPassword = "pg.password" - pgDBName = "pg.db" - pdSSLMode = "pg.sslmode" - // TODO: support batch and auto commit -) - -type pgCreator struct { -} - -type pgDB struct { - p *properties.Properties - db *sql.DB - verbose bool - - bufPool *util.BufPool - - dbName string -} - -type contextKey string - -const stateKey = contextKey("pgDB") - -type pgState struct { - // Do we need a LRU cache here? - stmtCache map[string]*sql.Stmt - - conn *sql.Conn -} - -func (c pgCreator) Create(p *properties.Properties) (ycsb.DB, error) { - d := new(pgDB) - d.p = p - - host := p.GetString(pgHost, "127.0.0.1") - port := p.GetInt(pgPort, 5432) - user := p.GetString(pgUser, "root") - password := p.GetString(pgPassword, "") - dbName := p.GetString(pgDBName, "test") - sslMode := p.GetString(pdSSLMode, "disable") - - dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s", user, password, host, port, dbName, sslMode) - var err error - db, err := sql.Open("postgres", dsn) - if err != nil { - fmt.Printf("open pg failed %v", err) - return nil, err - } - - threadCount := int(p.GetInt64(prop.ThreadCount, prop.ThreadCountDefault)) - db.SetMaxIdleConns(threadCount + 1) - db.SetMaxOpenConns(threadCount * 2) - - d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) - d.db = db - d.dbName = dbName - - d.bufPool = util.NewBufPool() - - if err := d.createTable(); err != nil { - return nil, err - } - - return d, nil -} - -func (db *pgDB) createTable() error { - tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) - - if db.p.GetBool(prop.DropData, prop.DropDataDefault) { - if _, err := db.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)); err != nil { - return err - } - } - - fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) - fieldLength := db.p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) - - buf := new(bytes.Buffer) - s := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (YCSB_KEY VARCHAR(64) PRIMARY KEY", tableName) - buf.WriteString(s) - - for i := int64(0); i < fieldCount; i++ { - buf.WriteString(fmt.Sprintf(", FIELD%d VARCHAR(%d)", i, fieldLength)) - } - - buf.WriteString(");") - - if db.verbose { - fmt.Println(buf.String()) - } - - _, err := db.db.Exec(buf.String()) - return err -} - -func (db *pgDB) Close() error { - if db.db == nil { - return nil - } - - return db.db.Close() -} - -func (db *pgDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - conn, err := db.db.Conn(ctx) - if err != nil { - panic(fmt.Sprintf("failed to create db conn %v", err)) - } - - state := &pgState{ - stmtCache: make(map[string]*sql.Stmt), - conn: conn, - } - - return context.WithValue(ctx, stateKey, state) -} - -func (db *pgDB) CleanupThread(ctx context.Context) { - state := ctx.Value(stateKey).(*pgState) - - for _, stmt := range state.stmtCache { - stmt.Close() - } - state.conn.Close() -} - -func (db *pgDB) getAndCacheStmt(ctx context.Context, query string) (*sql.Stmt, error) { - state := ctx.Value(stateKey).(*pgState) - - if stmt, ok := state.stmtCache[query]; ok { - return stmt, nil - } - - stmt, err := state.conn.PrepareContext(ctx, query) - if err == sql.ErrConnDone { - // Try build the connection and prepare again - if state.conn, err = db.db.Conn(ctx); err == nil { - stmt, err = state.conn.PrepareContext(ctx, query) - } - } - - if err != nil { - return nil, err - } - - state.stmtCache[query] = stmt - return stmt, nil -} - -func (db *pgDB) clearCacheIfFailed(ctx context.Context, query string, err error) { - if err == nil { - return - } - - state := ctx.Value(stateKey).(*pgState) - if stmt, ok := state.stmtCache[query]; ok { - stmt.Close() - } - delete(state.stmtCache, query) -} - -func (db *pgDB) queryRows(ctx context.Context, query string, count int, args ...interface{}) ([]map[string][]byte, error) { - if db.verbose { - fmt.Printf("%s %v\n", query, args) - } - - stmt, err := db.getAndCacheStmt(ctx, query) - if err != nil { - return nil, err - } - rows, err := stmt.QueryContext(ctx, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - cols, err := rows.Columns() - if err != nil { - return nil, err - } - - vs := make([]map[string][]byte, 0, count) - for rows.Next() { - m := make(map[string][]byte, len(cols)) - dest := make([]interface{}, len(cols)) - for i := 0; i < len(cols); i++ { - v := new([]byte) - dest[i] = v - } - if err = rows.Scan(dest...); err != nil { - return nil, err - } - - for i, v := range dest { - m[cols[i]] = *v.(*[]byte) - } - - vs = append(vs, m) - } - - return vs, rows.Err() -} - -func (db *pgDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - var query string - if len(fields) == 0 { - query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY = $1`, table) - } else { - query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY = $1`, strings.Join(fields, ","), table) - } - - rows, err := db.queryRows(ctx, query, 1, key) - db.clearCacheIfFailed(ctx, query, err) - - if err != nil { - return nil, err - } else if len(rows) == 0 { - return nil, nil - } - - return rows[0], nil -} - -func (db *pgDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - var query string - if len(fields) == 0 { - query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY >= $1 LIMIT $2`, table) - } else { - query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY >= $1 LIMIT $2`, strings.Join(fields, ","), table) - } - - rows, err := db.queryRows(ctx, query, count, startKey, count) - db.clearCacheIfFailed(ctx, query, err) - - return rows, err -} - -func (db *pgDB) execQuery(ctx context.Context, query string, args ...interface{}) error { - if db.verbose { - fmt.Printf("%s %v\n", query, args) - } - - stmt, err := db.getAndCacheStmt(ctx, query) - if err != nil { - return err - } - - _, err = stmt.ExecContext(ctx, args...) - db.clearCacheIfFailed(ctx, query, err) - return err -} - -func (db *pgDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - buf := bytes.NewBuffer(db.bufPool.Get()) - defer func() { - db.bufPool.Put(buf.Bytes()) - }() - - buf.WriteString("UPDATE ") - buf.WriteString(table) - buf.WriteString(" SET ") - firstField := true - args := make([]interface{}, 0, len(values)+1) - placeHolderIndex := 1 - pairs := util.NewFieldPairs(values) - for _, p := range pairs { - if firstField { - firstField = false - } else { - buf.WriteString(", ") - } - - buf.WriteString(fmt.Sprintf("%s = $%d", p.Field, placeHolderIndex)) - args = append(args, p.Value) - placeHolderIndex++ - } - buf.WriteString(fmt.Sprintf(" WHERE YCSB_KEY = $%d", placeHolderIndex)) - - args = append(args, key) - - return db.execQuery(ctx, buf.String(), args...) -} - -func (db *pgDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - args := make([]interface{}, 0, 1+len(values)) - args = append(args, key) - - buf := bytes.NewBuffer(db.bufPool.Get()) - defer func() { - db.bufPool.Put(buf.Bytes()) - }() - - buf.WriteString("INSERT INTO ") - buf.WriteString(table) - buf.WriteString(" (YCSB_KEY") - pairs := util.NewFieldPairs(values) - for _, p := range pairs { - args = append(args, p.Value) - buf.WriteString(" ,") - buf.WriteString(p.Field) - } - buf.WriteString(") VALUES ($1") - - for i := 0; i < len(pairs); i++ { - buf.WriteString(fmt.Sprintf(" ,$%d", i+2)) - } - - buf.WriteString(") ON CONFLICT DO NOTHING") - - return db.execQuery(ctx, buf.String(), args...) -} - -func (db *pgDB) Delete(ctx context.Context, table string, key string) error { - query := fmt.Sprintf(`DELETE FROM %s WHERE YCSB_KEY = $1`, table) - - return db.execQuery(ctx, query, key) -} - -func init() { - ycsb.RegisterDBCreator("pg", pgCreator{}) - ycsb.RegisterDBCreator("postgresql", pgCreator{}) - ycsb.RegisterDBCreator("cockroach", pgCreator{}) - ycsb.RegisterDBCreator("cdb", pgCreator{}) -} diff --git a/go-ycsb/db/redis/db.go b/go-ycsb/db/redis/db.go deleted file mode 100644 index 8032dd2b8..000000000 --- a/go-ycsb/db/redis/db.go +++ /dev/null @@ -1,358 +0,0 @@ -package redis - -import ( - "context" - "crypto/tls" - "encoding/json" - "fmt" - "strings" - "time" - - goredis "github.com/go-redis/redis/v9" - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -const HASH_DATATYPE string = "hash" -const STRING_DATATYPE string = "string" -const JSON_DATATYPE string = "json" -const JSON_SET string = "JSON.SET" -const JSON_GET string = "JSON.GET" -const HSET string = "HSET" -const HMGET string = "HMGET" - -type redisClient interface { - Get(ctx context.Context, key string) *goredis.StringCmd - Do(ctx context.Context, args ...interface{}) *goredis.Cmd - Pipeline() goredis.Pipeliner - Scan(ctx context.Context, cursor uint64, match string, count int64) *goredis.ScanCmd - Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *goredis.StatusCmd - Del(ctx context.Context, keys ...string) *goredis.IntCmd - FlushDB(ctx context.Context) *goredis.StatusCmd - Close() error -} - -type redis struct { - client redisClient - mode string - datatype string -} - -func (r *redis) Close() error { - return r.client.Close() -} - -func (r *redis) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (r *redis) CleanupThread(_ context.Context) { -} - -func (r *redis) Read(ctx context.Context, table string, key string, fields []string) (data map[string][]byte, err error) { - data = make(map[string][]byte, len(fields)) - err = nil - switch r.datatype { - case JSON_DATATYPE: - cmds := make([]*goredis.Cmd, len(fields)) - pipe := r.client.Pipeline() - for pos, fieldName := range fields { - cmds[pos] = pipe.Do(ctx, JSON_GET, getKeyName(table, key), getFieldJsonPath(fieldName)) - } - _, err = pipe.Exec(ctx) - if err != nil { - return - } - var s string = "" - for pos, fieldName := range fields { - s, err = cmds[pos].Text() - if err != nil { - return - } - data[fieldName] = []byte(s) - } - case HASH_DATATYPE: - args := make([]interface{}, 0, len(fields)+2) - args = append(args, HMGET, getKeyName(table, key)) - for _, fieldName := range fields { - args = append(args, fieldName) - } - sliceReply, errI := r.client.Do(ctx, args...).StringSlice() - if errI != nil { - return - } - for pos, slicePos := range sliceReply { - data[fields[pos]] = []byte(slicePos) - } - case STRING_DATATYPE: - fallthrough - default: - { - var res string = "" - res, err = r.client.Get(ctx, getKeyName(table, key)).Result() - if err != nil { - return - } - err = json.Unmarshal([]byte(res), &data) - return - } - } - return - -} - -func (r *redis) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - return nil, fmt.Errorf("scan is not supported") -} - -func (r *redis) Update(ctx context.Context, table string, key string, values map[string][]byte) (err error) { - err = nil - switch r.datatype { - case JSON_DATATYPE: - cmds := make([]*goredis.Cmd, 0, len(values)) - pipe := r.client.Pipeline() - for fieldName, bytes := range values { - cmd := pipe.Do(ctx, JSON_SET, getKeyName(table, key), getFieldJsonPath(fieldName), jsonEscape(bytes)) - cmds = append(cmds, cmd) - } - _, err = pipe.Exec(ctx) - if err != nil { - return - } - for _, cmd := range cmds { - err = cmd.Err() - if err != nil { - return - } - } - case HASH_DATATYPE: - args := make([]interface{}, 0, 2*len(values)+2) - args = append(args, HSET, getKeyName(table, key)) - for fieldName, bytes := range values { - args = append(args, fieldName, string(bytes)) - } - err = r.client.Do(ctx, args...).Err() - case STRING_DATATYPE: - fallthrough - default: - { - var initialEncodedJson string = "" - initialEncodedJson, err = r.client.Get(ctx, getKeyName(table, key)).Result() - if err != nil { - return - } - var encodedJson = make([]byte, 0) - err, encodedJson = mergeEncodedJsonWithMap(initialEncodedJson, values) - if err != nil { - return - } - return r.client.Set(ctx, getKeyName(table, key), string(encodedJson), 0).Err() - } - } - return -} - -func mergeEncodedJsonWithMap(stringReply string, values map[string][]byte) (err error, data []byte) { - curVal := map[string][]byte{} - err = json.Unmarshal([]byte(stringReply), &curVal) - if err != nil { - return - } - for k, v := range values { - curVal[k] = v - } - data, err = json.Marshal(curVal) - return -} - -func jsonEscape(bytes []byte) string { - return fmt.Sprintf("\"%s\"", string(bytes)) -} - -func getFieldJsonPath(fieldName string) string { - return fmt.Sprintf("$.%s", fieldName) -} - -func getKeyName(table string, key string) string { - return table + "/" + key -} - -func (r *redis) Insert(ctx context.Context, table string, key string, values map[string][]byte) (err error) { - data, err := json.Marshal(values) - if err != nil { - return err - } - switch r.datatype { - case JSON_DATATYPE: - err = r.client.Do(ctx, JSON_SET, getKeyName(table, key), ".", string(data)).Err() - case HASH_DATATYPE: - args := make([]interface{}, 0, 2*len(values)+2) - args = append(args, HSET, getKeyName(table, key)) - for fieldName, bytes := range values { - args = append(args, fieldName, string(bytes)) - } - err = r.client.Do(ctx, args...).Err() - case STRING_DATATYPE: - fallthrough - default: - err = r.client.Set(ctx, getKeyName(table, key), string(data), 0).Err() - } - return -} - -func (r *redis) Delete(ctx context.Context, table string, key string) error { - return r.client.Del(ctx, getKeyName(table, key)).Err() -} - -type redisCreator struct{} - -func (r redisCreator) Create(p *properties.Properties) (ycsb.DB, error) { - rds := &redis{} - - mode := p.GetString(redisMode, redisModeDefault) - switch mode { - case "cluster": - rds.client = goredis.NewClusterClient(getOptionsCluster(p)) - - if p.GetBool(prop.DropData, prop.DropDataDefault) { - err := rds.client.FlushDB(context.Background()).Err() - if err != nil { - return nil, err - } - } - case "single": - fallthrough - default: - mode = "single" - rds.client = goredis.NewClient(getOptionsSingle(p)) - - if p.GetBool(prop.DropData, prop.DropDataDefault) { - err := rds.client.FlushDB(context.Background()).Err() - if err != nil { - return nil, err - } - } - } - rds.mode = mode - rds.datatype = p.GetString(redisDatatype, redisDatatypeDefault) - fmt.Println(fmt.Sprintf("Using the redis datatype: %s", rds.datatype)) - - return rds, nil -} - -const ( - redisMode = "redis.mode" - redisModeDefault = "single" - redisDatatype = "redis.datatype" - redisDatatypeDefault = "hash" - redisNetwork = "redis.network" - redisNetworkDefault = "tcp" - redisAddr = "redis.addr" - redisAddrDefault = "localhost:6379" - redisPassword = "redis.password" - redisDB = "redis.db" - redisMaxRedirects = "redis.max_redirects" - redisReadOnly = "redis.read_only" - redisRouteByLatency = "redis.route_by_latency" - redisRouteRandomly = "redis.route_randomly" - redisMaxRetries = "redis.max_retries" - redisMinRetryBackoff = "redis.min_retry_backoff" - redisMaxRetryBackoff = "redis.max_retry_backoff" - redisDialTimeout = "redis.dial_timeout" - redisReadTimeout = "redis.read_timeout" - redisWriteTimeout = "redis.write_timeout" - redisPoolSize = "redis.pool_size" - redisPoolSizeDefault = 0 - redisMinIdleConns = "redis.min_idle_conns" - redisMaxConnAge = "redis.max_conn_age" - redisPoolTimeout = "redis.pool_timeout" - redisIdleTimeout = "redis.idle_timeout" - redisIdleCheckFreq = "redis.idle_check_frequency" - redisTLSCA = "redis.tls_ca" - redisTLSCert = "redis.tls_cert" - redisTLSKey = "redis.tls_key" - redisTLSInsecureSkipVerify = "redis.tls_insecure_skip_verify" -) - -func parseTLS(p *properties.Properties) *tls.Config { - caPath, _ := p.Get(redisTLSCA) - certPath, _ := p.Get(redisTLSCert) - keyPath, _ := p.Get(redisTLSKey) - insecureSkipVerify := p.GetBool(redisTLSInsecureSkipVerify, false) - if certPath != "" && keyPath != "" { - config, err := util.CreateTLSConfig(caPath, certPath, keyPath, insecureSkipVerify) - if err == nil { - return config - } - } - - return nil -} - -func getOptionsSingle(p *properties.Properties) *goredis.Options { - opts := &goredis.Options{} - - opts.Addr = p.GetString(redisAddr, redisAddrDefault) - opts.DB = p.GetInt(redisDB, 0) - opts.Network = p.GetString(redisNetwork, redisNetworkDefault) - opts.Password, _ = p.Get(redisPassword) - opts.MaxRetries = p.GetInt(redisMaxRetries, 0) - opts.MinRetryBackoff = p.GetDuration(redisMinRetryBackoff, time.Millisecond*8) - opts.MaxRetryBackoff = p.GetDuration(redisMaxRetryBackoff, time.Millisecond*512) - opts.DialTimeout = p.GetDuration(redisDialTimeout, time.Second*5) - opts.ReadTimeout = p.GetDuration(redisReadTimeout, time.Second*3) - opts.WriteTimeout = p.GetDuration(redisWriteTimeout, opts.ReadTimeout) - opts.PoolSize = p.GetInt(redisPoolSize, redisPoolSizeDefault) - threadCount := p.MustGetInt("threadcount") - if opts.PoolSize == 0 { - opts.PoolSize = threadCount - fmt.Println(fmt.Sprintf("Setting %s=%d (from ) given you haven't specified a value.", redisPoolSize, opts.PoolSize)) - } - opts.MinIdleConns = p.GetInt(redisMinIdleConns, 0) - opts.MaxConnAge = p.GetDuration(redisMaxConnAge, 0) - opts.PoolTimeout = p.GetDuration(redisPoolTimeout, time.Second+opts.ReadTimeout) - opts.IdleTimeout = p.GetDuration(redisIdleTimeout, time.Minute*5) - opts.IdleCheckFrequency = p.GetDuration(redisIdleCheckFreq, time.Minute) - opts.TLSConfig = parseTLS(p) - - return opts -} - -func getOptionsCluster(p *properties.Properties) *goredis.ClusterOptions { - opts := &goredis.ClusterOptions{} - - addresses, _ := p.Get(redisAddr) - opts.Addrs = strings.Split(addresses, ";") - opts.MaxRedirects = p.GetInt(redisMaxRedirects, 0) - opts.ReadOnly = p.GetBool(redisReadOnly, false) - opts.RouteByLatency = p.GetBool(redisRouteByLatency, false) - opts.RouteRandomly = p.GetBool(redisRouteRandomly, false) - opts.Password, _ = p.Get(redisPassword) - opts.MaxRetries = p.GetInt(redisMaxRetries, 0) - opts.MinRetryBackoff = p.GetDuration(redisMinRetryBackoff, time.Millisecond*8) - opts.MaxRetryBackoff = p.GetDuration(redisMaxRetryBackoff, time.Millisecond*512) - opts.DialTimeout = p.GetDuration(redisDialTimeout, time.Second*5) - opts.ReadTimeout = p.GetDuration(redisReadTimeout, time.Second*3) - opts.WriteTimeout = p.GetDuration(redisWriteTimeout, opts.ReadTimeout) - opts.PoolSize = p.GetInt(redisPoolSize, redisPoolSizeDefault) - threadCount := p.MustGetInt("threadcount") - if opts.PoolSize == 0 { - opts.PoolSize = threadCount - fmt.Println(fmt.Sprintf("Setting %s=%d (from ) given you haven't specified a value.", redisPoolSize, opts.PoolSize)) - } - opts.MinIdleConns = p.GetInt(redisMinIdleConns, 0) - opts.MaxConnAge = p.GetDuration(redisMaxConnAge, 0) - opts.PoolTimeout = p.GetDuration(redisPoolTimeout, time.Second+opts.ReadTimeout) - opts.IdleTimeout = p.GetDuration(redisIdleTimeout, time.Minute*5) - opts.IdleCheckFrequency = p.GetDuration(redisIdleCheckFreq, time.Minute) - - opts.TLSConfig = parseTLS(p) - - return opts -} - -func init() { - ycsb.RegisterDBCreator("redis", redisCreator{}) -} diff --git a/go-ycsb/db/rocksdb/db.go b/go-ycsb/db/rocksdb/db.go deleted file mode 100644 index 52ac77e80..000000000 --- a/go-ycsb/db/rocksdb/db.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build rocksdb - -package rocksdb - -import ( - "context" - "fmt" - "os" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - "github.com/pingcap/go-ycsb/pkg/ycsb" - "github.com/tecbot/gorocksdb" -) - -// properties -const ( - rocksdbDir = "rocksdb.dir" - // DBOptions - rocksdbAllowConcurrentMemtableWrites = "rocksdb.allow_concurrent_memtable_writes" - rocsdbAllowMmapReads = "rocksdb.allow_mmap_reads" - rocksdbAllowMmapWrites = "rocksdb.allow_mmap_writes" - rocksdbArenaBlockSize = "rocksdb.arena_block_size" - rocksdbDBWriteBufferSize = "rocksdb.db_write_buffer_size" - rocksdbHardPendingCompactionBytesLimit = "rocksdb.hard_pending_compaction_bytes_limit" - rocksdbLevel0FileNumCompactionTrigger = "rocksdb.level0_file_num_compaction_trigger" - rocksdbLevel0SlowdownWritesTrigger = "rocksdb.level0_slowdown_writes_trigger" - rocksdbLevel0StopWritesTrigger = "rocksdb.level0_stop_writes_trigger" - rocksdbMaxBytesForLevelBase = "rocksdb.max_bytes_for_level_base" - rocksdbMaxBytesForLevelMultiplier = "rocksdb.max_bytes_for_level_multiplier" - rocksdbMaxTotalWalSize = "rocksdb.max_total_wal_size" - rocksdbMemtableHugePageSize = "rocksdb.memtable_huge_page_size" - rocksdbNumLevels = "rocksdb.num_levels" - rocksdbUseDirectReads = "rocksdb.use_direct_reads" - rocksdbUseFsync = "rocksdb.use_fsync" - rocksdbWriteBufferSize = "rocksdb.write_buffer_size" - rocksdbMaxWriteBufferNumber = "rocksdb.max_write_buffer_number" - // TableOptions/BlockBasedTable - rocksdbBlockSize = "rocksdb.block_size" - rocksdbBlockSizeDeviation = "rocksdb.block_size_deviation" - rocksdbCacheIndexAndFilterBlocks = "rocksdb.cache_index_and_filter_blocks" - rocksdbNoBlockCache = "rocksdb.no_block_cache" - rocksdbPinL0FilterAndIndexBlocksInCache = "rocksdb.pin_l0_filter_and_index_blocks_in_cache" - rocksdbWholeKeyFiltering = "rocksdb.whole_key_filtering" - rocksdbBlockRestartInterval = "rocksdb.block_restart_interval" - rocksdbFilterPolicy = "rocksdb.filter_policy" - rocksdbIndexType = "rocksdb.index_type" - rocksdbWALDir = "rocksdb.wal_dir" - // TODO: add more configurations -) - -type rocksDBCreator struct{} - -type rocksDB struct { - p *properties.Properties - - db *gorocksdb.DB - - r *util.RowCodec - bufPool *util.BufPool - - readOpts *gorocksdb.ReadOptions - writeOpts *gorocksdb.WriteOptions -} - -type contextKey string - -func (c rocksDBCreator) Create(p *properties.Properties) (ycsb.DB, error) { - dir := p.GetString(rocksdbDir, "/tmp/rocksdb") - - if p.GetBool(prop.DropData, prop.DropDataDefault) { - os.RemoveAll(dir) - } - - opts := getOptions(p) - - db, err := gorocksdb.OpenDb(opts, dir) - if err != nil { - return nil, err - } - - return &rocksDB{ - p: p, - db: db, - r: util.NewRowCodec(p), - bufPool: util.NewBufPool(), - readOpts: gorocksdb.NewDefaultReadOptions(), - writeOpts: gorocksdb.NewDefaultWriteOptions(), - }, nil -} - -func getTableOptions(p *properties.Properties) *gorocksdb.BlockBasedTableOptions { - tblOpts := gorocksdb.NewDefaultBlockBasedTableOptions() - - tblOpts.SetBlockSize(p.GetInt(rocksdbBlockSize, 4<<10)) - tblOpts.SetBlockSizeDeviation(p.GetInt(rocksdbBlockSizeDeviation, 10)) - tblOpts.SetCacheIndexAndFilterBlocks(p.GetBool(rocksdbCacheIndexAndFilterBlocks, false)) - tblOpts.SetNoBlockCache(p.GetBool(rocksdbNoBlockCache, false)) - tblOpts.SetPinL0FilterAndIndexBlocksInCache(p.GetBool(rocksdbPinL0FilterAndIndexBlocksInCache, false)) - tblOpts.SetWholeKeyFiltering(p.GetBool(rocksdbWholeKeyFiltering, true)) - tblOpts.SetBlockRestartInterval(p.GetInt(rocksdbBlockRestartInterval, 16)) - - if b := p.GetString(rocksdbFilterPolicy, ""); len(b) > 0 { - if b == "rocksdb.BuiltinBloomFilter" { - const defaultBitsPerKey = 10 - tblOpts.SetFilterPolicy(gorocksdb.NewBloomFilter(defaultBitsPerKey)) - } - } - - indexType := p.GetString(rocksdbIndexType, "kBinarySearch") - if indexType == "kBinarySearch" { - tblOpts.SetIndexType(gorocksdb.KBinarySearchIndexType) - } else if indexType == "kHashSearch" { - tblOpts.SetIndexType(gorocksdb.KHashSearchIndexType) - } else if indexType == "kTwoLevelIndexSearch" { - tblOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType) - } - - return tblOpts -} - -func getOptions(p *properties.Properties) *gorocksdb.Options { - opts := gorocksdb.NewDefaultOptions() - opts.SetCreateIfMissing(true) - - opts.SetAllowConcurrentMemtableWrites(p.GetBool(rocksdbAllowConcurrentMemtableWrites, true)) - opts.SetAllowMmapReads(p.GetBool(rocsdbAllowMmapReads, false)) - opts.SetAllowMmapWrites(p.GetBool(rocksdbAllowMmapWrites, false)) - opts.SetArenaBlockSize(p.GetInt(rocksdbArenaBlockSize, 0)) - opts.SetDbWriteBufferSize(p.GetInt(rocksdbDBWriteBufferSize, 0)) - opts.SetHardPendingCompactionBytesLimit(p.GetUint64(rocksdbHardPendingCompactionBytesLimit, 256<<30)) - opts.SetLevel0FileNumCompactionTrigger(p.GetInt(rocksdbLevel0FileNumCompactionTrigger, 4)) - opts.SetLevel0SlowdownWritesTrigger(p.GetInt(rocksdbLevel0SlowdownWritesTrigger, 20)) - opts.SetLevel0StopWritesTrigger(p.GetInt(rocksdbLevel0StopWritesTrigger, 36)) - opts.SetMaxBytesForLevelBase(p.GetUint64(rocksdbMaxBytesForLevelBase, 256<<20)) - opts.SetMaxBytesForLevelMultiplier(p.GetFloat64(rocksdbMaxBytesForLevelMultiplier, 10)) - opts.SetMaxTotalWalSize(p.GetUint64(rocksdbMaxTotalWalSize, 0)) - opts.SetMemtableHugePageSize(p.GetInt(rocksdbMemtableHugePageSize, 0)) - opts.SetNumLevels(p.GetInt(rocksdbNumLevels, 7)) - opts.SetUseDirectReads(p.GetBool(rocksdbUseDirectReads, false)) - opts.SetUseFsync(p.GetBool(rocksdbUseFsync, false)) - opts.SetWriteBufferSize(p.GetInt(rocksdbWriteBufferSize, 64<<20)) - opts.SetMaxWriteBufferNumber(p.GetInt(rocksdbMaxWriteBufferNumber, 2)) - opts.SetWalDir(p.GetString(rocksdbWALDir, "")) - - opts.SetBlockBasedTableFactory(getTableOptions(p)) - - return opts -} - -func (db *rocksDB) Close() error { - db.db.Close() - return nil -} - -func (db *rocksDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (db *rocksDB) CleanupThread(_ context.Context) { -} - -func (db *rocksDB) getRowKey(table string, key string) []byte { - return util.Slice(fmt.Sprintf("%s:%s", table, key)) -} - -func cloneValue(v *gorocksdb.Slice) []byte { - return append([]byte(nil), v.Data()...) -} - -func (db *rocksDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - value, err := db.db.Get(db.readOpts, db.getRowKey(table, key)) - if err != nil { - return nil, err - } - defer value.Free() - - return db.r.Decode(cloneValue(value), fields) -} - -func (db *rocksDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - res := make([]map[string][]byte, count) - it := db.db.NewIterator(db.readOpts) - defer it.Close() - - rowStartKey := db.getRowKey(table, startKey) - - it.Seek(rowStartKey) - i := 0 - for it = it; it.Valid() && i < count; it.Next() { - value := it.Value() - m, err := db.r.Decode(cloneValue(value), fields) - if err != nil { - return nil, err - } - res[i] = m - i++ - } - - if err := it.Err(); err != nil { - return nil, err - } - - return res, nil -} - -func (db *rocksDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - m, err := db.Read(ctx, table, key, nil) - if err != nil { - return err - } - - for field, value := range values { - m[field] = value - } - - buf := db.bufPool.Get() - defer db.bufPool.Put(buf) - - buf, err = db.r.Encode(buf, m) - if err != nil { - return err - } - - rowKey := db.getRowKey(table, key) - - return db.db.Put(db.writeOpts, rowKey, buf) -} - -func (db *rocksDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - rowKey := db.getRowKey(table, key) - - buf := db.bufPool.Get() - defer db.bufPool.Put(buf) - - buf, err := db.r.Encode(buf, values) - if err != nil { - return err - } - return db.db.Put(db.writeOpts, rowKey, buf) -} - -func (db *rocksDB) Delete(ctx context.Context, table string, key string) error { - rowKey := db.getRowKey(table, key) - - return db.db.Delete(db.writeOpts, rowKey) -} - -func init() { - ycsb.RegisterDBCreator("rocksdb", rocksDBCreator{}) -} diff --git a/go-ycsb/db/rocksdb/doc.go b/go-ycsb/db/rocksdb/doc.go deleted file mode 100644 index bf741cd89..000000000 --- a/go-ycsb/db/rocksdb/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -package rocksdb - -// If you want to use RocksDB, please follow [INSTALL](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) to install it. diff --git a/go-ycsb/db/spanner/db.go b/go-ycsb/db/spanner/db.go deleted file mode 100644 index a5483e262..000000000 --- a/go-ycsb/db/spanner/db.go +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package spanner - -import ( - "bytes" - "context" - "fmt" - "os" - "os/user" - "path" - "regexp" - "strings" - - "cloud.google.com/go/spanner" - database "cloud.google.com/go/spanner/admin/database/apiv1" - "google.golang.org/api/iterator" - - adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" - - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -const ( - spannerDBName = "spanner.db" - spannerCredentials = "spanner.credentials" -) - -type spannerCreator struct { -} - -type spannerDB struct { - p *properties.Properties - client *spanner.Client - verbose bool -} - -type contextKey string - -const stateKey = contextKey("spannerDB") - -type spannerState struct { -} - -func (c spannerCreator) Create(p *properties.Properties) (ycsb.DB, error) { - d := new(spannerDB) - d.p = p - - credentials := p.GetString(spannerCredentials, "") - if len(credentials) == 0 { - // no credentials provided, try using ~/.spanner/credentials.json" - usr, err := user.Current() - if err != nil { - return nil, err - } - credentials = path.Join(usr.HomeDir, ".spanner/credentials.json") - } - os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credentials) - - ctx := context.Background() - - adminClient, err := database.NewDatabaseAdminClient(ctx) - if err != nil { - return nil, err - } - defer adminClient.Close() - - dbName := p.GetString(spannerDBName, "") - if len(dbName) == 0 { - return nil, fmt.Errorf("must provide a database like projects/xxxx/instances/xxxx/databases/xxx") - } - - d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) - - _, err = d.createDatabase(ctx, adminClient, dbName) - if err != nil { - return nil, err - } - - client, err := spanner.NewClient(ctx, dbName) - if err != nil { - return nil, err - } - d.client = client - - if err = d.createTable(ctx, adminClient, dbName); err != nil { - return nil, err - } - - return d, nil -} - -func (db *spannerDB) createDatabase(ctx context.Context, adminClient *database.DatabaseAdminClient, dbName string) (string, error) { - matches := regexp.MustCompile("^(.*)/databases/(.*)$").FindStringSubmatch(dbName) - if matches == nil || len(matches) != 3 { - return "", fmt.Errorf("Invalid database id %s", dbName) - } - - database, err := adminClient.GetDatabase(ctx, &adminpb.GetDatabaseRequest{ - Name: dbName, - }) - if err != nil { - return "", err - } - - if database.State == adminpb.Database_STATE_UNSPECIFIED { - op, err := adminClient.CreateDatabase(ctx, &adminpb.CreateDatabaseRequest{ - Parent: matches[1], - CreateStatement: "CREATE DATABASE `" + matches[2] + "`", - ExtraStatements: []string{}, - }) - if err != nil { - return "", err - } - if _, err := op.Wait(ctx); err != nil { - return "", err - } - } - - return matches[2], nil -} - -func (db *spannerDB) tableExisted(ctx context.Context, table string) (bool, error) { - stmt := spanner.NewStatement(`SELECT t.table_name FROM information_schema.tables AS t - WHERE t.table_catalog = '' AND t.table_schema = '' AND t.table_name = @name`) - stmt.Params["name"] = table - iter := db.client.Single().Query(ctx, stmt) - defer iter.Stop() - - found := false - for { - _, err := iter.Next() - if err == iterator.Done { - break - } - - if err != nil { - return false, err - } - found = true - break - } - - return found, nil -} - -func (db *spannerDB) createTable(ctx context.Context, adminClient *database.DatabaseAdminClient, dbName string) error { - tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) - fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) - fieldLength := db.p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) - - existed, err := db.tableExisted(ctx, tableName) - if err != nil { - return err - } - - if db.p.GetBool(prop.DropData, prop.DropDataDefault) && existed { - op, err := adminClient.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ - Database: dbName, - Statements: []string{ - fmt.Sprintf("DROP TABLE %s", tableName), - }, - }) - if err != nil { - return err - } - - if err = op.Wait(ctx); err != nil { - return err - } - existed = false - } - - if existed { - return nil - } - - buf := new(bytes.Buffer) - s := fmt.Sprintf("CREATE TABLE %s (YCSB_KEY STRING(%d)", tableName, fieldLength) - buf.WriteString(s) - - for i := int64(0); i < fieldCount; i++ { - buf.WriteString(fmt.Sprintf(", FIELD%d STRING(%d)", i, fieldLength)) - } - - buf.WriteString(") PRIMARY KEY (YCSB_KEY)") - - op, err := adminClient.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ - Database: dbName, - Statements: []string{ - buf.String(), - }, - }) - if err != nil { - return err - } - - if err := op.Wait(ctx); err != nil { - return err - } - - return nil -} - -func (db *spannerDB) Close() error { - if db.client == nil { - return nil - } - - db.client.Close() - return nil -} - -func (db *spannerDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - state := &spannerState{} - - return context.WithValue(ctx, stateKey, state) -} - -func (db *spannerDB) CleanupThread(ctx context.Context) { - // state := ctx.Value(stateKey).(*spanner) -} - -func (db *spannerDB) queryRows(ctx context.Context, stmt spanner.Statement, count int) ([]map[string][]byte, error) { - if db.verbose { - fmt.Printf("%s %v\n", stmt.SQL, stmt.Params) - } - - iter := db.client.Single().Query(ctx, stmt) - defer iter.Stop() - - vs := make([]map[string][]byte, 0, count) - for { - row, err := iter.Next() - if err == iterator.Done { - break - } - - if err != nil { - return nil, err - } - - rowSize := row.Size() - m := make(map[string][]byte, rowSize) - dest := make([]interface{}, rowSize) - for i := 0; i < rowSize; i++ { - v := new(spanner.NullString) - dest[i] = v - } - - if err := row.Columns(dest...); err != nil { - return nil, err - } - - for i := 0; i < rowSize; i++ { - v := dest[i].(*spanner.NullString) - if v.Valid { - m[row.ColumnName(i)] = util.Slice(v.StringVal) - } - } - - vs = append(vs, m) - } - - return vs, nil -} - -func (db *spannerDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - var query string - if len(fields) == 0 { - query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY = @key`, table) - } else { - query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY = @key`, strings.Join(fields, ","), table) - } - - stmt := spanner.NewStatement(query) - stmt.Params["key"] = key - - rows, err := db.queryRows(ctx, stmt, 1) - - if err != nil { - return nil, err - } else if len(rows) == 0 { - return nil, nil - } - - return rows[0], nil -} - -func (db *spannerDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - var query string - if len(fields) == 0 { - query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY >= @key LIMIT @limit`, table) - } else { - query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY >= @key LIMIT @limit`, strings.Join(fields, ","), table) - } - - stmt := spanner.NewStatement(query) - stmt.Params["key"] = startKey - stmt.Params["limit"] = count - - rows, err := db.queryRows(ctx, stmt, count) - - return rows, err -} - -func createMutations(key string, mutations map[string][]byte) ([]string, []interface{}) { - keys := make([]string, 0, 1+len(mutations)) - values := make([]interface{}, 0, 1+len(mutations)) - keys = append(keys, "YCSB_KEY") - values = append(values, key) - - for key, value := range mutations { - keys = append(keys, key) - values = append(values, util.String(value)) - } - - return keys, values -} - -func (db *spannerDB) Update(ctx context.Context, table string, key string, mutations map[string][]byte) error { - keys, values := createMutations(key, mutations) - m := spanner.Update(table, keys, values) - _, err := db.client.Apply(ctx, []*spanner.Mutation{m}) - return err -} - -func (db *spannerDB) Insert(ctx context.Context, table string, key string, mutations map[string][]byte) error { - keys, values := createMutations(key, mutations) - m := spanner.InsertOrUpdate(table, keys, values) - _, err := db.client.Apply(ctx, []*spanner.Mutation{m}) - return err -} - -func (db *spannerDB) Delete(ctx context.Context, table string, key string) error { - m := spanner.Delete(table, spanner.Key{key}) - _, err := db.client.Apply(ctx, []*spanner.Mutation{m}) - return err -} - -func init() { - ycsb.RegisterDBCreator("spanner", spannerCreator{}) -} diff --git a/go-ycsb/db/sqlite/db.go b/go-ycsb/db/sqlite/db.go deleted file mode 100644 index 0e06ab4e5..000000000 --- a/go-ycsb/db/sqlite/db.go +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build libsqlite3 - -package sqlite - -import ( - "bytes" - "context" - "database/sql" - "fmt" - "net/url" - "os" - "strings" - "time" - - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - - "github.com/magiconair/properties" - // sqlite package - "github.com/mattn/go-sqlite3" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// Sqlite properties -const ( - sqliteDBPath = "sqlite.db" - sqliteMode = "sqlite.mode" - sqliteJournalMode = "sqlite.journalmode" - sqliteCache = "sqlite.cache" - sqliteMaxOpenConns = "sqlite.maxopenconns" - sqliteMaxIdleConns = "sqlite.maxidleconns" - sqliteOptimistic = "sqlite.optimistic" - sqliteOptimisticBackoffMs = "sqlite.optimistic_backoff_ms" -) - -type sqliteCreator struct { -} - -type sqliteDB struct { - p *properties.Properties - db *sql.DB - verbose bool - optimistic bool - backoffMs int - - bufPool *util.BufPool -} - -func (c sqliteCreator) Create(p *properties.Properties) (ycsb.DB, error) { - d := new(sqliteDB) - d.p = p - - dbPath := p.GetString(sqliteDBPath, "/tmp/sqlite.db") - - if p.GetBool(prop.DropData, prop.DropDataDefault) { - os.RemoveAll(dbPath) - } - - mode := p.GetString(sqliteMode, "rwc") - journalMode := p.GetString(sqliteJournalMode, "WAL") - cache := p.GetString(sqliteCache, "shared") - maxOpenConns := p.GetInt(sqliteMaxOpenConns, 1) - maxIdleConns := p.GetInt(sqliteMaxIdleConns, 2) - - v := url.Values{} - v.Set("cache", cache) - v.Set("mode", mode) - v.Set("_journal_mode", journalMode) - dsn := fmt.Sprintf("file:%s?%s", dbPath, v.Encode()) - var err error - db, err := sql.Open("sqlite3", dsn) - if err != nil { - return nil, err - } - - db.SetMaxOpenConns(maxOpenConns) - db.SetMaxIdleConns(maxIdleConns) - - d.optimistic = p.GetBool(sqliteOptimistic, false) - d.backoffMs = p.GetInt(sqliteOptimisticBackoffMs, 5) - d.verbose = p.GetBool(prop.Verbose, prop.VerboseDefault) - d.db = db - - d.bufPool = util.NewBufPool() - - if err := d.createTable(); err != nil { - return nil, err - } - - return d, nil -} - -func (db *sqliteDB) createTable() error { - tableName := db.p.GetString(prop.TableName, prop.TableNameDefault) - - fieldCount := db.p.GetInt64(prop.FieldCount, prop.FieldCountDefault) - fieldLength := db.p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) - - buf := new(bytes.Buffer) - s := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (YCSB_KEY VARCHAR(64) PRIMARY KEY", tableName) - buf.WriteString(s) - - for i := int64(0); i < fieldCount; i++ { - buf.WriteString(fmt.Sprintf(", FIELD%d VARCHAR(%d)", i, fieldLength)) - } - - buf.WriteString(");") - - if db.verbose { - fmt.Println(buf.String()) - } - - _, err := db.db.Exec(buf.String()) - return err -} - -func (db *sqliteDB) Close() error { - if db.db == nil { - return nil - } - - return db.db.Close() -} - -func (db *sqliteDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (db *sqliteDB) CleanupThread(ctx context.Context) { - -} - -func (db *sqliteDB) optimisticTx(ctx context.Context, f func(tx *sql.Tx) error) error { - for { - tx, err := db.db.BeginTx(ctx, nil) - if err != nil { - return err - } - - if err = f(tx); err != nil { - tx.Rollback() - return err - } - - err = tx.Commit() - if err != nil && db.optimistic { - if err, ok := err.(sqlite3.Error); ok && (err.Code == sqlite3.ErrBusy || - err.ExtendedCode == sqlite3.ErrIoErrUnlock) { - time.Sleep(time.Duration(db.backoffMs) * time.Millisecond) - continue - } - } - return err - } -} - -func (db *sqliteDB) doQueryRows(ctx context.Context, tx *sql.Tx, query string, count int, args ...interface{}) ([]map[string][]byte, error) { - if db.verbose { - fmt.Printf("%s %v\n", query, args) - } - - rows, err := tx.QueryContext(ctx, query, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - cols, err := rows.Columns() - if err != nil { - return nil, err - } - - vs := make([]map[string][]byte, 0, count) - for rows.Next() { - m := make(map[string][]byte, len(cols)) - dest := make([]interface{}, len(cols)) - for i := 0; i < len(cols); i++ { - v := new([]byte) - dest[i] = v - } - if err = rows.Scan(dest...); err != nil { - return nil, err - } - - for i, v := range dest { - m[cols[i]] = *v.(*[]byte) - } - - vs = append(vs, m) - } - - return vs, rows.Err() -} - -func (db *sqliteDB) doRead(ctx context.Context, tx *sql.Tx, table string, key string, fields []string) (map[string][]byte, error) { - var query string - if len(fields) == 0 { - query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY = ?`, table) - } else { - query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY = ?`, strings.Join(fields, ","), table) - } - - rows, err := db.doQueryRows(ctx, tx, query, 1, key) - - if err != nil { - return nil, err - } else if len(rows) == 0 { - return nil, nil - } - - return rows[0], nil -} - -func (db *sqliteDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - var output map[string][]byte - err := db.optimisticTx(ctx, func(tx *sql.Tx) error { - res, err := db.doRead(ctx, tx, table, key, fields) - output = res - return err - }) - return output, err -} - -func (db *sqliteDB) doScan(ctx context.Context, tx *sql.Tx, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - var query string - if len(fields) == 0 { - query = fmt.Sprintf(`SELECT * FROM %s WHERE YCSB_KEY >= ? LIMIT ?`, table) - } else { - query = fmt.Sprintf(`SELECT %s FROM %s WHERE YCSB_KEY >= ? LIMIT ?`, strings.Join(fields, ","), table) - } - - rows, err := db.doQueryRows(ctx, tx, query, count, startKey, count) - - return rows, err -} - -func (db *sqliteDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - var output []map[string][]byte - err := db.optimisticTx(ctx, func(tx *sql.Tx) error { - res, err := db.doScan(ctx, tx, table, startKey, count, fields) - output = res - return err - }) - return output, err -} - -func (db *sqliteDB) doUpdate(ctx context.Context, tx *sql.Tx, table string, key string, values map[string][]byte) error { - buf := bytes.NewBuffer(db.bufPool.Get()) - defer func() { - db.bufPool.Put(buf.Bytes()) - }() - - buf.WriteString("UPDATE ") - buf.WriteString(table) - buf.WriteString(" SET ") - firstField := true - pairs := util.NewFieldPairs(values) - args := make([]interface{}, 0, len(values)+1) - for _, p := range pairs { - if firstField { - firstField = false - } else { - buf.WriteString(", ") - } - - buf.WriteString(p.Field) - buf.WriteString(`= ?`) - args = append(args, p.Value) - } - buf.WriteString(" WHERE YCSB_KEY = ?") - - args = append(args, key) - - _, err := tx.ExecContext(ctx, buf.String(), args...) - return err -} - -func (db *sqliteDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - return db.optimisticTx(ctx, func(tx *sql.Tx) error { - return db.doUpdate(ctx, tx, table, key, values) - }) -} - -func (db *sqliteDB) doInsert(ctx context.Context, tx *sql.Tx, table string, key string, values map[string][]byte) error { - args := make([]interface{}, 0, 1+len(values)) - args = append(args, key) - - buf := bytes.NewBuffer(db.bufPool.Get()) - defer func() { - db.bufPool.Put(buf.Bytes()) - }() - - buf.WriteString("INSERT OR IGNORE INTO ") - buf.WriteString(table) - buf.WriteString(" (YCSB_KEY") - - pairs := util.NewFieldPairs(values) - for _, p := range pairs { - args = append(args, p.Value) - buf.WriteString(" ,") - buf.WriteString(p.Field) - } - buf.WriteString(") VALUES (?") - - for i := 0; i < len(pairs); i++ { - buf.WriteString(" ,?") - } - - buf.WriteByte(')') - - _, err := tx.ExecContext(ctx, buf.String(), args...) - if err != nil && db.verbose { - fmt.Printf("error(doInsert): %s: %+v\n", buf.String(), err) - } - return err -} - -func (db *sqliteDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - return db.optimisticTx(ctx, func(tx *sql.Tx) error { return db.doInsert(ctx, tx, table, key, values) }) -} - -func (db *sqliteDB) doDelete(ctx context.Context, tx *sql.Tx, table string, key string) error { - query := fmt.Sprintf(`DELETE FROM %s WHERE YCSB_KEY = ?`, table) - _, err := tx.ExecContext(ctx, query, key) - return err -} - -func (db *sqliteDB) Delete(ctx context.Context, table string, key string) error { - return db.optimisticTx(ctx, func(tx *sql.Tx) error { return db.doDelete(ctx, tx, table, key) }) -} - -func (db *sqliteDB) BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) error { - return db.optimisticTx(ctx, func(tx *sql.Tx) error { - for i := 0; i < len(keys); i++ { - err := db.doInsert(ctx, tx, table, keys[i], values[i]) - if err != nil { - return err - } - } - return nil - }) -} - -func (db *sqliteDB) BatchRead(ctx context.Context, table string, keys []string, fields []string) ([]map[string][]byte, error) { - var output []map[string][]byte - err := db.optimisticTx(ctx, func(tx *sql.Tx) error { - for i := 0; i < len(keys); i++ { - res, err := db.doRead(ctx, tx, table, keys[i], fields) - if err != nil { - return err - } - output = append(output, res) - } - return nil - }) - return output, err -} - -func (db *sqliteDB) BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) error { - return db.optimisticTx(ctx, func(tx *sql.Tx) error { - for i := 0; i < len(keys); i++ { - err := db.doUpdate(ctx, tx, table, keys[i], values[i]) - if err != nil { - return err - } - } - return nil - }) -} - -func (db *sqliteDB) BatchDelete(ctx context.Context, table string, keys []string) error { - return db.optimisticTx(ctx, func(tx *sql.Tx) error { - for i := 0; i < len(keys); i++ { - err := db.doDelete(ctx, tx, table, keys[i]) - if err != nil { - return err - } - } - return nil - }) -} - -func init() { - ycsb.RegisterDBCreator("sqlite", sqliteCreator{}) -} - -var _ ycsb.BatchDB = (*sqliteDB)(nil) diff --git a/go-ycsb/db/sqlite/doc.go b/go-ycsb/db/sqlite/doc.go deleted file mode 100644 index a55792fe6..000000000 --- a/go-ycsb/db/sqlite/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -package sqlite - -// If you want to use sqlite, you must install [client](https://www.sqlite.org/download.html) libraray. diff --git a/go-ycsb/db/tinydb/db.go b/go-ycsb/db/tinydb/db.go deleted file mode 100644 index 25f20b80c..000000000 --- a/go-ycsb/db/tinydb/db.go +++ /dev/null @@ -1,160 +0,0 @@ -package tinydb - -import ( - "context" - "fmt" - - "github.com/magiconair/properties" - "github.com/pingcap-incubator/tinykv/kv/config" - "github.com/pingcap-incubator/tinykv/kv/server" - standalone_storage "github.com/pingcap-incubator/tinykv/kv/storage/standalone_storage" - "github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb" - "github.com/pingcap/go-ycsb/pkg/util" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -type tinydbCreator struct { -} - -type tinyDB struct { - p *properties.Properties - s *standalone_storage.StandAloneStorage - db *server.Server - - r *util.RowCodec - bufPool *util.BufPool -} - -func (c tinydbCreator) Create(p *properties.Properties) (ycsb.DB, error) { - fmt.Print("1\n") - conf := config.NewTestConfig() - s := standalone_storage.NewStandAloneStorage(conf) - s.Start() - server := server.NewServer(s) - - return &tinyDB{ - p: p, - s: s, - db: server, - r: util.NewRowCodec(p), - bufPool: util.NewBufPool(), - }, nil -} - -func (db *tinyDB) Close() error { - return db.s.Stop() -} - -func (db *tinyDB) InitThread(ctx context.Context, _ int, _ int) context.Context { - return ctx -} - -func (db *tinyDB) CleanupThread(_ context.Context) { -} - -func (db *tinyDB) Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) { - req := &kvrpcpb.RawGetRequest{ - Key: []byte(key), - Cf: table, - } - resp, _ := db.db.RawGet(nil, req) - value := resp.Value - return db.r.Decode(value, fields) -} - -func (db *tinyDB) Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) { - res := make([]map[string][]byte, count) - req := &kvrpcpb.RawScanRequest{ - StartKey: []byte(startKey), - Limit: uint32(count), - Cf: table, - } - resp, _ := db.db.RawScan(nil, req) - kvs := resp.Kvs - for i := 0; i < count; i++ { - value := kvs[i].Value - m, err := db.r.Decode(value, fields) - if err != nil { - return nil, err - } - res[i] = m - } - return res, nil -} - -func (db *tinyDB) Update(ctx context.Context, table string, key string, values map[string][]byte) error { - getReq := &kvrpcpb.RawGetRequest{ - Key: []byte(key), - Cf: table, - } - resp, err := db.db.RawGet(nil, getReq) - if err != nil { - return err - } - value := resp.Value - data, err := db.r.Decode(value, nil) - if err != nil { - return err - } - for field, value := range values { - data[field] = value - } - buf := db.bufPool.Get() - defer func() { - db.bufPool.Put(buf) - }() - - buf, err = db.r.Encode(buf, data) - if err != nil { - return err - } - putReq := &kvrpcpb.RawPutRequest{ - Key: []byte(key), - Value: buf, - Cf: table, - } - - _, err = db.db.RawPut(nil, putReq) - if err != nil { - return err - } - return nil -} - -func (db *tinyDB) Insert(ctx context.Context, table string, key string, values map[string][]byte) error { - buf := db.bufPool.Get() - defer func() { - db.bufPool.Put(buf) - }() - - buf, err := db.r.Encode(buf, values) - if err != nil { - return err - } - putReq := &kvrpcpb.RawPutRequest{ - Key: []byte(key), - Value: buf, - Cf: table, - } - - _, err = db.db.RawPut(nil, putReq) - if err != nil { - return err - } - return nil -} - -func (db *tinyDB) Delete(ctx context.Context, table string, key string) error { - req := &kvrpcpb.RawDeleteRequest{ - Key: []byte(key), - Cf: table, - } - _, err := db.db.RawDelete(nil, req) - - return err - -} - -func init() { - ycsb.RegisterDBCreator("tinydb", tinydbCreator{}) -} diff --git a/go-ycsb/fdb-dockerfile b/go-ycsb/fdb-dockerfile deleted file mode 100644 index 8022dfce9..000000000 --- a/go-ycsb/fdb-dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM golang:1.13.6-stretch - -ENV GOPATH /go - -RUN apt-get update \ - && apt-get install -y \ - wget \ - dpkg \ - python \ - git \ - net-tools - -RUN cd / \ - && wget https://www.foundationdb.org/downloads/6.2.11/ubuntu/installers/foundationdb-clients_6.2.11-1_amd64.deb \ - && dpkg -i foundationdb-clients_6.2.11-1_amd64.deb - -ADD . /go/src/github.com/pingcap/go-ycsb - -WORKDIR /go/src/github.com/pingcap/go-ycsb - -RUN GO111MODULE=on go build -tags "foundationdb" -o /go-ycsb ./cmd/* - -FROM ubuntu:18.04 - -RUN apt-get update \ - && apt-get install -y dpkg - -COPY --from=0 /foundationdb-clients_6.2.11-1_amd64.deb /foundationdb-clients_6.2.11-1_amd64.deb -RUN dpkg -i foundationdb-clients_6.2.11-1_amd64.deb - -COPY --from=0 /go-ycsb /go-ycsb - -ADD workloads /workloads - -EXPOSE 6060 - -ENTRYPOINT [ "/go-ycsb" ] diff --git a/go-ycsb/go.mod b/go-ycsb/go.mod deleted file mode 100644 index 55b4866c9..000000000 --- a/go-ycsb/go.mod +++ /dev/null @@ -1,176 +0,0 @@ -module github.com/pingcap/go-ycsb - -require ( - cloud.google.com/go/spanner v1.1.0 - github.com/HdrHistogram/hdrhistogram-go v1.1.2 - github.com/XiaoMi/pegasus-go-client v0.0.0-20181029071519-9400942c5d1c - github.com/aerospike/aerospike-client-go v1.35.2 - github.com/apple/foundationdb/bindings/go v0.0.0-20200112054404-407dc0907f4f - github.com/boltdb/bolt v1.3.1 - github.com/cenkalti/backoff/v4 v4.1.3 - github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e - github.com/dgraph-io/badger v1.6.0 - github.com/elastic/go-elasticsearch/v8 v8.3.0 - github.com/fortytw2/leaktest v1.3.0 // indirect - github.com/go-ini/ini v1.49.0 // indirect - github.com/go-redis/redis/v9 v9.0.0-beta.1 - github.com/go-sql-driver/mysql v1.6.0 - github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b - github.com/lib/pq v1.1.1 - github.com/magiconair/properties v1.8.0 - github.com/mattn/go-sqlite3 v2.0.1+incompatible - github.com/minio/minio-go v6.0.14+incompatible - github.com/olekukonko/tablewriter v0.0.5 - github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c - github.com/pkg/errors v0.9.1 // indirect - github.com/spf13/cobra v1.0.0 - github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c - go.mongodb.org/mongo-driver v1.5.1 - google.golang.org/api v0.87.0 - google.golang.org/genproto v0.0.0-20220718134204-073382fd740c -) - -require ( - github.com/aws/aws-sdk-go-v2 v1.16.16 - github.com/aws/aws-sdk-go-v2/config v1.17.8 - github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 - github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.26 - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1 - github.com/pingcap-incubator/tinykv v0.0.0-20200514052412-e01d729bd45c - go.etcd.io/etcd/client/pkg/v3 v3.5.6 - go.etcd.io/etcd/client/v3 v3.5.6 -) - -// require github.com/pingcap/pd v1.1.0-beta.0.20200106144140-f5a7aa985497 // indirect - -require ( - cloud.google.com/go v0.103.0 // indirect - cloud.google.com/go/compute v1.7.0 // indirect - github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 // indirect - github.com/BurntSushi/toml v0.3.1 // indirect - github.com/Connor1996/badger v1.5.1-0.20220222053432-2d2cbf472c77 // indirect - github.com/DataDog/zstd v1.4.1 // indirect - github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d // indirect - github.com/aws/aws-sdk-go v1.34.28 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.21 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 // indirect - github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 // indirect - github.com/aws/smithy-go v1.13.3 // indirect - github.com/benbjohnson/clock v1.1.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect - github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect - github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 // indirect - github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect - github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect - github.com/dgraph-io/ristretto v0.0.1 // indirect - github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-stack/stack v1.8.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.0.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.3 // indirect - github.com/google/btree v1.1.2 // indirect - github.com/google/go-cmp v0.5.8 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect - github.com/googleapis/gax-go/v2 v2.4.0 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect - github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jmhodges/levigo v1.0.0 // indirect - github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect - github.com/klauspost/compress v1.9.5 // indirect - github.com/klauspost/cpuid v1.2.1 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect - github.com/kr/pretty v0.2.1 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/ncw/directio v1.0.4 // indirect - github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac // indirect - github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef // indirect - github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99 // indirect - github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 // indirect - github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect - github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect - github.com/pingcap/tidb v1.1.0-beta.0.20200309111804-d8264d47f760 // indirect - github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60 // indirect - github.com/prometheus/client_golang v1.11.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect - github.com/shirou/gopsutil v3.21.11+incompatible // indirect - github.com/sirupsen/logrus v1.6.0 // indirect - github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/pflag v1.0.3 // indirect - github.com/tidwall/pretty v1.2.0 // indirect - github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.0.2 // indirect - github.com/xdg-go/stringprep v1.0.2 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 // indirect - go.etcd.io/etcd/api/v3 v3.5.6 // indirect - go.opencensus.io v0.23.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/goleak v1.1.12 // indirect - go.uber.org/multierr v1.7.0 // indirect - go.uber.org/zap v1.20.0 // indirect - golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect - golang.org/x/exp v0.0.0-20220428152302-39d4317da171 // indirect - golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect - golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 // indirect - golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect - golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/grpc v1.48.0 // indirect - google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.42.0 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect -) - -replace github.com/apache/thrift => github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d - -replace google.golang.org/grpc => google.golang.org/grpc v1.25.1 - -replace github.com/pingcap-incubator/tinykv => ../ - -replace github.com/jmhodges/levigo => ../levigo - -replace github.com/tecbot/gorocksdb => ../gorocksdb - -replace github.com/pingcap/tidb => github.com/pingcap-incubator/tinysql v0.0.0-20200518090433-a7d00f9e6aa7 - -go 1.18 diff --git a/go-ycsb/go.sum b/go-ycsb/go.sum deleted file mode 100644 index 371d2dde1..000000000 --- a/go-ycsb/go.sum +++ /dev/null @@ -1,1234 +0,0 @@ -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= -cloud.google.com/go v0.103.0 h1:YXtxp9ymmZjlGzxV7VrYQ8aaQuAgcqxSy6YhDX4I458= -cloud.google.com/go v0.103.0/go.mod h1:vwLx1nqLrzLX/fpwSMOXmFIqBOyHsvHbnAdbGSJ+mKk= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/spanner v1.1.0 h1:hIjiz2Pf6Hy3BWz+Oaw7XUqP+EzWDkj0/DtTkKazxzk= -cloud.google.com/go/spanner v1.1.0/go.mod h1:TzTaF9l2ZY2CIetNvVpUu6ZQy8YEOtzB6ICa5EwYjL0= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Connor1996/badger v1.5.1-0.20220222053432-2d2cbf472c77 h1:9wys07NWIZESExLQK8DFuaOsky842KZlQAM/mCNhAp0= -github.com/Connor1996/badger v1.5.1-0.20220222053432-2d2cbf472c77/go.mod h1:GnSYrIcuBMzgVM5wYQrQOh5OH6nFHVAAUfUmfqTetzU= -github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= -github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/XiaoMi/pegasus-go-client v0.0.0-20181029071519-9400942c5d1c h1:3fAhdHMhoSG57DjJ/dqLFfgD+FoooPbQH6szINbrr3k= -github.com/XiaoMi/pegasus-go-client v0.0.0-20181029071519-9400942c5d1c/go.mod h1:KcL6D/4RZ8RAYzQ5gKI0odcdWUmCVlbQTOlWrhP71CY= -github.com/aerospike/aerospike-client-go v1.35.2 h1:TWV2Bn59Ig7SM4Zue84fFsPGlfFJX/6xbuGHyYFS/ag= -github.com/aerospike/aerospike-client-go v1.35.2/go.mod h1:zj8LBEnWBDOVEIJt8LvaRvDG5ARAoa5dBeHaB472NRc= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d h1:b/FqDLjWXDQI6XBYvWDVgEKv3xOTs68qRkuqyU37lBc= -github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apple/foundationdb/bindings/go v0.0.0-20200112054404-407dc0907f4f h1:HkQOU77BCH+BZPpzwNxm3zjUAt+N7mJWRAGxyCwAGZw= -github.com/apple/foundationdb/bindings/go v0.0.0-20200112054404-407dc0907f4f/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= -github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= -github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= -github.com/aws/aws-sdk-go-v2/config v1.17.8 h1:b9LGqNnOdg9vR4Q43tBTVWk4J6F+W774MSchvKJsqnE= -github.com/aws/aws-sdk-go-v2/config v1.17.8/go.mod h1:UkCI3kb0sCdvtjiXYiU4Zx5h07BOpgBTtkPu/49r+kA= -github.com/aws/aws-sdk-go-v2/credentials v1.12.21 h1:4tjlyCD0hRGNQivh5dN8hbP30qQhMLBE/FgQR1vHHWM= -github.com/aws/aws-sdk-go-v2/credentials v1.12.21/go.mod h1:O+4XyAt4e+oBAoIwNUYkRg3CVMscaIJdmZBOcPgJ8D8= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 h1:bKbdstt7+PzIRSIXZ11Yo8Qh8t0AHn6jEYUfsbVcLjE= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0/go.mod h1:+CBJZMhsb1pTUcB/NTdS505bDX10xS4xnPMqDZj2Ptw= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.26 h1:4FF+S7M7T/f1ORJSwgUJfywZRjB25W8NdGeosQVy+fE= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.26/go.mod h1:RHt+Uh6nvd2kccFcEzgmtsDrGG8AoFCPzvznfUDSg6c= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1 h1:1QpTkQIAaZpR387it1L+erjB5bStGFCJRvmXsodpPEU= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1/go.mod h1:BZhn/C3z13ULTSstVi2Kymc62bgjFh/JwLO9Tm2OFYI= -github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 h1:V9q4A0qnUfDsfivspY1LQRQTOG3Y9FLHvXIaTbcU7XM= -github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20/go.mod h1:7qWU48SMzlrfOlNhHpazW3psFWlOIWrq4SmOr2/ESmk= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17 h1:o0Ia3nb56m8+8NvhbCDiSBiZRNUwIknVWobx5vks0Vk= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17/go.mod h1:WJD9FbkwzM2a1bZ36ntH6+5Jc+x41Q4K2AcLeHDLAS8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 h1:pwvCchFUEnlceKIgPUouBJwK81aCkQ8UDMORfeFtW10= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 h1:OwhhKc1P9ElfWbMKPIbMMZBV6hzJlL2JKD76wNNVzgQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc6lu6OQ97U7hmdesg= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= -github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= -github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= -github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= -github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= -github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 h1:NnLfQ77q0G4k2Of2c1ceQ0ec6MkLQyDp+IGdVM0D8XM= -github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2/go.mod h1:7qG7YFnOALvsx6tKTNmQot8d7cGFXM9TidzvRFLWYwM= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/parser v0.0.0-20181122101858-d773202d5b1f/go.mod h1:2B43mz36vGZNZEwkWi8ayRSSUXLfjL8OkbzwW4NcPMM= -github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGda1NeOBQwqlv7nUXpm+rIVHGxZZ4= -github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= -github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/cznic/y v0.0.0-20181122101901-b05e8c2e8d7b/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs= -github.com/dgraph-io/ristretto v0.0.1 h1:cJwdnj42uV8Jg4+KLrYovLiCgIfz9wtWm6E6KA+1tLs= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c h1:onA2RpIyeCPvYAj1LFYiiMTrSpqVINWMfYFRS7lofJs= -github.com/elastic/elastic-transport-go/v8 v8.0.0-20211216131617-bbee439d559c/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= -github.com/elastic/go-elasticsearch/v8 v8.3.0 h1:RF4iRbvWkiT6UksZ+OwSLeCEtBg/HO8r88xNiSmhb8U= -github.com/elastic/go-elasticsearch/v8 v8.3.0/go.mod h1:Usvydt+x0dv9a1TzEUaovqbJor8rmOHy5dSmPeMAE2k= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.49.0 h1:ymWFBUkwN3JFPjvjcJJ5TSTwh84M66QrH+8vOytLgRY= -github.com/go-ini/ini v1.49.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= -github.com/go-redis/redis/v9 v9.0.0-beta.1 h1:oW3jlPic5HhGUbYMH0lidnP+72BgsT+lCwlVud6o2Mc= -github.com/go-redis/redis/v9 v9.0.0-beta.1/go.mod h1:6gNX1bXdwkpEG0M/hEBNK/Fp8zdyCkjwwKc6vBbfCDI= -github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b h1:dnUw9Ih14dCKzbtZxm+pwQRYIb+9ypiwtZgsCQN4zmg= -github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= -github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= -github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/clock v0.0.0-20180524022203-d293bb356ca4/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= -github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9 h1:Y+lzErDTURqeXqlqYi4YBYbDd7ycU74gW1ADt57/bgY= -github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/retry v0.0.0-20160928201858-1998d01ba1c3/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4= -github.com/juju/testing v0.0.0-20200510222523-6c8c298c77a0 h1:+WWUkhnTjV6RNOxkcwk79qrjeyHEHvBzlneueBsatX4= -github.com/juju/testing v0.0.0-20200510222523-6c8c298c77a0/go.mod h1:hpGvhGHPVbNBraRLZEhoQwFLMrjK8PSlO4D3nDjKYXo= -github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk= -github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= -github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= -github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/ncw/directio v1.0.4 h1:CojwI07mCEmRkajgx42Pf8jyCwTs1ji9/Ij9/PJG12k= -github.com/ncw/directio v1.0.4/go.mod h1:CKGdcN7StAaqjT7Qack3lAXeX4pjnyc46YeqZH1yWVY= -github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac h1:wyheT2lPXRQqYPWY2IVW5BTLrbqCsnhL61zK2R5goLA= -github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac/go.mod h1:ueVCjKQllPmX7uEvCYnZD5b8qjidGf1TCH61arVe4SU= -github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= -github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= -github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= -github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= -github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99 h1:KcEvVBAvyHkUdFAygKAzwB6LAcZ6LS32WHmRD2VyXMI= -github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= -github.com/pingcap-incubator/tinysql v0.0.0-20200518090433-a7d00f9e6aa7 h1:OmyyQb7aA70tZX+LFzixFtNocNl/VIx0QyfYaoDN9iA= -github.com/pingcap-incubator/tinysql v0.0.0-20200518090433-a7d00f9e6aa7/go.mod h1:yuleLQAIKLLe1wiHaX1WdEyCF7K3JKJ9KfaqglC7Nqc= -github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= -github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= -github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= -github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= -github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= -github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= -github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60 h1:aJPXrT1u4VfUSGFA2oQVwl4pOXzqe+YI6wed01cjDH4= -github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5 h1:d9vJ/8gXbVnNk8QFOxFZ7MN7TuHiuvolK1usz5KXVDo= -github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc= -go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd/api/v3 v3.5.6 h1:Cy2qx3npLcYqTKqGJzMypnMv2tiRyifZJ17BlWIWA7A= -go.etcd.io/etcd/api/v3 v3.5.6/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= -go.etcd.io/etcd/client/pkg/v3 v3.5.6 h1:TXQWYceBKqLp4sa87rcPs11SXxUA/mHwH975v+BDvLU= -go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v3 v3.5.6 h1:coLs69PWCXE9G4FKquzNaSHrRyMCAXwF+IX1tAPVO8E= -go.etcd.io/etcd/client/v3 v3.5.6/go.mod h1:f6GRinRMCsFVv9Ht42EyY7nfsVGwrNO0WEoS2pRKzQk= -go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI= -go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= -go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= -go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4= -golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0= -golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 h1:VnGaRqoLmqZH/3TMLJwYCEWkR4j1nuIU1U9TvbqsDUw= -golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191105231337-689d0f08e67a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= -golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= -gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= -google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.87.0 h1:pUQVF/F+X7Tl1lo4LJoJf5BOpjtmINU80p9XpYTU2p4= -google.golang.org/api v0.87.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220718134204-073382fd740c h1:xDUAhRezFnKF6wopxkOfdWYvz2XCiRQzndyDdpwFgbc= -google.golang.org/genproto v0.0.0-20220718134204-073382fd740c/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= -google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4 h1:hILp2hNrRnYjZpmIbx70psAHbBSEcQ1NIzDcUbJ1b6g= -gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs= -gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= -gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/go-ycsb/pkg/client/client.go b/go-ycsb/pkg/client/client.go deleted file mode 100644 index 71f4d2f0c..000000000 --- a/go-ycsb/pkg/client/client.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package client - -import ( - "context" - "fmt" - "math/rand" - "os" - "sync" - "time" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/measurement" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -type worker struct { - p *properties.Properties - workDB ycsb.DB - workload ycsb.Workload - doTransactions bool - doBatch bool - batchSize int - opCount int64 - targetOpsPerMs float64 - threadID int - targetOpsTickNs int64 - opsDone int64 -} - -func newWorker(p *properties.Properties, threadID int, threadCount int, workload ycsb.Workload, db ycsb.DB) *worker { - w := new(worker) - w.p = p - w.doTransactions = p.GetBool(prop.DoTransactions, true) - w.batchSize = p.GetInt(prop.BatchSize, prop.DefaultBatchSize) - if w.batchSize > 1 { - w.doBatch = true - } - w.threadID = threadID - w.workload = workload - w.workDB = db - - var totalOpCount int64 - if w.doTransactions { - totalOpCount = p.GetInt64(prop.OperationCount, 0) - } else { - if _, ok := p.Get(prop.InsertCount); ok { - totalOpCount = p.GetInt64(prop.InsertCount, 0) - } else { - totalOpCount = p.GetInt64(prop.RecordCount, 0) - } - } - - if totalOpCount < int64(threadCount) { - fmt.Printf("totalOpCount(%s/%s/%s): %d should be bigger than threadCount: %d", - prop.OperationCount, - prop.InsertCount, - prop.RecordCount, - totalOpCount, - threadCount) - - os.Exit(-1) - } - - w.opCount = totalOpCount / int64(threadCount) - - targetPerThreadPerms := float64(-1) - if v := p.GetInt64(prop.Target, 0); v > 0 { - targetPerThread := float64(v) / float64(threadCount) - targetPerThreadPerms = targetPerThread / 1000.0 - } - - if targetPerThreadPerms > 0 { - w.targetOpsPerMs = targetPerThreadPerms - w.targetOpsTickNs = int64(1000000.0 / w.targetOpsPerMs) - } - - return w -} - -func (w *worker) throttle(ctx context.Context, startTime time.Time) { - if w.targetOpsPerMs <= 0 { - return - } - - d := time.Duration(w.opsDone * w.targetOpsTickNs) - d = startTime.Add(d).Sub(time.Now()) - if d < 0 { - return - } - select { - case <-ctx.Done(): - case <-time.After(d): - } -} - -func (w *worker) run(ctx context.Context) { - // spread the thread operation out so they don't all hit the DB at the same time - if w.targetOpsPerMs > 0.0 && w.targetOpsPerMs <= 1.0 { - time.Sleep(time.Duration(rand.Int63n(w.targetOpsTickNs))) - } - - startTime := time.Now() - - for w.opCount == 0 || w.opsDone < w.opCount { - var err error - opsCount := 1 - if w.doTransactions { - if w.doBatch { - err = w.workload.DoBatchTransaction(ctx, w.batchSize, w.workDB) - opsCount = w.batchSize - } else { - err = w.workload.DoTransaction(ctx, w.workDB) - } - } else { - if w.doBatch { - err = w.workload.DoBatchInsert(ctx, w.batchSize, w.workDB) - opsCount = w.batchSize - } else { - err = w.workload.DoInsert(ctx, w.workDB) - } - } - - if err != nil && !w.p.GetBool(prop.Silence, prop.SilenceDefault) { - fmt.Printf("operation err: %v\n", err) - } - - if measurement.IsWarmUpFinished() { - w.opsDone += int64(opsCount) - w.throttle(ctx, startTime) - } - - select { - case <-ctx.Done(): - return - default: - } - } -} - -// Client is a struct which is used the run workload to a specific DB. -type Client struct { - p *properties.Properties - workload ycsb.Workload - db ycsb.DB -} - -// NewClient returns a client with the given workload and DB. -// The workload and db can't be nil. -func NewClient(p *properties.Properties, workload ycsb.Workload, db ycsb.DB) *Client { - return &Client{p: p, workload: workload, db: db} -} - -// Run runs the workload to the target DB, and blocks until all workers end. -func (c *Client) Run(ctx context.Context) { - var wg sync.WaitGroup - threadCount := c.p.GetInt(prop.ThreadCount, 1) - - wg.Add(threadCount) - measureCtx, measureCancel := context.WithCancel(ctx) - measureCh := make(chan struct{}, 1) - go func() { - defer func() { - measureCh <- struct{}{} - }() - // load stage no need to warm up - if c.p.GetBool(prop.DoTransactions, true) { - dur := c.p.GetInt64(prop.WarmUpTime, 0) - select { - case <-ctx.Done(): - return - case <-time.After(time.Duration(dur) * time.Second): - } - } - // finish warming up - measurement.EnableWarmUp(false) - - dur := c.p.GetInt64(prop.LogInterval, 10) - t := time.NewTicker(time.Duration(dur) * time.Second) - defer t.Stop() - - for { - select { - case <-t.C: - measurement.Summary() - case <-measureCtx.Done(): - return - } - } - }() - - for i := 0; i < threadCount; i++ { - go func(threadId int) { - defer wg.Done() - - w := newWorker(c.p, threadId, threadCount, c.workload, c.db) - ctx := c.workload.InitThread(ctx, threadId, threadCount) - ctx = c.db.InitThread(ctx, threadId, threadCount) - w.run(ctx) - c.db.CleanupThread(ctx) - c.workload.CleanupThread(ctx) - }(i) - } - - wg.Wait() - if !c.p.GetBool(prop.DoTransactions, true) { - // when loading is finished, try to analyze table if possible. - if analyzeDB, ok := c.db.(ycsb.AnalyzeDB); ok { - analyzeDB.Analyze(ctx, c.p.GetString(prop.TableName, prop.TableNameDefault)) - } - } - measureCancel() - <-measureCh -} diff --git a/go-ycsb/pkg/client/dbwrapper.go b/go-ycsb/pkg/client/dbwrapper.go deleted file mode 100644 index d8ec94d25..000000000 --- a/go-ycsb/pkg/client/dbwrapper.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package client - -import ( - "context" - "fmt" - "time" - - "github.com/pingcap/go-ycsb/pkg/measurement" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// DbWrapper stores the pointer to a implementation of ycsb.DB. -type DbWrapper struct { - DB ycsb.DB -} - -func measure(start time.Time, op string, err error) { - lan := time.Now().Sub(start) - if err != nil { - measurement.Measure(fmt.Sprintf("%s_ERROR", op), start, lan) - return - } - - measurement.Measure(op, start, lan) -} - -func (db DbWrapper) Close() error { - return db.DB.Close() -} - -func (db DbWrapper) InitThread(ctx context.Context, threadID int, threadCount int) context.Context { - return db.DB.InitThread(ctx, threadID, threadCount) -} - -func (db DbWrapper) CleanupThread(ctx context.Context) { - db.DB.CleanupThread(ctx) -} - -func (db DbWrapper) Read(ctx context.Context, table string, key string, fields []string) (_ map[string][]byte, err error) { - start := time.Now() - defer func() { - measure(start, "READ", err) - }() - - return db.DB.Read(ctx, table, key, fields) -} - -func (db DbWrapper) BatchRead(ctx context.Context, table string, keys []string, fields []string) (_ []map[string][]byte, err error) { - batchDB, ok := db.DB.(ycsb.BatchDB) - if ok { - start := time.Now() - defer func() { - measure(start, "BATCH_READ", err) - }() - return batchDB.BatchRead(ctx, table, keys, fields) - } - for _, key := range keys { - _, err := db.DB.Read(ctx, table, key, fields) - if err != nil { - return nil, err - } - } - return nil, nil -} - -func (db DbWrapper) Scan(ctx context.Context, table string, startKey string, count int, fields []string) (_ []map[string][]byte, err error) { - start := time.Now() - defer func() { - measure(start, "SCAN", err) - }() - - return db.DB.Scan(ctx, table, startKey, count, fields) -} - -func (db DbWrapper) Update(ctx context.Context, table string, key string, values map[string][]byte) (err error) { - start := time.Now() - defer func() { - measure(start, "UPDATE", err) - }() - - return db.DB.Update(ctx, table, key, values) -} - -func (db DbWrapper) BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) (err error) { - batchDB, ok := db.DB.(ycsb.BatchDB) - if ok { - start := time.Now() - defer func() { - measure(start, "BATCH_UPDATE", err) - }() - return batchDB.BatchUpdate(ctx, table, keys, values) - } - for i := range keys { - err := db.DB.Update(ctx, table, keys[i], values[i]) - if err != nil { - return err - } - } - return nil -} - -func (db DbWrapper) Insert(ctx context.Context, table string, key string, values map[string][]byte) (err error) { - start := time.Now() - defer func() { - measure(start, "INSERT", err) - }() - - return db.DB.Insert(ctx, table, key, values) -} - -func (db DbWrapper) BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) (err error) { - batchDB, ok := db.DB.(ycsb.BatchDB) - if ok { - start := time.Now() - defer func() { - measure(start, "BATCH_INSERT", err) - }() - return batchDB.BatchInsert(ctx, table, keys, values) - } - for i := range keys { - err := db.DB.Insert(ctx, table, keys[i], values[i]) - if err != nil { - return err - } - } - return nil -} - -func (db DbWrapper) Delete(ctx context.Context, table string, key string) (err error) { - start := time.Now() - defer func() { - measure(start, "DELETE", err) - }() - - return db.DB.Delete(ctx, table, key) -} - -func (db DbWrapper) BatchDelete(ctx context.Context, table string, keys []string) (err error) { - batchDB, ok := db.DB.(ycsb.BatchDB) - if ok { - start := time.Now() - defer func() { - measure(start, "BATCH_DELETE", err) - }() - return batchDB.BatchDelete(ctx, table, keys) - } - for _, key := range keys { - err := db.DB.Delete(ctx, table, key) - if err != nil { - return err - } - } - return nil -} - -func (db DbWrapper) Analyze(ctx context.Context, table string) error { - if analyzeDB, ok := db.DB.(ycsb.AnalyzeDB); ok { - return analyzeDB.Analyze(ctx, table) - } - return nil -} diff --git a/go-ycsb/pkg/generator/acknowledged_counter.go b/go-ycsb/pkg/generator/acknowledged_counter.go deleted file mode 100644 index 885b99b82..000000000 --- a/go-ycsb/pkg/generator/acknowledged_counter.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import ( - "math/rand" - "sync/atomic" - - "github.com/pingcap/go-ycsb/pkg/util" -) - -const ( - // WindowSize is the size of window of pending acks. - WindowSize int64 = 1 << 20 - - // WindowMask is used to turn an ID into a slot in the window. - WindowMask int64 = WindowSize - 1 -) - -// AcknowledgedCounter reports generated integers via Last only -// after they have been acknoledged. -type AcknowledgedCounter struct { - c Counter - - lock util.SpinLock - - window []bool - limit int64 -} - -// NewAcknowledgedCounter creates the counter which starts at start. -func NewAcknowledgedCounter(start int64) *AcknowledgedCounter { - return &AcknowledgedCounter{ - c: Counter{counter: start}, - lock: util.SpinLock{}, - window: make([]bool, WindowSize), - limit: start - 1, - } -} - -// Next implements the Generator Next interface. -func (a *AcknowledgedCounter) Next(r *rand.Rand) int64 { - return a.c.Next(r) -} - -// Last implements the Generator Last interface. -func (a *AcknowledgedCounter) Last() int64 { - return atomic.LoadInt64(&a.limit) -} - -// Acknowledge makes a generated counter vaailable via Last. -func (a *AcknowledgedCounter) Acknowledge(value int64) { - currentSlot := value & WindowMask - if a.window[currentSlot] { - panic("Too many unacknowledged insertion keys.") - } - - a.window[currentSlot] = true - - if !a.lock.TryLock() { - return - } - - defer a.lock.Unlock() - - // move a contiguous sequence from the window - // over to the "limit" variable - - limit := atomic.LoadInt64(&a.limit) - beforeFirstSlot := limit & WindowMask - index := limit + 1 - for ; index != beforeFirstSlot; index++ { - slot := index & WindowMask - if !a.window[slot] { - break - } - - a.window[slot] = false - } - - atomic.StoreInt64(&a.limit, index-1) -} diff --git a/go-ycsb/pkg/generator/constant.go b/go-ycsb/pkg/generator/constant.go deleted file mode 100644 index 2bad6360c..000000000 --- a/go-ycsb/pkg/generator/constant.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import "math/rand" - -// Constant is a trivial generator that always returns the same value. -type Constant struct { - value int64 -} - -// NewConstant creates a constant generator -func NewConstant(value int64) *Constant { - return &Constant{value: value} -} - -// Next implements the Generator Next interface. -func (c *Constant) Next(_ *rand.Rand) int64 { - return c.value -} - -// Last implements the Generator Last interface. -func (c *Constant) Last() int64 { - return c.value -} diff --git a/go-ycsb/pkg/generator/counter.go b/go-ycsb/pkg/generator/counter.go deleted file mode 100644 index 1c0f856a2..000000000 --- a/go-ycsb/pkg/generator/counter.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import ( - "math/rand" - "sync/atomic" -) - -// Counter generates a sequence of integers. [0, 1, ...] -type Counter struct { - counter int64 -} - -// NewCounter creates the Counter generator. -func NewCounter(start int64) *Counter { - return &Counter{ - counter: start, - } -} - -// Next implements Generator Next interface. -func (c *Counter) Next(_ *rand.Rand) int64 { - return atomic.AddInt64(&c.counter, 1) - 1 -} - -// Last implements Generator Last interface. -func (c *Counter) Last() int64 { - return atomic.LoadInt64(&c.counter) - 1 -} diff --git a/go-ycsb/pkg/generator/discrete.go b/go-ycsb/pkg/generator/discrete.go deleted file mode 100644 index 475bca622..000000000 --- a/go-ycsb/pkg/generator/discrete.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import "math/rand" - -type discretePair struct { - Weight float64 - Value int64 -} - -// Discrete generates a distribution by choosing from a discrete set of values. -type Discrete struct { - Number - values []discretePair -} - -// NewDiscrete creates the generator. -func NewDiscrete() *Discrete { - return &Discrete{} -} - -// Next implements the Generator Next interface. -func (d *Discrete) Next(r *rand.Rand) int64 { - sum := float64(0) - - for _, p := range d.values { - sum += p.Weight - } - - val := r.Float64() - - for _, p := range d.values { - pw := p.Weight / sum - if val < pw { - d.SetLastValue(p.Value) - return p.Value - } - - val -= pw - } - - panic("oops, should not get here.") -} - -// Add adds a value with weight. -func (d *Discrete) Add(weight float64, value int64) { - d.values = append(d.values, discretePair{Weight: weight, Value: value}) -} diff --git a/go-ycsb/pkg/generator/exponential.go b/go-ycsb/pkg/generator/exponential.go deleted file mode 100644 index ec3e9a89c..000000000 --- a/go-ycsb/pkg/generator/exponential.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import ( - "math" - "math/rand" -) - -// Exponential generates an exponential distribution. It produces a sequence -// of time intervals according to an expontential distribution. -type Exponential struct { - Number - gamma float64 -} - -// NewExponentialWithMean creates an exponential generator with a mean arrival rate -// of gamma. -func NewExponentialWithMean(mean float64) *Exponential { - return &Exponential{gamma: 1.0 / mean} -} - -// NewExponential creats an exponential generator with percential and range. -func NewExponential(percentile float64, rng float64) *Exponential { - gamma := -math.Log(1.0-percentile/100.0) / rng - return &Exponential{ - gamma: gamma, - } -} - -// Next implements the Generator Next interface. -func (e *Exponential) Next(r *rand.Rand) int64 { - v := int64(-math.Log(r.Float64()) / e.gamma) - e.SetLastValue(v) - return v -} diff --git a/go-ycsb/pkg/generator/histogram.go b/go-ycsb/pkg/generator/histogram.go deleted file mode 100644 index 0b3857d85..000000000 --- a/go-ycsb/pkg/generator/histogram.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import ( - "io/ioutil" - "math/rand" - "strconv" - "strings" - - "github.com/pingcap/go-ycsb/pkg/util" -) - -// Histogram generates integers according to a histogram distribution. -type Histogram struct { - Number - blockSize int64 - buckets []int64 - area int64 - weightedArea int64 -} - -// NewHistogram creates a Histogram generator. -func NewHistogram(buckets []int64, blockSize int64) *Histogram { - var ( - area int64 - weightedArea int64 - ) - - for i, b := range buckets { - area += b - weightedArea += int64(i) * b - } - - return &Histogram{ - buckets: buckets, - blockSize: blockSize, - area: area, - weightedArea: weightedArea, - } -} - -type bucketInfo struct { - location int64 - value int64 -} - -// NewHistogramFromFile creates a Histogram generator from file. -func NewHistogramFromFile(name string) *Histogram { - data, err := ioutil.ReadFile(name) - if err != nil { - util.Fatalf("load histogram file %s failed %v", name, err) - } - - lines := strings.Split(strings.TrimSpace(string(data)), "\n") - line := strings.Split(strings.TrimSpace(lines[0]), "\t") - if line[0] != "BlockSize" { - util.Fatalf("First line of histogram is not the BlockSize but %s", line) - } - - blockSize, err := strconv.ParseInt(line[1], 10, 64) - if err != nil { - util.Fatalf("parse BlockSize failed %v", err) - } - - ay := make([]bucketInfo, 0, len(lines[1:])) - maxLocation := int64(0) - for _, s := range lines[1:] { - s = strings.TrimSpace(s) - if len(s) == 0 { - break - } - - line = strings.Split(s, "\t") - location, _ := strconv.ParseInt(line[0], 10, 64) - if maxLocation < location { - maxLocation = location - } - value, _ := strconv.ParseInt(line[1], 10, 64) - ay = append(ay, bucketInfo{location: location, value: value}) - } - - buckets := make([]int64, maxLocation+1) - for _, b := range ay { - buckets[b.location] = b.value - } - - return NewHistogram(buckets, blockSize) -} - -// Next implements the Generator Next interface. -func (h *Histogram) Next(r *rand.Rand) int64 { - n := r.Int63n(h.area) - - i := int64(0) - for ; i < int64(len(h.buckets))-1; i++ { - n -= h.buckets[i] - if n <= 0 { - return (i + 1) * h.blockSize - } - } - - v := i * h.blockSize - h.SetLastValue(v) - return v -} diff --git a/go-ycsb/pkg/generator/hotspot.go b/go-ycsb/pkg/generator/hotspot.go deleted file mode 100644 index 874d9b20e..000000000 --- a/go-ycsb/pkg/generator/hotspot.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import "math/rand" - -// Hotspot generates integers resembling a hotspot distribution where %x of operations -// access y% of data items. -type Hotspot struct { - Number - lowerBound int64 - upperBound int64 - hotInterval int64 - coldInterval int64 - hotsetFraction float64 - hotOpnFraction float64 -} - -// NewHotspot creates a Hotspot generator. -// lowerBound: the lower bound of the distribution. -// upperBound: the upper bound of the distribution. -// hotsetFraction: percentage of data itme. -// hotOpnFraction: percentage of operations accessing the hot set. -func NewHotspot(lowerBound int64, upperBound int64, hotsetFraction float64, hotOpnFraction float64) *Hotspot { - if hotsetFraction < 0.0 || hotsetFraction > 1.0 { - hotsetFraction = 0.0 - } - - if hotOpnFraction < 0.0 || hotOpnFraction > 1.0 { - hotOpnFraction = 0.0 - } - - if lowerBound > upperBound { - lowerBound, upperBound = upperBound, lowerBound - } - - interval := upperBound - lowerBound + 1 - hotInterval := int64(float64(interval) * hotsetFraction) - return &Hotspot{ - lowerBound: lowerBound, - upperBound: upperBound, - hotsetFraction: hotsetFraction, - hotOpnFraction: hotOpnFraction, - hotInterval: hotInterval, - coldInterval: interval - hotInterval, - } -} - -// Next implements the Generator Next interface. -func (h *Hotspot) Next(r *rand.Rand) int64 { - value := int64(0) - if r.Float64() < h.hotOpnFraction { - value = h.lowerBound + r.Int63n(h.hotInterval) - } else { - value = h.lowerBound + h.hotInterval + r.Int63n(h.coldInterval) - } - h.SetLastValue(value) - return value -} diff --git a/go-ycsb/pkg/generator/number.go b/go-ycsb/pkg/generator/number.go deleted file mode 100644 index dced17ffa..000000000 --- a/go-ycsb/pkg/generator/number.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -// Number is a common generator. -type Number struct { - LastValue int64 -} - -// SetLastValue sets the last value generated. -func (n *Number) SetLastValue(value int64) { - n.LastValue = value -} - -// Last implements the Generator Last interface. -func (n *Number) Last() int64 { - return n.LastValue -} diff --git a/go-ycsb/pkg/generator/scrambled_zipfian.go b/go-ycsb/pkg/generator/scrambled_zipfian.go deleted file mode 100644 index a9bdf7d92..000000000 --- a/go-ycsb/pkg/generator/scrambled_zipfian.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import ( - "math/rand" - - "github.com/pingcap/go-ycsb/pkg/util" -) - -// ScrambledZipfian produces a sequence of items, such that some items are more popular than -// others, according to a zipfian distribution -type ScrambledZipfian struct { - Number - gen *Zipfian - min int64 - max int64 - itemCount int64 -} - -// NewScrambledZipfian creates a ScrambledZipfian generator. -func NewScrambledZipfian(min int64, max int64, zipfianConstant float64) *ScrambledZipfian { - const ( - zetan = float64(26.46902820178302) - usedZipfianConstant = float64(0.99) - itemCount = int64(10000000000) - ) - - s := new(ScrambledZipfian) - s.min = min - s.max = max - s.itemCount = max - min + 1 - if zipfianConstant == usedZipfianConstant { - s.gen = NewZipfian(0, itemCount, zipfianConstant, zetan) - } else { - s.gen = NewZipfianWithRange(0, itemCount, zipfianConstant) - } - return s -} - -// Next implements the Generator Next interface. -func (s *ScrambledZipfian) Next(r *rand.Rand) int64 { - n := s.gen.Next(r) - - n = s.min + util.Hash64(n)%s.itemCount - s.SetLastValue(n) - return n -} diff --git a/go-ycsb/pkg/generator/sequential.go b/go-ycsb/pkg/generator/sequential.go deleted file mode 100644 index 7ef1a5881..000000000 --- a/go-ycsb/pkg/generator/sequential.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import ( - "math/rand" - "sync/atomic" -) - -// Sequential generates a sequence of integers 0, 1, ... -type Sequential struct { - counter int64 - interval int64 - start int64 -} - -// NewSequential creates a counter that starts at countStart. -func NewSequential(countStart int64, countEnd int64) *Sequential { - return &Sequential{ - counter: 0, - start: countStart, - interval: countEnd - countStart + 1, - } -} - -// Next implements the Generator Next interface. -func (s *Sequential) Next(_ *rand.Rand) int64 { - n := s.start + atomic.AddInt64(&s.counter, 1)%s.interval - return n -} - -// Last implements the Generator Last interface. -func (s *Sequential) Last() int64 { - return atomic.LoadInt64(&s.counter) + 1 -} diff --git a/go-ycsb/pkg/generator/skewedlatest.go b/go-ycsb/pkg/generator/skewedlatest.go deleted file mode 100644 index d4d2258ac..000000000 --- a/go-ycsb/pkg/generator/skewedlatest.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import ( - "math/rand" - "time" - - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -// SkewedLatest generates a popularity distribution of items, -// skewed to favor recent items significantly more than older items. -type SkewedLatest struct { - Number - basis ycsb.Generator - zipfian *Zipfian -} - -// NewSkewedLatest creates the SkewedLatest generator. -// basis is Counter or AcknowledgedCounter -func NewSkewedLatest(basis ycsb.Generator) *SkewedLatest { - zipfian := NewZipfianWithItems(basis.Last(), ZipfianConstant) - s := &SkewedLatest{ - basis: basis, - zipfian: zipfian, - } - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - s.Next(r) - return s -} - -// Next implements the Generator Next interface. -func (s *SkewedLatest) Next(r *rand.Rand) int64 { - max := s.basis.Last() - next := max - s.zipfian.next(r, max) - s.SetLastValue(next) - return next -} diff --git a/go-ycsb/pkg/generator/uniform.go b/go-ycsb/pkg/generator/uniform.go deleted file mode 100644 index 10678895d..000000000 --- a/go-ycsb/pkg/generator/uniform.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import "math/rand" - -// Uniform generates integers randomly. -type Uniform struct { - Number - lb int64 - ub int64 - interval int64 -} - -// NewUniform creates the Uniform generator. -func NewUniform(lb int64, ub int64) *Uniform { - return &Uniform{ - lb: lb, - ub: ub, - interval: ub - lb + 1, - } -} - -// Next implements the Generator Next interface. -func (u *Uniform) Next(r *rand.Rand) int64 { - n := r.Int63n(u.interval) + u.lb - u.SetLastValue(n) - return n -} diff --git a/go-ycsb/pkg/generator/zipfian.go b/go-ycsb/pkg/generator/zipfian.go deleted file mode 100644 index 4bac60750..000000000 --- a/go-ycsb/pkg/generator/zipfian.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package generator - -import ( - "fmt" - "math" - "math/rand" - "time" - - "github.com/pingcap/go-ycsb/pkg/util" -) - -const ( - // ZipfianConstant is the default constant for the zipfian. - ZipfianConstant = float64(0.99) -) - -// Zipfian generates the zipfian distribution. It produces a sequence of items, such that some items are more popular than -// others, according to a zipfian distribution. When you construct an instance of this class, you specify the number -// of items in the set to draw from, either by specifying an itemcount (so that the sequence is of items from 0 to -// itemcount-1) or by specifying a min and a max (so that the sequence is of items from min to max inclusive). After -// you construct the instance, you can change the number of items by calling Next(*rand.Rand). -// -// Note that the popular items will be clustered together, e.g. item 0 is the most popular, item 1 the second most -// popular, and so on (or min is the most popular, min+1 the next most popular, etc.) If you don't want this clustering, -// and instead want the popular items scattered throughout the item space, then use ScrambledZipfian(located in scrambled_zipfian.go) instead. -// -// Be aware: initializing this generator may take a long time if there are lots of items to choose from (e.g. over a -// minute for 100 million objects). This is because certain mathematical values need to be computed to properly -// generate a zipfian skew, and one of those values (zeta) is a sum sequence from 1 to n, where n is the itemcount. -// Note that if you increase the number of items in the set, we can compute a new zeta incrementally, so it should be -// fast unless you have added millions of items. However, if you decrease the number of items, we recompute zeta from -// scratch, so this can take a long time. -// -// The algorithm used here is from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994. -type Zipfian struct { - Number - - lock util.SpinLock - - items int64 - base int64 - - zipfianConstant float64 - - alpha float64 - zetan float64 - theta float64 - eta float64 - zeta2Theta float64 - - countForZeta int64 - - allowItemCountDecrease bool -} - -// NewZipfianWithItems creates the Zipfian generator. -func NewZipfianWithItems(items int64, zipfianConstant float64) *Zipfian { - return NewZipfianWithRange(0, items-1, zipfianConstant) -} - -// NewZipfianWithRange creates the Zipfian generator. -func NewZipfianWithRange(min int64, max int64, zipfianConstant float64) *Zipfian { - return NewZipfian(min, max, zipfianConstant, zetaStatic(0, max-min+1, zipfianConstant, 0)) -} - -// NewZipfian creates the Zipfian generator. -func NewZipfian(min int64, max int64, zipfianConstant float64, zetan float64) *Zipfian { - items := max - min + 1 - z := new(Zipfian) - - z.items = items - z.base = min - - z.zipfianConstant = zipfianConstant - theta := z.zipfianConstant - z.theta = theta - - z.zeta2Theta = z.zeta(0, 2, theta, 0) - - z.alpha = 1.0 / (1.0 - theta) - z.zetan = zetan - z.countForZeta = items - z.eta = (1 - math.Pow(2.0/float64(items), 1-theta)) / (1 - z.zeta2Theta/z.zetan) - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - z.Next(r) - return z -} - -func (z *Zipfian) zeta(st int64, n int64, thetaVal float64, initialSum float64) float64 { - z.countForZeta = n - return zetaStatic(st, n, thetaVal, initialSum) -} - -func zetaStatic(st int64, n int64, theta float64, initialSum float64) float64 { - sum := initialSum - - for i := st; i < n; i++ { - sum += 1 / math.Pow(float64(i+1), theta) - } - - return sum -} - -func (z *Zipfian) next(r *rand.Rand, itemCount int64) int64 { - if itemCount != z.countForZeta { - z.lock.Lock() - if itemCount > z.countForZeta { - //we have added more items. can compute zetan incrementally, which is cheaper - z.zetan = z.zeta(z.countForZeta, itemCount, z.theta, z.zetan) - z.eta = (1 - math.Pow(2.0/float64(z.items), 1-z.theta)) / (1 - z.zeta2Theta/z.zetan) - } else if itemCount < z.countForZeta && z.allowItemCountDecrease { - //note : for large itemsets, this is very slow. so don't do it! - fmt.Printf("recomputing Zipfian distribution, should be avoided,item count %v, count for zeta %v\n", itemCount, z.countForZeta) - z.zetan = z.zeta(0, itemCount, z.theta, 0) - z.eta = (1 - math.Pow(2.0/float64(z.items), 1-z.theta)) / (1 - z.zeta2Theta/z.zetan) - } - z.lock.Unlock() - } - - u := r.Float64() - uz := u * z.zetan - - if uz < 1.0 { - return z.base - } - - if uz < 1.0+math.Pow(0.5, z.theta) { - return z.base + 1 - } - - ret := z.base + int64(float64(itemCount)*math.Pow(z.eta*u-z.eta+1, z.alpha)) - z.SetLastValue(ret) - return ret -} - -// Next implements the Generator Next interface. -func (z *Zipfian) Next(r *rand.Rand) int64 { - return z.next(r, z.items) -} diff --git a/go-ycsb/pkg/measurement/csv.go b/go-ycsb/pkg/measurement/csv.go deleted file mode 100644 index 0014c6fe6..000000000 --- a/go-ycsb/pkg/measurement/csv.go +++ /dev/null @@ -1,51 +0,0 @@ -package measurement - -import ( - "fmt" - "io" - "time" -) - -type csventry struct { - // start time of the operation in us from unix epoch - startUs int64 - // latency of the operation in us - latencyUs int64 -} - -type csvs struct { - opCsv map[string][]csventry -} - -func InitCSV() *csvs { - return &csvs{ - opCsv: make(map[string][]csventry), - } -} - -func (c *csvs) Measure(op string, start time.Time, lan time.Duration) { - c.opCsv[op] = append(c.opCsv[op], csventry{ - startUs: start.UnixMicro(), - latencyUs: lan.Microseconds(), - }) -} - -func (c *csvs) Output(w io.Writer) error { - _, err := fmt.Fprintln(w, "operation,timestamp_us,latency_us") - if err != nil { - return err - } - for op, entries := range c.opCsv { - for _, entry := range entries { - _, err := fmt.Fprintf(w, "%s,%d,%d\n", op, entry.startUs, entry.latencyUs) - if err != nil { - return err - } - } - } - return nil -} - -func (c *csvs) Summary() { - // do nothing as csvs don't keep a summary -} diff --git a/go-ycsb/pkg/measurement/histogram.go b/go-ycsb/pkg/measurement/histogram.go deleted file mode 100644 index 5da22c9e8..000000000 --- a/go-ycsb/pkg/measurement/histogram.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package measurement - -import ( - "sort" - "time" - - hdrhistogram "github.com/HdrHistogram/hdrhistogram-go" - "github.com/pingcap/go-ycsb/pkg/util" -) - -type histogram struct { - boundCounts util.ConcurrentMap - startTime time.Time - hist *hdrhistogram.Histogram -} - -// Metric name. -const ( - ELAPSED = "ELAPSED" - COUNT = "COUNT" - QPS = "QPS" - AVG = "AVG" - MIN = "MIN" - MAX = "MAX" - PER99TH = "PER99TH" - PER999TH = "PER999TH" - PER9999TH = "PER9999TH" -) - -func newHistogram() *histogram { - h := new(histogram) - h.startTime = time.Now() - h.hist = hdrhistogram.New(1, 24*60*60*1000*1000, 3) - return h -} - -func (h *histogram) Measure(latency time.Duration) { - h.hist.RecordValue(latency.Microseconds()) -} - -func (h *histogram) Summary() []string { - res := h.getInfo() - - return []string{ - util.FloatToOneString(res[ELAPSED]), - util.IntToString(res[COUNT]), - util.FloatToOneString(res[QPS]), - util.IntToString(res[AVG]), - util.IntToString(res[MIN]), - util.IntToString(res[MAX]), - util.IntToString(res[PER99TH]), - util.IntToString(res[PER999TH]), - util.IntToString(res[PER9999TH]), - } -} - -func (h *histogram) getInfo() map[string]interface{} { - min := h.hist.Min() - max := h.hist.Max() - avg := int64(h.hist.Mean()) - count := h.hist.TotalCount() - - bounds := h.boundCounts.Keys() - sort.Ints(bounds) - - per99 := h.hist.ValueAtPercentile(99) - per999 := h.hist.ValueAtPercentile(99.9) - per9999 := h.hist.ValueAtPercentile(99.99) - - elapsed := time.Now().Sub(h.startTime).Seconds() - qps := float64(count) / elapsed - res := make(map[string]interface{}) - res[ELAPSED] = elapsed - res[COUNT] = count - res[QPS] = qps - res[AVG] = avg - res[MIN] = min - res[MAX] = max - res[PER99TH] = per99 - res[PER999TH] = per999 - res[PER9999TH] = per9999 - - return res -} diff --git a/go-ycsb/pkg/measurement/histograms.go b/go-ycsb/pkg/measurement/histograms.go deleted file mode 100644 index 4e0d0a054..000000000 --- a/go-ycsb/pkg/measurement/histograms.go +++ /dev/null @@ -1,76 +0,0 @@ -package measurement - -import ( - "io" - "os" - "sort" - "time" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" -) - -type histograms struct { - p *properties.Properties - - histograms map[string]*histogram -} - -func (h *histograms) Measure(op string, start time.Time, lan time.Duration) { - opM, ok := h.histograms[op] - if !ok { - opM = newHistogram() - h.histograms[op] = opM - } - - opM.Measure(lan) -} - -func (h *histograms) summary() map[string][]string { - summaries := make(map[string][]string, len(h.histograms)) - for op, opM := range h.histograms { - summaries[op] = opM.Summary() - } - return summaries -} - -func (h *histograms) Summary() { - h.Output(os.Stdout) -} - -func (h *histograms) Output(w io.Writer) error { - summaries := h.summary() - keys := make([]string, 0, len(summaries)) - for k := range summaries { - keys = append(keys, k) - } - sort.Strings(keys) - - lines := [][]string{} - for _, op := range keys { - line := []string{op} - line = append(line, summaries[op]...) - lines = append(lines, line) - } - - outputStyle := h.p.GetString(prop.OutputStyle, util.OutputStylePlain) - switch outputStyle { - case util.OutputStylePlain: - util.RenderString(w, "%-6s - %s\n", header, lines) - case util.OutputStyleJson: - util.RenderJson(w, header, lines) - case util.OutputStyleTable: - util.RenderTable(w, header, lines) - default: - panic("unsupported outputstyle: " + outputStyle) - } - return nil -} - -func InitHistograms(p *properties.Properties) *histograms { - return &histograms{ - p: p, - histograms: make(map[string]*histogram, 16), - } -} diff --git a/go-ycsb/pkg/measurement/measurement.go b/go-ycsb/pkg/measurement/measurement.go deleted file mode 100644 index 7260bb31d..000000000 --- a/go-ycsb/pkg/measurement/measurement.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package measurement - -import ( - "bufio" - "os" - "sync" - "sync/atomic" - "time" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -var header = []string{"Operation", "Takes(s)", "Count", "OPS", "Avg(us)", "Min(us)", "Max(us)", "99th(us)", "99.9th(us)", "99.99th(us)"} - -type measurement struct { - sync.RWMutex - - p *properties.Properties - - measurer ycsb.Measurer -} - -func (m *measurement) measure(op string, start time.Time, lan time.Duration) { - m.Lock() - m.measurer.Measure(op, start, lan) - m.Unlock() -} - -func (m *measurement) output() { - m.RLock() - defer m.RUnlock() - - outFile := m.p.GetString(prop.MeasurementRawOutputFile, "") - var w *bufio.Writer - if outFile == "" { - w = bufio.NewWriter(os.Stdout) - } else { - f, err := os.Create(outFile) - if err != nil { - panic("failed to create output file: " + err.Error()) - } - defer f.Close() - w = bufio.NewWriter(f) - } - - err := globalMeasure.measurer.Output(w) - if err != nil { - panic("failed to write output: " + err.Error()) - } - - err = w.Flush() - if err != nil { - panic("failed to flush output: " + err.Error()) - } -} - -func (m *measurement) summary() { - m.RLock() - globalMeasure.measurer.Summary() - m.RUnlock() -} - -// InitMeasure initializes the global measurement. -func InitMeasure(p *properties.Properties) { - globalMeasure = new(measurement) - globalMeasure.p = p - measurementType := p.GetString(prop.MeasurementType, prop.MeasurementTypeDefault) - switch measurementType { - case "histogram": - globalMeasure.measurer = InitHistograms(p) - case "raw", "csv": - globalMeasure.measurer = InitCSV() - default: - panic("unsupported measurement type: " + measurementType) - } - EnableWarmUp(p.GetInt64(prop.WarmUpTime, 0) > 0) -} - -// Output prints the complete measurements. -func Output() { - globalMeasure.output() -} - -// Summary prints the measurement summary. -func Summary() { - globalMeasure.summary() -} - -// EnableWarmUp sets whether to enable warm-up. -func EnableWarmUp(b bool) { - if b { - atomic.StoreInt32(&warmUp, 1) - } else { - atomic.StoreInt32(&warmUp, 0) - } -} - -// IsWarmUpFinished returns whether warm-up is finished or not. -func IsWarmUpFinished() bool { - return atomic.LoadInt32(&warmUp) == 0 -} - -// Measure measures the operation. -func Measure(op string, start time.Time, lan time.Duration) { - if IsWarmUpFinished() { - globalMeasure.measure(op, start, lan) - } -} - -var globalMeasure *measurement -var warmUp int32 // use as bool, 1 means in warmup progress, 0 means warmup finished. diff --git a/go-ycsb/pkg/prop/prop.go b/go-ycsb/pkg/prop/prop.go deleted file mode 100644 index 7ca5d6c17..000000000 --- a/go-ycsb/pkg/prop/prop.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package prop - -// Properties -const ( - InsertStart = "insertstart" - InsertCount = "insertcount" - InsertStartDefault = int64(0) - - OperationCount = "operationcount" - RecordCount = "recordcount" - RecordCountDefault = int64(0) - Workload = "workload" - DB = "db" - Exporter = "exporter" - ExportFile = "exportfile" - ThreadCount = "threadcount" - ThreadCountDefault = int64(200) - Target = "target" - MaxExecutiontime = "maxexecutiontime" - WarmUpTime = "warmuptime" - DoTransactions = "dotransactions" - Status = "status" - Label = "label" - // batch mode - BatchSize = "batch.size" - DefaultBatchSize = int(1) - - TableName = "table" - TableNameDefault = "usertable" - FieldCount = "fieldcount" - FieldCountDefault = int64(10) - // "uniform", "zipfian", "constant", "histogram" - FieldLengthDistribution = "fieldlengthdistribution" - FieldLengthDistributionDefault = "constant" - FieldLength = "fieldlength" - FieldLengthDefault = int64(100) - // Used if fieldlengthdistribution is "histogram" - FieldLengthHistogramFile = "fieldlengthhistogram" - FieldLengthHistogramFileDefault = "hist.txt" - ReadAllFields = "readallfields" - ReadALlFieldsDefault = true - WriteAllFields = "writeallfields" - WriteAllFieldsDefault = false - DataIntegrity = "dataintegrity" - DataIntegrityDefault = false - ReadProportion = "readproportion" - ReadProportionDefault = float64(0.95) - UpdateProportion = "updateproportion" - UpdateProportionDefault = float64(0.05) - InsertProportion = "insertproportion" - InsertProportionDefault = float64(0.0) - ScanProportion = "scanproportion" - ScanProportionDefault = float64(0.0) - ReadModifyWriteProportion = "readmodifywriteproportion" - ReadModifyWriteProportionDefault = float64(0.0) - // "uniform", "zipfian", "latest" - RequestDistribution = "requestdistribution" - RequestDistributionDefault = "uniform" - ZeroPadding = "zeropadding" - ZeroPaddingDefault = int64(1) - MaxScanLength = "maxscanlength" - MaxScanLengthDefault = int64(1000) - // "uniform", "zipfian" - ScanLengthDistribution = "scanlengthdistribution" - ScanLengthDistributionDefault = "uniform" - // "ordered", "hashed" - InsertOrder = "insertorder" - InsertOrderDefault = "hashed" - HotspotDataFraction = "hotspotdatafraction" - HotspotDataFractionDefault = float64(0.2) - HotspotOpnFraction = "hotspotopnfraction" - HotspotOpnFractionDefault = float64(0.8) - InsertionRetryLimit = "core_workload_insertion_retry_limit" - InsertionRetryLimitDefault = int64(0) - InsertionRetryInterval = "core_workload_insertion_retry_interval" - InsertionRetryIntervalDefault = int64(3) - - ExponentialPercentile = "exponential.percentile" - ExponentialPercentileDefault = float64(95) - ExponentialFrac = "exponential.frac" - ExponentialFracDefault = float64(0.8571428571) - - DebugPprof = "debug.pprof" - DebugPprofDefault = ":6060" - - Verbose = "verbose" - VerboseDefault = false - DropData = "dropdata" - DropDataDefault = false - - Silence = "silence" - SilenceDefault = true - - KeyPrefix = "keyprefix" - KeyPrefixDefault = "user" - - LogInterval = "measurement.interval" - - MeasurementType = "measurementtype" - MeasurementTypeDefault = "histogram" - MeasurementRawOutputFile = "measurement.output_file" - - Command = "command" - - OutputStyle = "outputstyle" -) diff --git a/go-ycsb/pkg/util/concurrent_map.go b/go-ycsb/pkg/util/concurrent_map.go deleted file mode 100644 index e50e5b025..000000000 --- a/go-ycsb/pkg/util/concurrent_map.go +++ /dev/null @@ -1,319 +0,0 @@ -package util - -import ( - "encoding/json" - "sync" -) - -/* - This module is modified from github.com/orcaman/concurrent-map. - MIT license. -*/ - -// A "thread" safe map of type int:int64. -// To avoid lock bottlenecks this map is dived to several (SHARD_COUNT) map shards. -type ConcurrentMap struct { - shards []*ConcurrentMapShared - shardCount int -} - -// A "thread" safe int to int64 map. -type ConcurrentMapShared struct { - items map[int]int64 - sync.RWMutex // Read Write mutex, guards access to internal map. -} - -// Creates a new concurrent map. -func New(shardCount int) ConcurrentMap { - m := new(ConcurrentMap) - m.shardCount = shardCount - m.shards = make([]*ConcurrentMapShared, shardCount) - for i := 0; i < shardCount; i++ { - m.shards[i] = &ConcurrentMapShared{items: make(map[int]int64)} - } - return *m -} - -// GetShard returns shard under given key -func (m ConcurrentMap) GetShard(key int) *ConcurrentMapShared { - return m.shards[uint(m.fnv32(key))%uint(m.shardCount)] -} - -func (m ConcurrentMap) MSet(data map[int]int64) { - for key, value := range data { - m.Set(key, value) - } -} - -// Sets the given value under the specified key. -func (m ConcurrentMap) Set(key int, value int64) { - // Get map shard. - shard := m.GetShard(key) - shard.Lock() - shard.items[key] = value - shard.Unlock() -} - -// Callback to return new element to be inserted into the map -// It is called while lock is held, therefore it MUST NOT -// try to access other keys in same map, as it can lead to deadlock since -// Go sync.RWLock is not reentrant -type UpsertCb func(exist bool, valueInMap int64, newValue int64) int64 - -// Insert or Update - updates existing element or inserts a new one using UpsertCb -func (m ConcurrentMap) Upsert(key int, value int64, cb UpsertCb) (res int64) { - shard := m.GetShard(key) - shard.Lock() - v, ok := shard.items[key] - res = cb(ok, v, value) - shard.items[key] = res - shard.Unlock() - return res -} - -// Sets the given value under the specified key if no value was associated with it. -func (m ConcurrentMap) SetIfAbsent(key int, value int64) bool { - // Get map shard. - shard := m.GetShard(key) - shard.Lock() - _, ok := shard.items[key] - if !ok { - shard.items[key] = value - } - shard.Unlock() - return !ok -} - -// Get retrieves an element from map under given key. -func (m ConcurrentMap) Get(key int) (int64, bool) { - // Get shard - shard := m.GetShard(key) - shard.RLock() - // Get item from shard. - val, ok := shard.items[key] - shard.RUnlock() - return val, ok -} - -// Count returns the number of elements within the map. -func (m ConcurrentMap) Count() int { - count := 0 - for i := 0; i < m.shardCount; i++ { - shard := m.shards[i] - shard.RLock() - count += len(shard.items) - shard.RUnlock() - } - return count -} - -// Looks up an item under specified key -func (m ConcurrentMap) Has(key int) bool { - // Get shard - shard := m.GetShard(key) - shard.RLock() - // See if element is within shard. - _, ok := shard.items[key] - shard.RUnlock() - return ok -} - -// Remove removes an element from the map. -func (m ConcurrentMap) Remove(key int) { - // Try to get shard. - shard := m.GetShard(key) - shard.Lock() - delete(shard.items, key) - shard.Unlock() -} - -// RemoveCb is a callback executed in a map.RemoveCb() call, while Lock is held -// If returns true, the element will be removed from the map -type RemoveCb func(key int, v int64, exists bool) bool - -// RemoveCb locks the shard containing the key, retrieves its current value and calls the callback with those params -// If callback returns true and element exists, it will remove it from the map -// Returns the value returned by the callback (even if element was not present in the map) -func (m ConcurrentMap) RemoveCb(key int, cb RemoveCb) bool { - // Try to get shard. - shard := m.GetShard(key) - shard.Lock() - v, ok := shard.items[key] - remove := cb(key, v, ok) - if remove && ok { - delete(shard.items, key) - } - shard.Unlock() - return remove -} - -// Pop removes an element from the map and returns it -func (m ConcurrentMap) Pop(key int) (v int64, exists bool) { - // Try to get shard. - shard := m.GetShard(key) - shard.Lock() - v, exists = shard.items[key] - delete(shard.items, key) - shard.Unlock() - return v, exists -} - -// IsEmpty checks if map is empty. -func (m ConcurrentMap) IsEmpty() bool { - return m.Count() == 0 -} - -// Used by the Iter & IterBuffered functions to wrap two variables together over a channel, -type Tuple struct { - Key int - Val int64 -} - -// Iter returns an iterator which could be used in a for range loop. -// -// Deprecated: using IterBuffered() will get a better performence -func (m ConcurrentMap) Iter() <-chan Tuple { - chans := snapshot(m) - ch := make(chan Tuple) - go fanIn(chans, ch) - return ch -} - -// IterBuffered returns a buffered iterator which could be used in a for range loop. -func (m ConcurrentMap) IterBuffered() <-chan Tuple { - chans := snapshot(m) - total := 0 - for _, c := range chans { - total += cap(c) - } - ch := make(chan Tuple, total) - go fanIn(chans, ch) - return ch -} - -// Returns a array of channels that contains elements in each shard, -// which likely takes a snapshot of `m`. -// It returns once the size of each buffered channel is determined, -// before all the channels are populated using goroutines. -func snapshot(m ConcurrentMap) (chans []chan Tuple) { - chans = make([]chan Tuple, m.shardCount) - wg := sync.WaitGroup{} - wg.Add(m.shardCount) - // Foreach shard. - for index, shard := range m.shards { - go func(index int, shard *ConcurrentMapShared) { - // Foreach key, value pair. - shard.RLock() - chans[index] = make(chan Tuple, len(shard.items)) - wg.Done() - for key, val := range shard.items { - chans[index] <- Tuple{key, val} - } - shard.RUnlock() - close(chans[index]) - }(index, shard) - } - wg.Wait() - return chans -} - -// fanIn reads elements from channels `chans` into channel `out` -func fanIn(chans []chan Tuple, out chan Tuple) { - wg := sync.WaitGroup{} - wg.Add(len(chans)) - for _, ch := range chans { - go func(ch chan Tuple) { - for t := range ch { - out <- t - } - wg.Done() - }(ch) - } - wg.Wait() - close(out) -} - -// Items returns all items as map[int]int64 -func (m ConcurrentMap) Items() map[int]int64 { - tmp := make(map[int]int64) - - // Insert items to temporary map. - for item := range m.IterBuffered() { - tmp[item.Key] = item.Val - } - - return tmp -} - -// Iterator callback,called for every key,value found in -// maps. RLock is held for all calls for a given shard -// therefore callback sess consistent view of a shard, -// but not across the shards -type IterCb func(key int, v int64) - -// Callback based iterator, cheapest way to read -// all elements in a map. -func (m ConcurrentMap) IterCb(fn IterCb) { - for idx := range m.shards { - shard := m.shards[idx] - shard.RLock() - for key, value := range shard.items { - fn(key, value) - } - shard.RUnlock() - } -} - -// Keys returns all keys as []int -func (m ConcurrentMap) Keys() []int { - count := m.Count() - ch := make(chan int, count) - go func() { - // Foreach shard. - wg := sync.WaitGroup{} - wg.Add(m.shardCount) - for _, shard := range m.shards { - go func(shard *ConcurrentMapShared) { - // Foreach key, value pair. - shard.RLock() - for key := range shard.items { - ch <- key - } - shard.RUnlock() - wg.Done() - }(shard) - } - wg.Wait() - close(ch) - }() - - // Generate keys - keys := make([]int, 0, count) - for k := range ch { - keys = append(keys, k) - } - return keys -} - -//Reviles ConcurrentMap "private" variables to json marshal. -func (m ConcurrentMap) MarshalJSON() ([]byte, error) { - // Create a temporary map, which will hold all item spread across shards. - tmp := make(map[int]int64) - - // Insert items to temporary map. - for item := range m.IterBuffered() { - tmp[item.Key] = item.Val - } - return json.Marshal(tmp) -} - -func (m ConcurrentMap) fnv32(key int) uint32 { - key = (key + 0x7ed55d16) + (key << 12) - key = (key ^ 0xc761c23c) ^ (key >> 19) - key = (key + 0x165667b1) + (key << 5) - key = (key + 0xd3a2646c) ^ (key << 9) - key = (key + 0xfd7046c5) + (key << 3) - key = (key ^ 0xb55a4f09) ^ (key >> 16) - key = key % m.shardCount - return uint32(key) -} diff --git a/go-ycsb/pkg/util/core.go b/go-ycsb/pkg/util/core.go deleted file mode 100644 index 87d44b12f..000000000 --- a/go-ycsb/pkg/util/core.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "fmt" - "sort" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/prop" -) - -// createFieldIndices is a helper function to create a field -> index mapping -// for the core workload -func createFieldIndices(p *properties.Properties) map[string]int64 { - fieldCount := p.GetInt64(prop.FieldCount, prop.FieldCountDefault) - m := make(map[string]int64, fieldCount) - for i := int64(0); i < fieldCount; i++ { - field := fmt.Sprintf("field%d", i) - m[field] = i - } - return m -} - -// allFields is a helper function to create all fields -func allFields(p *properties.Properties) []string { - fieldCount := p.GetInt64(prop.FieldCount, prop.FieldCountDefault) - fields := make([]string, 0, fieldCount) - for i := int64(0); i < fieldCount; i++ { - field := fmt.Sprintf("field%d", i) - fields = append(fields, field) - } - return fields -} - -// RowCodec is a helper struct to encode and decode TiDB format row -type RowCodec struct { - fieldIndices map[string]int64 - fields []string -} - -// NewRowCodec creates the RowCodec -func NewRowCodec(p *properties.Properties) *RowCodec { - return &RowCodec{ - fieldIndices: createFieldIndices(p), - fields: allFields(p), - } -} - -// Decode decodes the row and returns a field-value map -func (r *RowCodec) Decode(row []byte, fields []string) (map[string][]byte, error) { - if len(fields) == 0 { - fields = r.fields - } - - data, err := DecodeRow(row) - if err != nil { - return nil, err - } - - res := make(map[string][]byte, len(fields)) - for _, field := range fields { - i := r.fieldIndices[field] - if v, ok := data[i]; ok { - res[field] = v - } - } - - return res, nil -} - -// Encode encodes the values -func (r *RowCodec) Encode(buf []byte, values map[string][]byte) ([]byte, error) { - cols := make([][]byte, 0, len(values)) - colIDs := make([]int64, 0, len(values)) - - for k, v := range values { - i := r.fieldIndices[k] - cols = append(cols, v) - colIDs = append(colIDs, i) - } - - rowData, err := EncodeRow(cols, colIDs, buf) - return rowData, err -} - -// FieldPair is a pair to hold field + value. -type FieldPair struct { - Field string - Value []byte -} - -// FieldPairs implements sort interface for []FieldPair -type FieldPairs []FieldPair - -// Len implements sort interface Len -func (s FieldPairs) Len() int { - return len(s) -} - -// Len implements sort interface Swap -func (s FieldPairs) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -// Len implements sort interface Less -func (s FieldPairs) Less(i, j int) bool { - return s[i].Field < s[j].Field -} - -// NewFieldPairs sorts the map by fields and return a sorted slice of FieldPair. -func NewFieldPairs(values map[string][]byte) FieldPairs { - pairs := make(FieldPairs, 0, len(values)) - for field, value := range values { - pairs = append(pairs, FieldPair{ - Field: field, - Value: value, - }) - } - - sort.Sort(pairs) - return pairs -} diff --git a/go-ycsb/pkg/util/core_test.go b/go-ycsb/pkg/util/core_test.go deleted file mode 100644 index 0bae2e2c6..000000000 --- a/go-ycsb/pkg/util/core_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package util - -import ( - "reflect" - "testing" -) - -func TestFieldPair(t *testing.T) { - m := map[string][]byte{ - "f2": []byte("b"), - "f1": []byte("a"), - } - - p := NewFieldPairs(m) - - check := FieldPairs{ - {"f1", []byte("a")}, - {"f2", []byte("b")}, - } - - if !reflect.DeepEqual(p, check) { - t.Errorf("want %v, but got %v", check, p) - } -} diff --git a/go-ycsb/pkg/util/hack.go b/go-ycsb/pkg/util/hack.go deleted file mode 100644 index 4cce8d695..000000000 --- a/go-ycsb/pkg/util/hack.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "reflect" - "unsafe" -) - -// String converts slice to string without copy. -// Use at your own risk. -func String(b []byte) (s string) { - if len(b) == 0 { - return "" - } - pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) - pstring.Data = pbytes.Data - pstring.Len = pbytes.Len - return -} - -// Slice converts string to slice without copy. -// Use at your own risk. -func Slice(s string) (b []byte) { - pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) - pbytes.Data = pstring.Data - pbytes.Len = pstring.Len - pbytes.Cap = pstring.Len - return -} diff --git a/go-ycsb/pkg/util/hash.go b/go-ycsb/pkg/util/hash.go deleted file mode 100644 index c7ee6138e..000000000 --- a/go-ycsb/pkg/util/hash.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "encoding/binary" - "hash/fnv" -) - -// Hash64 returns a fnv Hash of the integer. -func Hash64(n int64) int64 { - var b [8]byte - binary.BigEndian.PutUint64(b[0:8], uint64(n)) - hash := fnv.New64a() - hash.Write(b[0:8]) - result := int64(hash.Sum64()) - if result < 0 { - return -result - } - return result -} - -// BytesHash64 returns the fnv hash of a bytes -func BytesHash64(b []byte) int64 { - hash := fnv.New64a() - hash.Write(b) - return int64(hash.Sum64()) -} - -// StringHash64 returns the fnv hash of a string -func StringHash64(s string) int64 { - hash := fnv.New64a() - hash.Write(Slice(s)) - return int64(hash.Sum64()) -} diff --git a/go-ycsb/pkg/util/output.go b/go-ycsb/pkg/util/output.go deleted file mode 100644 index c982d6e5b..000000000 --- a/go-ycsb/pkg/util/output.go +++ /dev/null @@ -1,77 +0,0 @@ -package util - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "strings" - - "github.com/olekukonko/tablewriter" -) - -// output style -const ( - OutputStylePlain = "plain" - OutputStyleTable = "table" - OutputStyleJson = "json" -) - -// RenderString renders headers and values according to the format provided -func RenderString(w io.Writer, format string, headers []string, values [][]string) { - if len(values) == 0 { - return - } - - buf := new(bytes.Buffer) - for _, value := range values { - args := make([]string, len(headers)-1) - for i, header := range headers[1:] { - args[i] = header + ": " + value[i+1] - } - buf.WriteString(fmt.Sprintf(format, value[0], strings.Join(args, ", "))) - } - fmt.Fprint(w, buf.String()) -} - -// RenderTable will use given headers and values to render a table style output -func RenderTable(w io.Writer, headers []string, values [][]string) { - if len(values) == 0 { - return - } - tb := tablewriter.NewWriter(w) - tb.SetHeader(headers) - tb.AppendBulk(values) - tb.Render() -} - -// RnederJson will combine the headers and values and print a json string -func RenderJson(w io.Writer, headers []string, values [][]string) { - if len(values) == 0 { - return - } - data := make([]map[string]string, 0, len(values)) - for _, value := range values { - line := make(map[string]string, 0) - for i, header := range headers { - line[header] = value[i] - } - data = append(data, line) - } - outStr, err := json.Marshal(data) - if err != nil { - fmt.Fprintln(w, err) - return - } - fmt.Fprintln(w, string(outStr)) -} - -// IntToString formats int value to string -func IntToString(i interface{}) string { - return fmt.Sprintf("%d", i) -} - -// FloatToOneString formats float into string with one digit after dot -func FloatToOneString(f interface{}) string { - return fmt.Sprintf("%.1f", f) -} diff --git a/go-ycsb/pkg/util/row.go b/go-ycsb/pkg/util/row.go deleted file mode 100644 index c2ecd9417..000000000 --- a/go-ycsb/pkg/util/row.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "encoding/binary" - - "github.com/pingcap/errors" -) - -// EncodeRow encodes row data and column ids into a slice of byte. -// Row layout: colID1, value1, colID2, value2, ... -// valBuf and values pass by caller, for reducing EncodeRow allocates temporary bufs. If you pass valBuf and values as nil, -// EncodeRow will allocate it. -// It is a simplified and specialized version of `github.com/pingcap/tidb/tablecodec.EncodeRow`. -func EncodeRow(cols [][]byte, colIDs []int64, valBuf []byte) ([]byte, error) { - if len(cols) != len(colIDs) { - return nil, errors.Errorf("EncodeRow error: cols and colIDs count not match %d vs %d", len(cols), len(colIDs)) - } - valBuf = valBuf[:0] - if len(cols) == 0 { - return append(valBuf, 0), nil - } - for i := range cols { - valBuf = encodeInt64(valBuf, colIDs[i]) - valBuf = encodeBytes(valBuf, cols[i]) - } - return valBuf, nil -} - -const ( - compactBytesFlag byte = 2 - varintFlag byte = 8 -) - -func encodeInt64(b []byte, v int64) []byte { - b = append(b, varintFlag) - return appendVarint(b, v) -} - -func encodeBytes(b []byte, v []byte) []byte { - b = append(b, compactBytesFlag) - b = appendVarint(b, int64(len(v))) - return append(b, v...) -} - -func appendVarint(b []byte, v int64) []byte { - var data [binary.MaxVarintLen64]byte - n := binary.PutVarint(data[:], v) - return append(b, data[:n]...) -} - -// DecodeRow decodes a byte slice into columns. -// Row layout: colID1, value1, colID2, value2, ..... -// It is a simplified and specialized version of `github.com/pingcap/tidb/tablecodec.DecodeRow`. -func DecodeRow(b []byte) (map[int64][]byte, error) { - row := make(map[int64][]byte) - if len(b) == 0 { - return row, nil - } - if len(b) == 1 && b[0] == 0 { - return row, nil - } - for len(b) > 0 { - remain, rowID, err := decodeInt64(b) - if err != nil { - return row, err - } - var v []byte - remain, v, err = decodeBytes(remain) - if err != nil { - return row, err - } - row[rowID] = v - b = remain - } - return row, nil -} - -func decodeInt64(b []byte) ([]byte, int64, error) { - return decodeVarint(b[1:]) -} - -func decodeVarint(b []byte) ([]byte, int64, error) { - v, n := binary.Varint(b) - if n > 0 { - return b[n:], v, nil - } - if n < 0 { - return nil, 0, errors.New("value larger than 64 bits") - } - return nil, 0, errors.New("insufficient bytes to decode value") -} - -func decodeBytes(b []byte) ([]byte, []byte, error) { - remain, n, err := decodeVarint(b[1:]) - if err != nil { - return nil, nil, err - } - if int64(len(remain)) < n { - return nil, nil, errors.Errorf("insufficient bytes to decode value, expected length: %v", n) - } - return remain[n:], remain[:n], nil -} diff --git a/go-ycsb/pkg/util/row_test.go b/go-ycsb/pkg/util/row_test.go deleted file mode 100644 index 2f292f925..000000000 --- a/go-ycsb/pkg/util/row_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "bytes" - "testing" -) - -func TestCodec(t *testing.T) { - colIDs := []int64{1, 4, 7, 2, 5, 8} - cols := [][]byte{[]byte("147"), []byte("258"), []byte("147258"), []byte(""), []byte("258147"), []byte("369")} - - buf, err := EncodeRow(cols, colIDs, nil) - if err != nil { - t.Fatal(err) - } - row, err := DecodeRow(buf) - if err != nil { - t.Fatal(err) - } - for i, id := range colIDs { - if !bytes.Equal(cols[i], row[id]) { - t.Fatalf("id:%v, before:%q, after:%q", id, cols[i], row[id]) - } - } -} diff --git a/go-ycsb/pkg/util/spinlock.go b/go-ycsb/pkg/util/spinlock.go deleted file mode 100644 index a65a23558..000000000 --- a/go-ycsb/pkg/util/spinlock.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2014 OneOfOne -// https://github.com/OneOfOne/go-utils -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "runtime" - "sync/atomic" -) - -// SpinLock implements a simple atomic spin lock, the zero value for a SpinLock is an unlocked spinlock. -type SpinLock struct { - f uint32 -} - -// Lock locks sl. If the lock is already in use, the caller blocks until Unlock is called -func (sl *SpinLock) Lock() { - for !sl.TryLock() { - runtime.Gosched() //allow other goroutines to do stuff. - } -} - -// Unlock unlocks sl, unlike [Mutex.Unlock](http://golang.org/pkg/sync/#Mutex.Unlock), -// there's no harm calling it on an unlocked SpinLock -func (sl *SpinLock) Unlock() { - atomic.StoreUint32(&sl.f, 0) -} - -// TryLock will try to lock sl and return whether it succeed or not without blocking. -func (sl *SpinLock) TryLock() bool { - return atomic.CompareAndSwapUint32(&sl.f, 0, 1) -} - -func (sl *SpinLock) String() string { - if atomic.LoadUint32(&sl.f) == 1 { - return "Locked" - } - return "Unlocked" -} diff --git a/go-ycsb/pkg/util/tls.go b/go-ycsb/pkg/util/tls.go deleted file mode 100644 index 389218233..000000000 --- a/go-ycsb/pkg/util/tls.go +++ /dev/null @@ -1,59 +0,0 @@ -package util - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "io/ioutil" -) - -// CreateTLSConfig creats a TLS configuration. -func CreateTLSConfig(caPath, certPath, keyPath string, insecureSkipVerify bool) (*tls.Config, error) { - tlsConfig := &tls.Config{ - InsecureSkipVerify: insecureSkipVerify, - Renegotiation: tls.RenegotiateNever, - } - - if caPath != "" { - pool, err := makeCertPool([]string{caPath}) - if err != nil { - return nil, err - } - tlsConfig.RootCAs = pool - } - - if certPath != "" && keyPath != "" { - err := loadCertificate(tlsConfig, certPath, keyPath) - if err != nil { - return nil, err - } - } - - return tlsConfig, nil -} - -func makeCertPool(certFiles []string) (*x509.CertPool, error) { - pool := x509.NewCertPool() - for _, certFile := range certFiles { - pem, err := ioutil.ReadFile(certFile) - if err != nil { - return nil, fmt.Errorf("could not read certificate %q: %v", certFile, err) - } - ok := pool.AppendCertsFromPEM(pem) - if !ok { - return nil, fmt.Errorf("could not parse any PEM certificates %q: %v", certFile, err) - } - } - return pool, nil -} - -func loadCertificate(config *tls.Config, certFile, keyFile string) error { - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return fmt.Errorf("could not load keypair %s:%s: %v", certFile, keyFile, err) - } - - config.Certificates = []tls.Certificate{cert} - config.BuildNameToCertificate() - return nil -} diff --git a/go-ycsb/pkg/util/util.go b/go-ycsb/pkg/util/util.go deleted file mode 100644 index 2fed70072..000000000 --- a/go-ycsb/pkg/util/util.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "fmt" - "math/rand" - "os" - "sync" -) - -// Fatalf prints the message and exits the program. -func Fatalf(format string, args ...interface{}) { - fmt.Fprintf(os.Stderr, format, args...) - fmt.Println("") - os.Exit(1) -} - -// Fatal prints the message and exits the program. -func Fatal(args ...interface{}) { - fmt.Fprint(os.Stderr, args...) - fmt.Println("") - os.Exit(1) -} - -var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -// RandBytes fills the bytes with alphabetic characters randomly -func RandBytes(r *rand.Rand, b []byte) { - for i := range b { - b[i] = letters[r.Intn(len(letters))] - } -} - -// BufPool is a bytes.Buffer pool -type BufPool struct { - p *sync.Pool -} - -// NewBufPool creates a buffer pool. -func NewBufPool() *BufPool { - p := &sync.Pool{ - New: func() interface{} { - return []byte(nil) - }, - } - return &BufPool{ - p: p, - } -} - -// Get gets a buffer. -func (b *BufPool) Get() []byte { - buf := b.p.Get().([]byte) - buf = buf[:0] - return buf -} - -// Put returns a buffer. -func (b *BufPool) Put(buf []byte) { - b.p.Put(buf) -} diff --git a/go-ycsb/pkg/workload/core.go b/go-ycsb/pkg/workload/core.go deleted file mode 100644 index 1f0f2991c..000000000 --- a/go-ycsb/pkg/workload/core.go +++ /dev/null @@ -1,705 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package workload - -import ( - "bytes" - "context" - "fmt" - "math" - "math/rand" - "strconv" - "strings" - "sync" - "time" - - "github.com/magiconair/properties" - "github.com/pingcap/go-ycsb/pkg/generator" - "github.com/pingcap/go-ycsb/pkg/measurement" - "github.com/pingcap/go-ycsb/pkg/prop" - "github.com/pingcap/go-ycsb/pkg/util" - "github.com/pingcap/go-ycsb/pkg/ycsb" -) - -type contextKey string - -const stateKey = contextKey("core") - -type coreState struct { - r *rand.Rand - // fieldNames is a copy of core.fieldNames to be goroutine-local - fieldNames []string -} - -type operationType int64 - -const ( - read operationType = iota + 1 - update - insert - scan - readModifyWrite -) - -// Core is the core benchmark scenario. Represents a set of clients doing simple CRUD operations. -type core struct { - p *properties.Properties - - table string - fieldCount int64 - fieldNames []string - - fieldLengthGenerator ycsb.Generator - readAllFields bool - writeAllFields bool - dataIntegrity bool - - keySequence ycsb.Generator - operationChooser *generator.Discrete - keyChooser ycsb.Generator - fieldChooser ycsb.Generator - transactionInsertKeySequence *generator.AcknowledgedCounter - scanLength ycsb.Generator - orderedInserts bool - recordCount int64 - zeroPadding int64 - insertionRetryLimit int64 - insertionRetryInterval int64 - - valuePool sync.Pool -} - -func getFieldLengthGenerator(p *properties.Properties) ycsb.Generator { - var fieldLengthGenerator ycsb.Generator - fieldLengthDistribution := p.GetString(prop.FieldLengthDistribution, prop.FieldLengthDistributionDefault) - fieldLength := p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) - fieldLengthHistogram := p.GetString(prop.FieldLengthHistogramFile, prop.FieldLengthHistogramFileDefault) - - switch strings.ToLower(fieldLengthDistribution) { - case "constant": - fieldLengthGenerator = generator.NewConstant(fieldLength) - case "uniform": - fieldLengthGenerator = generator.NewUniform(1, fieldLength) - case "zipfian": - fieldLengthGenerator = generator.NewZipfianWithRange(1, fieldLength, generator.ZipfianConstant) - case "histogram": - fieldLengthGenerator = generator.NewHistogramFromFile(fieldLengthHistogram) - default: - util.Fatalf("unknown field length distribution %s", fieldLengthDistribution) - } - - return fieldLengthGenerator -} - -func createOperationGenerator(p *properties.Properties) *generator.Discrete { - readProportion := p.GetFloat64(prop.ReadProportion, prop.ReadProportionDefault) - updateProportion := p.GetFloat64(prop.UpdateProportion, prop.UpdateProportionDefault) - insertProportion := p.GetFloat64(prop.InsertProportion, prop.InsertProportionDefault) - scanProportion := p.GetFloat64(prop.ScanProportion, prop.ScanProportionDefault) - readModifyWriteProportion := p.GetFloat64(prop.ReadModifyWriteProportion, prop.ReadModifyWriteProportionDefault) - - operationChooser := generator.NewDiscrete() - if readProportion > 0 { - operationChooser.Add(readProportion, int64(read)) - } - - if updateProportion > 0 { - operationChooser.Add(updateProportion, int64(update)) - } - - if insertProportion > 0 { - operationChooser.Add(insertProportion, int64(insert)) - } - - if scanProportion > 0 { - operationChooser.Add(scanProportion, int64(scan)) - } - - if readModifyWriteProportion > 0 { - operationChooser.Add(readModifyWriteProportion, int64(readModifyWrite)) - } - - return operationChooser -} - -// Load implements the Workload Load interface. -func (c *core) Load(ctx context.Context, db ycsb.DB, totalCount int64) error { - return nil -} - -// InitThread implements the Workload InitThread interface. -func (c *core) InitThread(ctx context.Context, _ int, _ int) context.Context { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - fieldNames := make([]string, len(c.fieldNames)) - copy(fieldNames, c.fieldNames) - state := &coreState{ - r: r, - fieldNames: fieldNames, - } - return context.WithValue(ctx, stateKey, state) -} - -// CleanupThread implements the Workload CleanupThread interface. -func (c *core) CleanupThread(_ context.Context) { - -} - -// Close implements the Workload Close interface. -func (c *core) Close() error { - return nil -} - -func (c *core) buildKeyName(keyNum int64) string { - if !c.orderedInserts { - keyNum = util.Hash64(keyNum) - } - - prefix := c.p.GetString(prop.KeyPrefix, prop.KeyPrefixDefault) - return fmt.Sprintf("%s%0[3]*[2]d", prefix, keyNum, c.zeroPadding) -} - -func (c *core) buildSingleValue(state *coreState, key string) map[string][]byte { - values := make(map[string][]byte, 1) - - r := state.r - fieldKey := state.fieldNames[c.fieldChooser.Next(r)] - - var buf []byte - if c.dataIntegrity { - buf = c.buildDeterministicValue(state, key, fieldKey) - } else { - buf = c.buildRandomValue(state) - } - - values[fieldKey] = buf - - return values -} - -func (c *core) buildValues(state *coreState, key string) map[string][]byte { - values := make(map[string][]byte, c.fieldCount) - - for _, fieldKey := range state.fieldNames { - var buf []byte - if c.dataIntegrity { - buf = c.buildDeterministicValue(state, key, fieldKey) - } else { - buf = c.buildRandomValue(state) - } - - values[fieldKey] = buf - } - return values -} - -func (c *core) getValueBuffer(size int) []byte { - buf := c.valuePool.Get().([]byte) - if cap(buf) >= size { - return buf[0:size] - } - - return make([]byte, size) -} - -func (c *core) putValues(values map[string][]byte) { - for _, value := range values { - c.valuePool.Put(value) - } -} - -func (c *core) buildRandomValue(state *coreState) []byte { - // TODO: use pool for the buffer - r := state.r - buf := c.getValueBuffer(int(c.fieldLengthGenerator.Next(r))) - util.RandBytes(r, buf) - return buf -} - -func (c *core) buildDeterministicValue(state *coreState, key string, fieldKey string) []byte { - // TODO: use pool for the buffer - r := state.r - size := c.fieldLengthGenerator.Next(r) - buf := c.getValueBuffer(int(size + 21)) - b := bytes.NewBuffer(buf[0:0]) - b.WriteString(key) - b.WriteByte(':') - b.WriteString(strings.ToLower(fieldKey)) - for int64(b.Len()) < size { - b.WriteByte(':') - n := util.BytesHash64(b.Bytes()) - b.WriteString(strconv.FormatUint(uint64(n), 10)) - } - b.Truncate(int(size)) - return b.Bytes() -} - -func (c *core) verifyRow(state *coreState, key string, values map[string][]byte) { - if len(values) == 0 { - // null data here, need panic? - return - } - - for fieldKey, value := range values { - expected := c.buildDeterministicValue(state, key, fieldKey) - if !bytes.Equal(expected, value) { - util.Fatalf("unexpected deterministic value, expect %q, but got %q", expected, value) - } - } -} - -// DoInsert implements the Workload DoInsert interface. -func (c *core) DoInsert(ctx context.Context, db ycsb.DB) error { - state := ctx.Value(stateKey).(*coreState) - r := state.r - keyNum := c.keySequence.Next(r) - dbKey := c.buildKeyName(keyNum) - values := c.buildValues(state, dbKey) - defer c.putValues(values) - - numOfRetries := int64(0) - - var err error - for { - err = db.Insert(ctx, c.table, dbKey, values) - if err == nil { - break - } - - select { - case <-ctx.Done(): - if ctx.Err() == context.Canceled { - return nil - } - default: - } - - // Retry if configured. Without retrying, the load process will fail - // even if one single insertion fails. User can optionally configure - // an insertion retry limit (default is 0) to enable retry. - numOfRetries++ - if numOfRetries > c.insertionRetryLimit { - break - } - - // Sleep for a random time betweensz [0.8, 1.2)*insertionRetryInterval - sleepTimeMs := float64((c.insertionRetryInterval * 1000)) * (0.8 + 0.4*r.Float64()) - - time.Sleep(time.Duration(sleepTimeMs) * time.Millisecond) - } - - return err -} - -// DoBatchInsert implements the Workload DoBatchInsert interface. -func (c *core) DoBatchInsert(ctx context.Context, batchSize int, db ycsb.DB) error { - batchDB, ok := db.(ycsb.BatchDB) - if !ok { - return fmt.Errorf("the %T does't implement the batchDB interface", db) - } - state := ctx.Value(stateKey).(*coreState) - r := state.r - var keys []string - var values []map[string][]byte - for i := 0; i < batchSize; i++ { - keyNum := c.keySequence.Next(r) - dbKey := c.buildKeyName(keyNum) - keys = append(keys, dbKey) - values = append(values, c.buildValues(state, dbKey)) - } - defer func() { - for _, value := range values { - c.putValues(value) - } - }() - - numOfRetries := int64(0) - var err error - for { - err = batchDB.BatchInsert(ctx, c.table, keys, values) - if err == nil { - break - } - - select { - case <-ctx.Done(): - if ctx.Err() == context.Canceled { - return nil - } - default: - } - - // Retry if configured. Without retrying, the load process will fail - // even if one single insertion fails. User can optionally configure - // an insertion retry limit (default is 0) to enable retry. - numOfRetries++ - if numOfRetries > c.insertionRetryLimit { - break - } - - // Sleep for a random time betweensz [0.8, 1.2)*insertionRetryInterval - sleepTimeMs := float64((c.insertionRetryInterval * 1000)) * (0.8 + 0.4*r.Float64()) - - time.Sleep(time.Duration(sleepTimeMs) * time.Millisecond) - } - return err -} - -// DoTransaction implements the Workload DoTransaction interface. -func (c *core) DoTransaction(ctx context.Context, db ycsb.DB) error { - state := ctx.Value(stateKey).(*coreState) - r := state.r - - operation := operationType(c.operationChooser.Next(r)) - switch operation { - case read: - return c.doTransactionRead(ctx, db, state) - case update: - return c.doTransactionUpdate(ctx, db, state) - case insert: - return c.doTransactionInsert(ctx, db, state) - case scan: - return c.doTransactionScan(ctx, db, state) - default: - return c.doTransactionReadModifyWrite(ctx, db, state) - } -} - -// DoBatchTransaction implements the Workload DoBatchTransaction interface -func (c *core) DoBatchTransaction(ctx context.Context, batchSize int, db ycsb.DB) error { - batchDB, ok := db.(ycsb.BatchDB) - if !ok { - return fmt.Errorf("the %T does't implement the batchDB interface", db) - } - state := ctx.Value(stateKey).(*coreState) - r := state.r - - operation := operationType(c.operationChooser.Next(r)) - switch operation { - case read: - return c.doBatchTransactionRead(ctx, batchSize, batchDB, state) - case insert: - return c.doBatchTransactionInsert(ctx, batchSize, batchDB, state) - case update: - return c.doBatchTransactionUpdate(ctx, batchSize, batchDB, state) - case scan: - panic("The batch mode don't support the scan operation") - default: - return nil - } -} - -func (c *core) nextKeyNum(state *coreState) int64 { - r := state.r - keyNum := int64(0) - if _, ok := c.keyChooser.(*generator.Exponential); ok { - keyNum = -1 - for keyNum < 0 { - keyNum = c.transactionInsertKeySequence.Last() - c.keyChooser.Next(r) - } - } else { - keyNum = c.keyChooser.Next(r) - } - return keyNum -} - -func (c *core) doTransactionRead(ctx context.Context, db ycsb.DB, state *coreState) error { - r := state.r - keyNum := c.nextKeyNum(state) - keyName := c.buildKeyName(keyNum) - - var fields []string - if !c.readAllFields { - fieldName := state.fieldNames[c.fieldChooser.Next(r)] - fields = append(fields, fieldName) - } else { - fields = state.fieldNames - } - - values, err := db.Read(ctx, c.table, keyName, fields) - if err != nil { - return err - } - - if c.dataIntegrity { - c.verifyRow(state, keyName, values) - } - - return nil -} - -func (c *core) doTransactionReadModifyWrite(ctx context.Context, db ycsb.DB, state *coreState) error { - start := time.Now() - defer func() { - measurement.Measure("READ_MODIFY_WRITE", start, time.Now().Sub(start)) - }() - - r := state.r - keyNum := c.nextKeyNum(state) - keyName := c.buildKeyName(keyNum) - - var fields []string - if !c.readAllFields { - fieldName := state.fieldNames[c.fieldChooser.Next(r)] - fields = append(fields, fieldName) - } else { - fields = state.fieldNames - } - - var values map[string][]byte - if c.writeAllFields { - values = c.buildValues(state, keyName) - } else { - values = c.buildSingleValue(state, keyName) - } - defer c.putValues(values) - - readValues, err := db.Read(ctx, c.table, keyName, fields) - if err != nil { - return err - } - - if err := db.Update(ctx, c.table, keyName, values); err != nil { - return err - } - - if c.dataIntegrity { - c.verifyRow(state, keyName, readValues) - } - - return nil -} - -func (c *core) doTransactionInsert(ctx context.Context, db ycsb.DB, state *coreState) error { - r := state.r - keyNum := c.transactionInsertKeySequence.Next(r) - defer c.transactionInsertKeySequence.Acknowledge(keyNum) - dbKey := c.buildKeyName(keyNum) - values := c.buildValues(state, dbKey) - defer c.putValues(values) - - return db.Insert(ctx, c.table, dbKey, values) -} - -func (c *core) doTransactionScan(ctx context.Context, db ycsb.DB, state *coreState) error { - r := state.r - keyNum := c.nextKeyNum(state) - startKeyName := c.buildKeyName(keyNum) - - scanLen := c.scanLength.Next(r) - - var fields []string - if !c.readAllFields { - fieldName := state.fieldNames[c.fieldChooser.Next(r)] - fields = append(fields, fieldName) - } else { - fields = state.fieldNames - } - - _, err := db.Scan(ctx, c.table, startKeyName, int(scanLen), fields) - - return err -} - -func (c *core) doTransactionUpdate(ctx context.Context, db ycsb.DB, state *coreState) error { - keyNum := c.nextKeyNum(state) - keyName := c.buildKeyName(keyNum) - - var values map[string][]byte - if c.writeAllFields { - values = c.buildValues(state, keyName) - } else { - values = c.buildSingleValue(state, keyName) - } - - defer c.putValues(values) - - return db.Update(ctx, c.table, keyName, values) -} - -func (c *core) doBatchTransactionRead(ctx context.Context, batchSize int, db ycsb.BatchDB, state *coreState) error { - r := state.r - var fields []string - - if !c.readAllFields { - fieldName := state.fieldNames[c.fieldChooser.Next(r)] - fields = append(fields, fieldName) - } else { - fields = state.fieldNames - } - - keys := make([]string, batchSize) - for i := 0; i < batchSize; i++ { - keys[i] = c.buildKeyName(c.nextKeyNum(state)) - } - - _, err := db.BatchRead(ctx, c.table, keys, fields) - if err != nil { - return err - } - - // TODO should we verify the result? - return nil -} - -func (c *core) doBatchTransactionInsert(ctx context.Context, batchSize int, db ycsb.BatchDB, state *coreState) error { - r := state.r - keys := make([]string, batchSize) - values := make([]map[string][]byte, batchSize) - for i := 0; i < batchSize; i++ { - keyNum := c.transactionInsertKeySequence.Next(r) - keyName := c.buildKeyName(keyNum) - keys[i] = keyName - if c.writeAllFields { - values[i] = c.buildValues(state, keyName) - } else { - values[i] = c.buildSingleValue(state, keyName) - } - c.transactionInsertKeySequence.Acknowledge(keyNum) - } - - defer func() { - for _, value := range values { - c.putValues(value) - } - }() - - return db.BatchInsert(ctx, c.table, keys, values) -} - -func (c *core) doBatchTransactionUpdate(ctx context.Context, batchSize int, db ycsb.BatchDB, state *coreState) error { - keys := make([]string, batchSize) - values := make([]map[string][]byte, batchSize) - for i := 0; i < batchSize; i++ { - keyNum := c.nextKeyNum(state) - keyName := c.buildKeyName(keyNum) - keys[i] = keyName - if c.writeAllFields { - values[i] = c.buildValues(state, keyName) - } else { - values[i] = c.buildSingleValue(state, keyName) - } - } - - defer func() { - for _, value := range values { - c.putValues(value) - } - }() - - return db.BatchUpdate(ctx, c.table, keys, values) -} - -// CoreCreator creates the Core workload. -type coreCreator struct { -} - -// Create implements the WorkloadCreator Create interface. -func (coreCreator) Create(p *properties.Properties) (ycsb.Workload, error) { - c := new(core) - c.p = p - c.table = p.GetString(prop.TableName, prop.TableNameDefault) - c.fieldCount = p.GetInt64(prop.FieldCount, prop.FieldCountDefault) - c.fieldNames = make([]string, c.fieldCount) - for i := int64(0); i < c.fieldCount; i++ { - c.fieldNames[i] = fmt.Sprintf("field%d", i) - } - c.fieldLengthGenerator = getFieldLengthGenerator(p) - c.recordCount = p.GetInt64(prop.RecordCount, prop.RecordCountDefault) - if c.recordCount == 0 { - c.recordCount = int64(math.MaxInt32) - } - - requestDistrib := p.GetString(prop.RequestDistribution, prop.RequestDistributionDefault) - maxScanLength := p.GetInt64(prop.MaxScanLength, prop.MaxScanLengthDefault) - scanLengthDistrib := p.GetString(prop.ScanLengthDistribution, prop.ScanLengthDistributionDefault) - - insertStart := p.GetInt64(prop.InsertStart, prop.InsertStartDefault) - insertCount := p.GetInt64(prop.InsertCount, c.recordCount-insertStart) - if c.recordCount < insertStart+insertCount { - util.Fatalf("record count %d must be bigger than insert start %d + count %d", - c.recordCount, insertStart, insertCount) - } - c.zeroPadding = p.GetInt64(prop.ZeroPadding, prop.ZeroPaddingDefault) - c.readAllFields = p.GetBool(prop.ReadAllFields, prop.ReadALlFieldsDefault) - c.writeAllFields = p.GetBool(prop.WriteAllFields, prop.WriteAllFieldsDefault) - c.dataIntegrity = p.GetBool(prop.DataIntegrity, prop.DataIntegrityDefault) - fieldLengthDistribution := p.GetString(prop.FieldLengthDistribution, prop.FieldLengthDistributionDefault) - if c.dataIntegrity && fieldLengthDistribution != "constant" { - util.Fatal("must have constant field size to check data integrity") - } - - if p.GetString(prop.InsertOrder, prop.InsertOrderDefault) == "hashed" { - c.orderedInserts = false - } else { - c.orderedInserts = true - } - - c.keySequence = generator.NewCounter(insertStart) - c.operationChooser = createOperationGenerator(p) - var keyrangeLowerBound int64 = insertStart - var keyrangeUpperBound int64 = insertStart + insertCount - 1 - - c.transactionInsertKeySequence = generator.NewAcknowledgedCounter(c.recordCount) - switch requestDistrib { - case "uniform": - c.keyChooser = generator.NewUniform(keyrangeLowerBound, keyrangeUpperBound) - case "sequential": - c.keyChooser = generator.NewSequential(keyrangeLowerBound, keyrangeUpperBound) - case "zipfian": - insertProportion := p.GetFloat64(prop.InsertProportion, prop.InsertProportionDefault) - opCount := p.GetInt64(prop.OperationCount, 0) - expectedNewKeys := int64(float64(opCount) * insertProportion * 2.0) - keyrangeUpperBound = insertStart + insertCount + expectedNewKeys - c.keyChooser = generator.NewScrambledZipfian(keyrangeLowerBound, keyrangeUpperBound, generator.ZipfianConstant) - case "latest": - c.keyChooser = generator.NewSkewedLatest(c.transactionInsertKeySequence) - case "hotspot": - hotsetFraction := p.GetFloat64(prop.HotspotDataFraction, prop.HotspotDataFractionDefault) - hotopnFraction := p.GetFloat64(prop.HotspotOpnFraction, prop.HotspotOpnFractionDefault) - c.keyChooser = generator.NewHotspot(keyrangeLowerBound, keyrangeUpperBound, hotsetFraction, hotopnFraction) - case "exponential": - percentile := p.GetFloat64(prop.ExponentialPercentile, prop.ExponentialPercentileDefault) - frac := p.GetFloat64(prop.ExponentialFrac, prop.ExponentialFracDefault) - c.keyChooser = generator.NewExponential(percentile, float64(c.recordCount)*frac) - default: - util.Fatalf("unknown request distribution %s", requestDistrib) - } - fmt.Println(fmt.Sprintf("Using request distribution '%s' a keyrange of [%d %d]", requestDistrib, keyrangeLowerBound, keyrangeUpperBound)) - - c.fieldChooser = generator.NewUniform(0, c.fieldCount-1) - switch scanLengthDistrib { - case "uniform": - c.scanLength = generator.NewUniform(1, maxScanLength) - case "zipfian": - c.scanLength = generator.NewZipfianWithRange(1, maxScanLength, generator.ZipfianConstant) - default: - util.Fatalf("distribution %s not allowed for scan length", scanLengthDistrib) - } - - c.insertionRetryLimit = p.GetInt64(prop.InsertionRetryLimit, prop.InsertionRetryLimitDefault) - c.insertionRetryInterval = p.GetInt64(prop.InsertionRetryInterval, prop.InsertionRetryIntervalDefault) - - fieldLength := p.GetInt64(prop.FieldLength, prop.FieldLengthDefault) - c.valuePool = sync.Pool{ - New: func() interface{} { - return make([]byte, fieldLength) - }, - } - - return c, nil -} - -func init() { - ycsb.RegisterWorkloadCreator("core", coreCreator{}) -} diff --git a/go-ycsb/pkg/ycsb/db.go b/go-ycsb/pkg/ycsb/db.go deleted file mode 100644 index c4545c835..000000000 --- a/go-ycsb/pkg/ycsb/db.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package ycsb - -import ( - "context" - "fmt" - - "github.com/magiconair/properties" -) - -// DBCreator creates a database layer. -type DBCreator interface { - Create(p *properties.Properties) (DB, error) -} - -// DB is the layer to access the database to be benchmarked. -type DB interface { - // Close closes the database layer. - Close() error - - // InitThread initializes the state associated to the goroutine worker. - // The Returned context will be passed to the following usage. - InitThread(ctx context.Context, threadID int, threadCount int) context.Context - - // CleanupThread cleans up the state when the worker finished. - CleanupThread(ctx context.Context) - - // Read reads a record from the database and returns a map of each field/value pair. - // table: The name of the table. - // key: The record key of the record to read. - // fields: The list of fields to read, nil|empty for reading all. - Read(ctx context.Context, table string, key string, fields []string) (map[string][]byte, error) - - // Scan scans records from the database. - // table: The name of the table. - // startKey: The first record key to read. - // count: The number of records to read. - // fields: The list of fields to read, nil|empty for reading all. - Scan(ctx context.Context, table string, startKey string, count int, fields []string) ([]map[string][]byte, error) - - // Update updates a record in the database. Any field/value pairs will be written into the - // database or overwritten the existing values with the same field name. - // table: The name of the table. - // key: The record key of the record to update. - // values: A map of field/value pairs to update in the record. - Update(ctx context.Context, table string, key string, values map[string][]byte) error - - // Insert inserts a record in the database. Any field/value pairs will be written into the - // database. - // table: The name of the table. - // key: The record key of the record to insert. - // values: A map of field/value pairs to insert in the record. - Insert(ctx context.Context, table string, key string, values map[string][]byte) error - - // Delete deletes a record from the database. - // table: The name of the table. - // key: The record key of the record to delete. - Delete(ctx context.Context, table string, key string) error -} - -type BatchDB interface { - // BatchInsert inserts batch records in the database. - // table: The name of the table. - // keys: The keys of batch records. - // values: The values of batch records. - BatchInsert(ctx context.Context, table string, keys []string, values []map[string][]byte) error - - // BatchRead reads records from the database. - // table: The name of the table. - // keys: The keys of records to read. - // fields: The list of fields to read, nil|empty for reading all. - BatchRead(ctx context.Context, table string, keys []string, fields []string) ([]map[string][]byte, error) - - // BatchUpdate updates records in the database. - // table: The name of table. - // keys: The keys of records to update. - // values: The values of records to update. - BatchUpdate(ctx context.Context, table string, keys []string, values []map[string][]byte) error - - // BatchDelete deletes records from the database. - // table: The name of the table. - // keys: The keys of the records to delete. - BatchDelete(ctx context.Context, table string, keys []string) error -} - -// AnalyzeDB is the interface for the DB that can perform an analysis on given table. -type AnalyzeDB interface { - // Analyze performs a key distribution analysis for the table. - // table: The name of the table. - Analyze(ctx context.Context, table string) error -} - -var dbCreators = map[string]DBCreator{} - -// RegisterDBCreator registers a creator for the database -func RegisterDBCreator(name string, creator DBCreator) { - _, ok := dbCreators[name] - if ok { - panic(fmt.Sprintf("duplicate register database %s", name)) - } - - dbCreators[name] = creator -} - -// GetDBCreator gets the DBCreator for the database -func GetDBCreator(name string) DBCreator { - return dbCreators[name] -} diff --git a/go-ycsb/pkg/ycsb/generator.go b/go-ycsb/pkg/ycsb/generator.go deleted file mode 100644 index cce4cf6cc..000000000 --- a/go-ycsb/pkg/ycsb/generator.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package ycsb - -import "math/rand" - -// Generator generates a sequence of values, following some distribution (Uniform, Zipfian, etc.). -type Generator interface { - // Next generates the next value in the distribution. - Next(r *rand.Rand) int64 - - // Last returns the previous value generated by the distribution, e.g. the - // last Next call. - // Calling Last should not advance the distribution or have any side effect. - // If Next has not been called, Last should return something reasonable. - Last() int64 -} diff --git a/go-ycsb/pkg/ycsb/measurement.go b/go-ycsb/pkg/ycsb/measurement.go deleted file mode 100644 index 60e4c46c8..000000000 --- a/go-ycsb/pkg/ycsb/measurement.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package ycsb - -import ( - "io" - "time" -) - -// Measurer is used to capture measurements. -type Measurer interface { - // Measure measures the latency of an operation. - Measure(op string, start time.Time, latency time.Duration) - - // Summary writes a summary of the current measurement results to stdout. - Summary() - - // Output writes the measurement results to the specified writer. - Output(w io.Writer) error -} diff --git a/go-ycsb/pkg/ycsb/workload.go b/go-ycsb/pkg/ycsb/workload.go deleted file mode 100644 index 7be59ed4a..000000000 --- a/go-ycsb/pkg/ycsb/workload.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package ycsb - -import ( - "context" - "fmt" - - "github.com/magiconair/properties" -) - -// WorkloadCreator creates a Workload -type WorkloadCreator interface { - Create(p *properties.Properties) (Workload, error) -} - -// Workload defines different workload for YCSB. -type Workload interface { - // Close closes the workload. - Close() error - - // InitThread initializes the state associated to the goroutine worker. - // The Returned context will be passed to the following DoInsert and DoTransaction. - InitThread(ctx context.Context, threadID int, threadCount int) context.Context - - // CleanupThread cleans up the state when the worker finished. - CleanupThread(ctx context.Context) - - // Load data into DB. - Load(ctx context.Context, db DB, totalCount int64) error - - // DoInsert does one insert operation. - DoInsert(ctx context.Context, db DB) error - - // DoBatchInsert does batch insert. - DoBatchInsert(ctx context.Context, batchSize int, db DB) error - - // DoTransaction does one transaction operation. - DoTransaction(ctx context.Context, db DB) error - - // DoBatchTransaction does the batch transaction operation. - DoBatchTransaction(ctx context.Context, batchSize int, db DB) error -} - -var workloadCreators = map[string]WorkloadCreator{} - -// RegisterWorkloadCreator registers a creator for the workload -func RegisterWorkloadCreator(name string, creator WorkloadCreator) { - _, ok := workloadCreators[name] - if ok { - panic(fmt.Sprintf("duplicate register workload %s", name)) - } - - workloadCreators[name] = creator -} - -// GetWorkloadCreator gets the WorkloadCreator for the database -func GetWorkloadCreator(name string) WorkloadCreator { - return workloadCreators[name] -} diff --git a/go-ycsb/tool/binary/bench.sh b/go-ycsb/tool/binary/bench.sh deleted file mode 100755 index 864a3084f..000000000 --- a/go-ycsb/tool/binary/bench.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -TYPE=$1 -DB=$2 - -# Directory to save data -DATA=./data -CMD=../../bin/go-ycsb -# Direcotry to save logs -LOG=./logs - -RECORDCOUNT=100000000 -OPERATIONCOUNT=100000000 -THREADCOUNT=16 -FIELDCOUNT=10 -FIELDLENGTH=100 -MAXSCANLENGTH=10 - -PROPS="-p recordcount=${RECORDCOUNT} \ - -p operationcount=${OPERATIONCOUNT} \ - -p threadcount=${THREADCOUNT} \ - -p fieldcount=${FIELDCOUNT} \ - -p fieldlength=${FIELDLENGTH} \ - -p maxscanlength=${MAXSCANLENGTH}" -PROPS+=" ${@:3}" -WORKLOADS= - -mkdir -p ${LOG} - -DBDATA=${DATA}/${DB} - -if [ ${DB} == 'rocksdb' ]; then - PROPS+=" -p rocksdb.dir=${DBDATA}" - WORKLOADS="-P property/rocksdb" -elif [ ${DB} == 'badger' ]; then - PROPS+=" -p badger.dir=${DBDATA}" - WORKLOADS="-P property/badger" -fi - - -if [ ${TYPE} == 'load' ]; then - echo "clear data before load" - PROPS+=" -p dropdata=true" -fi - -echo ${TYPE} ${DB} ${WORKLOADS} ${PROPS} - -if [ ${TYPE} == 'load' ]; then - $CMD load ${DB} ${WORKLOADS} -p=workload=core ${PROPS} | tee ${LOG}/${DB}_load.log -elif [ ${TYPE} == 'run' ]; then - for workload in a b c d e f - do - $CMD run ${DB} -P ../../workloads/workload${workload} ${WORKLOADS} ${PROPS} | tee ${LOG}/${DB}_workload${workload}.log - done -else - echo "invalid type ${TYPE}" - exit 1 -fi - diff --git a/go-ycsb/tool/binary/property/badger b/go-ycsb/tool/binary/property/badger deleted file mode 100644 index 3028b9214..000000000 --- a/go-ycsb/tool/binary/property/badger +++ /dev/null @@ -1,16 +0,0 @@ -badger.sync_writes = false -badger.num_versions_to_keep = 1 -badger.max_table_size = 67108864 -badger.level_size_multiplier = 10 -badger.max_levels = 7 -badger.value_threshold = 32 -badger.num_memtables = 5 -badger.num_level0_tables = 5 -badger.num_level0_tables_stall = 10 -badger.level_one_size = 268435456 -badger.value_log_file_size = 1073741824 -badger.value_log_max_entries = 1000000 -badger.num_compactors = 3 -badger.do_not_compact = false -badger.table_loading_mode = LoadToRAM -badger.value_log_loading_mode = MemoryMap \ No newline at end of file diff --git a/go-ycsb/tool/binary/property/rocksdb b/go-ycsb/tool/binary/property/rocksdb deleted file mode 100644 index 778addd9e..000000000 --- a/go-ycsb/tool/binary/property/rocksdb +++ /dev/null @@ -1,27 +0,0 @@ -rocksdb.allow_concurrent_memtable_writes = true -rocksdb.allow_mmap_reads = false -rocksdb.allow_mmap_writes = false -rocksdb.arena_block_size = 0 -rocksdb.db_write_buffer_size = 0 -rocksdb.hard_pending_compaction_bytes_limit = 274877906944 -rocksdb.level0_file_num_compaction_trigger = 4 -rocksdb.level0_slowdown_writes_trigger = 20 -rocksdb.level0_stop_writes_trigger = 36 -rocksdb.max_background_flushes = 1 -rocksdb.max_bytes_for_level_base = 268435456 -rocksdb.max_bytes_for_level_multiplier = 10 -rocksdb.max_total_wal_size = 0 -rocksdb.memtable_huge_page_size = 0 -rocksdb.num_levels = 7 -rocksdb.use_direct_reads = false -rocksdb.use_fsync = false -rocksdb.write_buffer_size = 67108864 - -# TableOptions/BlockBasedTable -rocksdb.block_size = 4096 -rocksdb.block_size_deviation = 10 -rocksdb.cache_index_and_filter_blocks = false -rocksdb.no_block_cache = false -rocksdb.pin_l0_filter_and_index_blocks_in_cache = false -rocksdb.whole_key_filtering = true -rocksdb.block_restart_interval = 16 diff --git a/go-ycsb/tool/docker/bench.sh b/go-ycsb/tool/docker/bench.sh deleted file mode 100755 index 0dea3c38a..000000000 --- a/go-ycsb/tool/docker/bench.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -TYPE=$1 -DB=$2 - -# Direcotry to save logs -LOG=./logs - -RECORDCOUNT=100000 -OPERATIONCOUNT=100000 -THREADCOUNT=20 -FIELDCOUNT=5 -FIELDLENGTH=16 -MAXSCANLENGTH=10 - -PROPS="-p recordcount=${RECORDCOUNT} \ - -p operationcount=${OPERATIONCOUNT} \ - -p threadcount=${THREADCOUNT} \ - -p fieldcount=${FIELDCOUNT} \ - -p fieldlength=${FIELDLENGTH} \ - -p maxscanlength=${MAXSCANLENGTH}" -PROPS+=" ${@:3}" -WORKLOADS= -SLEEPTIME=10 - -mkdir -p ${LOG} - -BENCH_DB=${DB} - -case ${DB} in - mysql) - PROPS+=" -p mysql.host=mysql" - SLEEPTIME=30 - ;; - mysql8) - PROPS+=" -p mysql.host=mysql" - SLEEPTIME=30 - DB="mysql" - ;; - mariadb) - PROPS+=" -p mysql.host=mariadb" - SLEEPTIME=60 - DB="mysql" - ;; - pg) - PROPS+=" -p pg.host=pg" - SLEEPTIME=30 - ;; - tikv) - PROPS+=" -p tikv.pd=pd:2379 -p tikv.type=txn" - ;; - raw) - PROPS+=" -p tikv.pd=pd:2379 -p tikv.type=raw" - DB="tikv" - ;; - tidb) - PROPS+=" -p mysql.host=tidb -p mysql.port=4000" - ;; - cockroach) - PROPS+=" -p pg.host=cockroach -p pg.port=26257" - ;; - sqlite) - PROPS+=" -p sqlite.db=/data/sqlite.db" - ;; - cassandra) - PROPS+=" -p cassandra.cluster=cassandra" - SLEEPTIME=30 - ;; - scylla) - PROPS+=" -p cassandra.cluster=scylla" - SLEEPTIME=30 - ;; - *) - ;; -esac - -echo ${TYPE} ${DB} ${WORKLOADS} ${PROPS} - -CMD="docker-compose -f ${BENCH_DB}.yml" - -if [ ${TYPE} == 'load' ]; then - $CMD down --remove-orphans - rm -rf ./data/${BENCH_DB} - $CMD up -d - sleep ${SLEEPTIME} - - $CMD run ycsb load ${DB} ${WORKLOADS} -p workload=core ${PROPS} | tee ${LOG}/${BENCH_DB}_load.log - - $CMD down -elif [ ${TYPE} == 'run' ]; then - $CMD up -d - sleep ${SLEEPTIME} - - for workload in a b c d e f - do - $CMD run --rm ycsb run ${DB} -P ../../workloads/workload${workload} ${WORKLOADS} ${PROPS} | tee ${LOG}/${BENCH_DB}_workload${workload}.log - done - - $CMD down -else - echo "invalid type ${TYPE}" - exit 1 -fi diff --git a/go-ycsb/tool/docker/cassandra.yml b/go-ycsb/tool/docker/cassandra.yml deleted file mode 100644 index 0e43a65a4..000000000 --- a/go-ycsb/tool/docker/cassandra.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: '2.1' - -services: - cassandra: - image: cassandra:latest - volumes: - - ./data/cassandra:/var/lib/cassandra - ports: - - "9042:9042" - restart: on-failure - - db-init: - image: cassandra:latest - volumes: - - ./config/init_cassandra.sh:/init_cassandra.sh - entrypoint: - - "/bin/bash" - command: /init_cassandra.sh cassandra - depends_on: - - "cassandra" - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/clear.sh b/go-ycsb/tool/docker/clear.sh deleted file mode 100755 index ddf27ad2f..000000000 --- a/go-ycsb/tool/docker/clear.sh +++ /dev/null @@ -1,7 +0,0 @@ - -for db in pg cockroach mysql mysql8 tidb tikv -do - docker-compose -f ${db}.yml down --remove-orphans -done - -rm -rf ./data \ No newline at end of file diff --git a/go-ycsb/tool/docker/cockroach.yml b/go-ycsb/tool/docker/cockroach.yml deleted file mode 100644 index a6181636e..000000000 --- a/go-ycsb/tool/docker/cockroach.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: '2.1' - -services: - cockroach: - image: cockroachdb/cockroach:latest - volumes: - - ./data/cockroach:/cockroach/cockroach-data - ports: - - "26257:26257" - - "8080:8080" - command: start --insecure - restart: on-failure - - db-init: - image: cockroachdb/cockroach:latest - volumes: - - ./config/init_cockroach.sh:/init_cockroach.sh - entrypoint: - - "/bin/bash" - command: /init_cockroach.sh - depends_on: - - "cockroach" - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/config/init_cassandra.sh b/go-ycsb/tool/docker/config/init_cassandra.sh deleted file mode 100755 index c6e64ee5c..000000000 --- a/go-ycsb/tool/docker/config/init_cassandra.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -echo "Wait for servers to be up" - -for _ in {1..10} -do - sleep 5 - cqlsh $1 -e "CREATE KEYSPACE test WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor' : 1};" - if [ $? -eq 0 ] - then - echo "create keyspace ok" - break - fi - echo "create keyspace failed, wait 5s" -done diff --git a/go-ycsb/tool/docker/config/init_cockroach.sh b/go-ycsb/tool/docker/config/init_cockroach.sh deleted file mode 100755 index 1996f2689..000000000 --- a/go-ycsb/tool/docker/config/init_cockroach.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -echo "Wait for servers to be up" -sleep 5 - -HOSTPARAMS="--host cockroach --insecure" -SQL="/cockroach/cockroach.sh sql $HOSTPARAMS" - -$SQL -e "CREATE DATABASE test;" \ No newline at end of file diff --git a/go-ycsb/tool/docker/config/pd.toml b/go-ycsb/tool/docker/config/pd.toml deleted file mode 100644 index e091e695b..000000000 --- a/go-ycsb/tool/docker/config/pd.toml +++ /dev/null @@ -1,91 +0,0 @@ -# PD Configuration. - -name = "pd" -data-dir = "default.pd" - -client-urls = "http://127.0.0.1:2379" -# if not set, use ${client-urls} -advertise-client-urls = "" - -peer-urls = "http://127.0.0.1:2380" -# if not set, use ${peer-urls} -advertise-peer-urls = "" - -initial-cluster = "pd=http://127.0.0.1:2380" -initial-cluster-state = "new" - -lease = 3 -tso-save-interval = "3s" - -namespace-classifier = "table" - -enable-prevote = true - -[security] -# Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty -cacert-path = "" -# Path of file that contains X509 certificate in PEM format. -cert-path = "" -# Path of file that contains X509 key in PEM format. -key-path = "" - -[log] -level = "info" - -# log format, one of json, text, console -#format = "text" - -# disable automatic timestamps in output -#disable-timestamp = false - -# file logging -[log.file] -#filename = "" -# max log file size in MB -#max-size = 300 -# max log file keep days -#max-days = 28 -# maximum number of old log files to retain -#max-backups = 7 -# rotate log by day -#log-rotate = true - -[metric] -# prometheus client push interval, set "0s" to disable prometheus. -interval = "15s" -# prometheus pushgateway address, leaves it empty will disable prometheus. -address = "" - -[schedule] -max-merge-region-size = 20 -max-merge-region-keys = 200000 -split-merge-interval = "1h" -max-snapshot-count = 3 -max-pending-peer-count = 16 -max-store-down-time = "30m" -leader-schedule-limit = 4 -region-schedule-limit = 4 -replica-schedule-limit = 8 -merge-schedule-limit = 8 -tolerant-size-ratio = 5.0 - -# customized schedulers, the format is as below -# if empty, it will use balance-leader, balance-region, hot-region as default -# [[schedule.schedulers]] -# type = "evict-leader" -# args = ["1"] - -[replication] -# The number of replicas for each region. -max-replicas = 3 -# The label keys specified the location of a store. -# The placement priorities is implied by the order of label keys. -# For example, ["zone", "rack"] means that we should place replicas to -# different zones first, then to different racks if we don't have enough zones. -location-labels = [] - -[label-property] -# Do not assign region leaders to stores that have these tags. -# [[label-property.reject-leader]] -# key = "zone" -# value = "cn1 diff --git a/go-ycsb/tool/docker/config/tidb.toml b/go-ycsb/tool/docker/config/tidb.toml deleted file mode 100644 index 376052d49..000000000 --- a/go-ycsb/tool/docker/config/tidb.toml +++ /dev/null @@ -1,260 +0,0 @@ -# TiDB Configuration. - -# TiDB server host. -host = "0.0.0.0" - -# tidb server advertise IP. -advertise-address = "" - -# TiDB server port. -port = 4000 - -# Registered store name, [tikv, mocktikv] -store = "mocktikv" - -# TiDB storage path. -path = "/tmp/tidb" - -# The socket file to use for connection. -socket = "" - -# Run ddl worker on this tidb-server. -run-ddl = true - -# Schema lease duration, very dangerous to change only if you know what you do. -lease = "45s" - -# When create table, split a separated region for it. It is recommended to -# turn off this option if there will be a large number of tables created. -split-table = true - -# The limit of concurrent executed sessions. -token-limit = 1000 - -# Only print a log when out of memory quota. -# Valid options: ["log", "cancel"] -oom-action = "log" - -# Set the memory quota for a query in bytes. Default: 32GB -mem-quota-query = 34359738368 - -# Enable coprocessor streaming. -enable-streaming = false - -# Set system variable 'lower_case_table_names' -lower-case-table-names = 2 - -# Make "kill query" behavior compatible with MySQL. It's not recommend to -# turn on this option when TiDB server is behind a proxy. -compatible-kill-query = false - -[log] -# Log level: debug, info, warn, error, fatal. -level = "info" - -# Log format, one of json, text, console. -format = "text" - -# Disable automatic timestamp in output -disable-timestamp = false - -# Stores slow query log into separated files. -slow-query-file = "" - -# Queries with execution time greater than this value will be logged. (Milliseconds) -slow-threshold = 300 - -# Queries with internal result greater than this value will be logged. -expensive-threshold = 10000 - -# Maximum query length recorded in log. -query-log-max-len = 2048 - -# File logging. -[log.file] -# Log file name. -filename = "" - -# Max log file size in MB (upper limit to 4096MB). -max-size = 300 - -# Max log file keep days. No clean up by default. -max-days = 0 - -# Maximum number of old log files to retain. No clean up by default. -max-backups = 0 - -# Rotate log by day -log-rotate = true - -[security] -# Path of file that contains list of trusted SSL CAs for connection with mysql client. -ssl-ca = "" - -# Path of file that contains X509 certificate in PEM format for connection with mysql client. -ssl-cert = "" - -# Path of file that contains X509 key in PEM format for connection with mysql client. -ssl-key = "" - -# Path of file that contains list of trusted SSL CAs for connection with cluster components. -cluster-ssl-ca = "" - -# Path of file that contains X509 certificate in PEM format for connection with cluster components. -cluster-ssl-cert = "" - -# Path of file that contains X509 key in PEM format for connection with cluster components. -cluster-ssl-key = "" - -[status] -# If enable status report HTTP service. -report-status = true - -# TiDB status port. -status-port = 10080 - -# Prometheus pushgateway address, leaves it empty will disable prometheus push. -metrics-addr = "" - -# Prometheus client push interval in second, set \"0\" to disable prometheus push. -metrics-interval = 15 - -[performance] -# Max CPUs to use, 0 use number of CPUs in the machine. -max-procs = 0 - -# Max memory size to use, 0 use the total usable memory in the machine. -max-memory = 0 - -# StmtCountLimit limits the max count of statement inside a transaction. -stmt-count-limit = 5000 - -# Set keep alive option for tcp connection. -tcp-keep-alive = true - -# Whether support cartesian product. -cross-join = true - -# Stats lease duration, which influences the time of analyze and stats load. -stats-lease = "3s" - -# Run auto analyze worker on this tidb-server. -run-auto-analyze = true - -# Probability to use the query feedback to update stats, 0 or 1 for always false/true. -feedback-probability = 0.05 - -# The max number of query feedback that cache in memory. -query-feedback-limit = 1024 - -# Pseudo stats will be used if the ratio between the modify count and -# row count in statistics of a table is greater than it. -pseudo-estimate-ratio = 0.8 - -# Force the priority of all statements in a specified priority. -# The value could be "NO_PRIORITY", "LOW_PRIORITY", "HIGH_PRIORITY" or "DELAYED". -force-priority = "NO_PRIORITY" - -[proxy-protocol] -# PROXY protocol acceptable client networks. -# Empty string means disable PROXY protocol, * means all networks. -networks = "" - -# PROXY protocol header read timeout, unit is second -header-timeout = 5 - -[prepared-plan-cache] -enabled = false -capacity = 100 -memory-guard-ratio = 0.1 - -[opentracing] -# Enable opentracing. -enable = false - -# Whether to enable the rpc metrics. -rpc-metrics = false - -[opentracing.sampler] -# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote -type = "const" - -# Param is a value passed to the sampler. -# Valid values for Param field are: -# - for "const" sampler, 0 or 1 for always false/true respectively -# - for "probabilistic" sampler, a probability between 0 and 1 -# - for "rateLimiting" sampler, the number of spans per second -# - for "remote" sampler, param is the same as for "probabilistic" -# and indicates the initial sampling rate before the actual one -# is received from the mothership -param = 1.0 - -# SamplingServerURL is the address of jaeger-agent's HTTP sampling server -sampling-server-url = "" - -# MaxOperations is the maximum number of operations that the sampler -# will keep track of. If an operation is not tracked, a default probabilistic -# sampler will be used rather than the per operation specific sampler. -max-operations = 0 - -# SamplingRefreshInterval controls how often the remotely controlled sampler will poll -# jaeger-agent for the appropriate sampling strategy. -sampling-refresh-interval = 0 - -[opentracing.reporter] -# QueueSize controls how many spans the reporter can keep in memory before it starts dropping -# new spans. The queue is continuously drained by a background go-routine, as fast as spans -# can be sent out of process. -queue-size = 0 - -# BufferFlushInterval controls how often the buffer is force-flushed, even if it's not full. -# It is generally not useful, as it only matters for very low traffic services. -buffer-flush-interval = 0 - -# LogSpans, when true, enables LoggingReporter that runs in parallel with the main reporter -# and logs all submitted spans. Main Configuration.Logger must be initialized in the code -# for this option to have any effect. -log-spans = false - -# LocalAgentHostPort instructs reporter to send spans to jaeger-agent at this address -local-agent-host-port = "" - -[tikv-client] -# Max gRPC connections that will be established with each tikv-server. -grpc-connection-count = 16 - -# After a duration of this time in seconds if the client doesn't see any activity it pings -# the server to see if the transport is still alive. -grpc-keepalive-time = 10 - -# After having pinged for keepalive check, the client waits for a duration of Timeout in seconds -# and if no activity is seen even after that the connection is closed. -grpc-keepalive-timeout = 3 - -# max time for commit command, must be twice bigger than raft election timeout. -commit-timeout = "41s" - -# The max time a Txn may use (in seconds) from its startTS to commitTS. -# We use it to guarantee GC worker will not influence any active txn. Please make sure that this -# value is less than gc_life_time - 10s. -max-txn-time-use = 590 - -[txn-local-latches] -# Enable local latches for transactions. Enable it when -# there are lots of conflicts between transactions. -enabled = true -capacity = 2048000 - -[binlog] -# enable to write binlog. -enable = false - -# WriteTimeout specifies how long it will wait for writing binlog to pump. -write-timeout = "15s" - -# If IgnoreError is true, when writting binlog meets error, TiDB would stop writting binlog, -# but still provide service. -ignore-error = false - -# use socket file to write binlog, for compatible with kafka version tidb-binlog. -binlog-socket = "" diff --git a/go-ycsb/tool/docker/config/tikv.toml b/go-ycsb/tool/docker/config/tikv.toml deleted file mode 100644 index 29deb85ff..000000000 --- a/go-ycsb/tool/docker/config/tikv.toml +++ /dev/null @@ -1,689 +0,0 @@ -## TiKV config template -## Human-readable big numbers: -## File size(based on byte): KB, MB, GB, TB, PB -## e.g.: 1_048_576 = "1MB" -## Time(based on ms): ms, s, m, h -## e.g.: 78_000 = "1.3m" - -## Log levels: trace, debug, info, warning, error, critical. -## Note that `debug` and `trace` are only available in development builds. -# log-level = "info" - -## File to store logs. -## If it is not set, logs will be appended to stderr. -# log-file = "" - -## Timespan between rotating the log files. -## Once this timespan passes, log files will be rotated, i.e. existing log file will have a -## timestamp appended to its name and a new file will be created. -# log-rotation-timespan = "24h" - -[readpool.storage] -## Size of the thread pool for high-priority operations. -# high-concurrency = 4 - -## Size of the thread pool for normal-priority operations. -# normal-concurrency = 4 - -## Size of the thread pool for low-priority operations. -# low-concurrency = 4 - -## Max running high-priority operations of each worker, reject if exceeded. -# max-tasks-per-worker-high = 2000 - -## Max running normal-priority operations of each worker, reject if exceeded. -# max-tasks-per-worker-normal = 2000 - -## Max running low-priority operations of each worker, reject if exceeded. -# max-tasks-per-worker-low = 2000 - -## Size of the stack for each thread in the thread pool. -# stack-size = "10MB" - -[readpool.coprocessor] -## Most read requests from TiDB are sent to the coprocessor of TiKV. high/normal/low-concurrency is -## used to set the number of threads of the coprocessor. -## If there are many read requests, you can increase these config values (but keep it within the -## number of system CPU cores). For example, for a 32-core machine deployed with TiKV, you can even -## set these config to 30 in heavy read scenarios. -## If CPU_NUM > 8, the default thread pool size for coprocessors is set to CPU_NUM * 0.8. - -# high-concurrency = 8 -# normal-concurrency = 8 -# low-concurrency = 8 -# max-tasks-per-worker-high = 2000 -# max-tasks-per-worker-normal = 2000 -# max-tasks-per-worker-low = 2000 -# stack-size = "10MB" - -[server] -## Listening address. -# addr = "127.0.0.1:20160" - -## Advertise listening address for client communication. -## If not set, `addr` will be used. -# advertise-addr = "" - -## Status address. -## This is used for reporting the status of TiKV directly through the HTTP address. -## Empty string means disabling it. -# status-addr = "127.0.0.1:20180" - -## Set the maximum number of worker threads for the status report HTTP service. -# status-thread-pool-size = 1 - -## Compression type for gRPC channel: none, deflate or gzip. -# grpc-compression-type = "none" - -## Size of the thread pool for the gRPC server. -# grpc-concurrency = 4 - -## The number of max concurrent streams/requests on a client connection. -# grpc-concurrent-stream = 1024 - -## The number of connections with each TiKV server to send Raft messages. -# grpc-raft-conn-num = 10 - -## Amount to read ahead on individual gRPC streams. -# grpc-stream-initial-window-size = "2MB" - -## Time to wait before sending out a ping to check if server is still alive. -## This is only for communications between TiKV instances. -# grpc-keepalive-time = "10s" - -## Time to wait before closing the connection without receiving KeepAlive ping Ack. -# grpc-keepalive-timeout = "3s" - -## How many snapshots can be sent concurrently. -# concurrent-send-snap-limit = 32 - -## How many snapshots can be received concurrently. -# concurrent-recv-snap-limit = 32 - -## Max allowed recursion level when decoding Coprocessor DAG expression. -# end-point-recursion-limit = 1000 - -## Max time to handle Coprocessor requests before timeout. -# end-point-request-max-handle-duration = "60s" - -## Max bytes that snapshot can be written to disk in one second. -## It should be set based on your disk performance. -# snap-max-write-bytes-per-sec = "100MB" - -## Attributes about this server, e.g. `{ zone = "us-west-1", disk = "ssd" }`. -# labels = {} - -[storage] -## The path to RocksDB directory. -# data-dir = "/tmp/tikv/store" - -## Internal notify capacity of Scheduler's channel. -# scheduler-notify-capacity = 10240 - -## The number of slots in Scheduler latches, which controls write concurrency. -## In most cases you can use the default value. When importing data, you can set it to a larger -## value. -# scheduler-concurrency = 2048000 - -## Scheduler's worker pool size, i.e. the number of write threads. -## It should be less than total CPU cores. When there are frequent write operations, set it to a -## higher value. More specifically, you can run `top -H -p tikv-pid` to check whether the threads -## named `sched-worker-pool` are busy. -# scheduler-worker-pool-size = 4 - -## When the pending write bytes exceeds this threshold, the "scheduler too busy" error is displayed. -# scheduler-pending-write-threshold = "100MB" - -[pd] -## PD endpoints. -# endpoints = [] - -[raftstore] -## Whether to force to flush logs. -## Set to `true` (default) for best reliability, which prevents data loss when there is a power -## failure. Set to `false` for higher performance (ensure that you run multiple TiKV nodes!). -# sync-log = true - -## Whether to enable Raft prevote. -## Prevote minimizes disruption when a partitioned node rejoins the cluster by using a two phase -## election. -# prevote = true - -## The path to RaftDB directory. -## If not set, it will be `{data-dir}/raft`. -## If there are multiple disks on the machine, storing the data of Raft RocksDB on differen disks -## can improve TiKV performance. -# raftdb-path = "" - -## Store capacity, i.e. max data size allowed. -## If it is not set, disk capacity is used. -# capacity = 0 - -## Internal notify capacity. -## 40960 is suitable for about 7000 Regions. It is recommended to use the default value. -# notify-capacity = 40960 - -## Maximum number of internal messages to process in a tick. -# messages-per-tick = 4096 - -## Region heartbeat tick interval for reporting to PD. -# pd-heartbeat-tick-interval = "60s" - -## Store heartbeat tick interval for reporting to PD. -# pd-store-heartbeat-tick-interval = "10s" - -## The threshold of triggering Region split check. -## When Region size change exceeds this config, TiKV will check whether the Region should be split -## or not. To reduce the cost of scanning data in the checking process, you can set the value to -## 32MB during checking and set it back to the default value in normal operations. -# region-split-check-diff = "6MB" - -## The interval of triggering Region split check. -# split-region-check-tick-interval = "10s" - -## When the number of Raft entries exceeds the max size, TiKV rejects to propose the entry. -# raft-entry-max-size = "8MB" - -## Interval to GC unnecessary Raft log. -# raft-log-gc-tick-interval = "10s" - -## Threshold to GC stale Raft log, must be >= 1. -# raft-log-gc-threshold = 50 - -## When the entry count exceeds this value, GC will be forced to trigger. -# raft-log-gc-count-limit = 72000 - -## When the approximate size of Raft log entries exceeds this value, GC will be forced trigger. -## It's recommanded to set it to 3/4 of `region-split-size`. -# raft-log-gc-size-limit = "72MB" - -## How long the peer will be considered down and reported to PD when it hasn't been active for this -## time. -# max-peer-down-duration = "5m" - -## Interval to check whether to start manual compaction for a Region. -# region-compact-check-interval = "5m" - -## Number of Regions for each time to check. -# region-compact-check-step = 100 - -## The minimum number of delete tombstones to trigger manual compaction. -# region-compact-min-tombstones = 10000 - -## The minimum percentage of delete tombstones to trigger manual compaction. -## It should be set between 1 and 100. Manual compaction is only triggered when the number of -## delete tombstones exceeds `region-compact-min-tombstones` and the percentage of delete tombstones -## exceeds `region-compact-tombstones-percent`. -# region-compact-tombstones-percent = 30 - -## Interval to check whether to start a manual compaction for Lock Column Family. -## If written bytes reach `lock-cf-compact-bytes-threshold` for Lock Column Family, TiKV will -## trigger a manual compaction for Lock Column Family. -# lock-cf-compact-interval = "10m" -# lock-cf-compact-bytes-threshold = "256MB" - -## Interval (s) to check Region whether the data are consistent. -# consistency-check-interval = 0 - -## Delay time before deleting a stale peer. -# clean-stale-peer-delay = "10m" - -## Interval to clean up import SST files. -# cleanup-import-sst-interval = "10m" - -[coprocessor] -## When it is set to `true`, TiKV will try to split a Region with table prefix if that Region -## crosses tables. -## It is recommended to turn off this option if there will be a large number of tables created. -# split-region-on-table = true - -## One split check produces several split keys in batch. This config limits the number of produced -## split keys in one batch. -# batch-split-limit = 10 - -## When Region [a,e) size exceeds `region_max_size`, it will be split into several Regions [a,b), -## [b,c), [c,d), [d,e) and the size of [a,b), [b,c), [c,d) will be `region_split_size` (or a -## little larger). -# region-max-size = "144MB" -# region-split-size = "96MB" - -## When the number of keys in Region [a,e) exceeds the `region_max_keys`, it will be split into -## several Regions [a,b), [b,c), [c,d), [d,e) and the number of keys in [a,b), [b,c), [c,d) will be -## `region_split_keys`. -# region-max-keys = 1440000 -# region-split-keys = 960000 - -[rocksdb] -## Maximum number of threads of RocksDB background jobs. -## The background tasks include compaction and flush. For detailed information why RocksDB needs to -## do compaction, see RocksDB-related materials. When write traffic (like the importing data size) -## is big, it is recommended to enable more threads. But set the number of the enabled threads -## smaller than that of CPU cores. For example, when importing data, for a machine with a 32-core -## CPU, set the value to 28. -# max-background-jobs = 8 - -## Represents the maximum number of threads that will concurrently perform a sub-compaction job by -## breaking it into multiple, smaller ones running simultaneously. -# max-sub-compactions = 1 - -## Number of open files that can be used by the DB. -## Value -1 means files opened are always kept open and RocksDB will prefetch index and filter -## blocks into block cache at startup. So if your database has a large working set, it will take -## several minutes to open the DB. You may need to increase this if your database has a large -## working set. You can estimate the number of files based on `target-file-size-base` and -## `target_file_size_multiplier` for level-based compaction. -max-open-files = 1024 - -## Max size of RocksDB's MANIFEST file. -## For detailed explanation, please refer to https://github.com/facebook/rocksdb/wiki/MANIFEST -# max-manifest-file-size = "20MB" - -## If the value is `true`, the database will be created if it is missing. -# create-if-missing = true - -## RocksDB Write-Ahead Logs (WAL) recovery mode. -## 0 : TolerateCorruptedTailRecords, tolerate incomplete record in trailing data on all logs; -## 1 : AbsoluteConsistency, We don't expect to find any corruption in the WAL; -## 2 : PointInTimeRecovery, Recover to point-in-time consistency; -## 3 : SkipAnyCorruptedRecords, Recovery after a disaster; -# wal-recovery-mode = 2 - -## RocksDB WAL directory. -## This config specifies the absolute directory path for WAL. -## If it is not set, the log files will be in the same directory as data. When you set the path to -## RocksDB directory in memory like in `/dev/shm`, you may want to set`wal-dir` to a directory on a -## persistent storage. See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database . -## If there are two disks on the machine, storing RocksDB data and WAL logs on different disks can -## improve performance. -# wal-dir = "/tmp/tikv/store" - -## The following two fields affect how archived WAL will be deleted. -## 1. If both values are set to 0, logs will be deleted ASAP and will not get into the archive. -## 2. If `wal-ttl-seconds` is 0 and `wal-size-limit` is not 0, WAL files will be checked every 10 -## min and if total size is greater than `wal-size-limit`, they will be deleted starting with the -## earliest until `wal-size-limit` is met. All empty files will be deleted. -## 3. If `wal-ttl-seconds` is not 0 and `wal-size-limit` is 0, then WAL files will be checked every -## `wal-ttl-seconds / 2` and those that are older than `wal-ttl-seconds` will be deleted. -## 4. If both are not 0, WAL files will be checked every 10 min and both checks will be performed -## with ttl being first. -## When you set the path to RocksDB directory in memory like in `/dev/shm`, you may want to set -## `wal-ttl-seconds` to a value greater than 0 (like 86400) and backup your DB on a regular basis. -## See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database . -# wal-ttl-seconds = 0 -# wal-size-limit = 0 - -## Max RocksDB WAL size in total -# max-total-wal-size = "4GB" - -## RocksDB Statistics provides cumulative stats over time. -## Turning statistics on will introduce about 5%-10% overhead for RocksDB, but it can help you to -## know the internal status of RocksDB. -# enable-statistics = true - -## Dump statistics periodically in information logs. -## Same as RocksDB's default value (10 min). -# stats-dump-period = "10m" - -## Refer to: https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ -## If you want to use RocksDB on multi disks or spinning disks, you should set value at least 2MB. -# compaction-readahead-size = 0 - -## Max buffer size that is used by WritableFileWrite. -# writable-file-max-buffer-size = "1MB" - -## Use O_DIRECT for both reads and writes in background flush and compactions. -# use-direct-io-for-flush-and-compaction = false - -## Limit the disk IO of compaction and flush. -## Compaction and flush can cause terrible spikes if they exceed a certain threshold. Consider -## setting this to 50% ~ 80% of the disk throughput for a more stable result. However, in heavy -## write workload, limiting compaction and flush speed can cause write stalls too. -# rate-bytes-per-sec = 0 - -## Enable or disable the pipelined write. -# enable-pipelined-write = true - -## Allows OS to incrementally sync files to disk while they are being written, asynchronously, -## in the background. -# bytes-per-sync = "1MB" - -## Allows OS to incrementally sync WAL to disk while it is being written. -# wal-bytes-per-sync = "512KB" - -## Specify the maximal size of the RocksDB info log file. -## If the log file is larger than this config, a new info log file will be created. -## If it is set to 0, all logs will be written to one log file. -# info-log-max-size = "1GB" - -## Time for the RocksDB info log file to roll (in seconds). -## If the log file has been active longer than this config, it will be rolled. -## If it is set to 0, rolling will be disabled. -# info-log-roll-time = "0" - -## Maximal RocksDB info log files to be kept. -# info-log-keep-log-file-num = 10 - -## Specifies the RocksDB info log directory. -## If it is empty, the log files will be in the same directory as data. -## If it is not empty, the log files will be in the specified directory, and the DB data directory's -## absolute path will be used as the log file name's prefix. -# info-log-dir = "" - -## Options for `Titan`. -[rocksdb.titan] -## Enables `Titan` -## default: false -# enabled = false - -## Specifies `Titan` blob files directory -## default: "titandb" (if not specific or empty) -# dirname = "" - -## Disable blob file gc -# default: false -# disable-gc = false - -## Maximum number of threads of `Titan` background gc jobs. -# default: 1 -# max-background-gc = 1 - -## Options for "Default" Column Family, which stores actual user data. -[rocksdb.defaultcf] -## Compression method (if any) is used to compress a block. -## no: kNoCompression -## snappy: kSnappyCompression -## zlib: kZlibCompression -## bzip2: kBZip2Compression -## lz4: kLZ4Compression -## lz4hc: kLZ4HCCompression -## zstd: kZSTD -## `lz4` is a compression algorithm with moderate speed and compression ratio. The compression -## ratio of `zlib` is high. It is friendly to the storage space, but its compression speed is -## slow. This compression occupies many CPU resources. - -## Per level compression. -## This config should be chosen carefully according to CPU and I/O resources. For example, if you -## use the compression mode of "no:no:lz4:lz4:lz4:zstd:zstd" and find much I/O pressure of the -## system (run the `iostat` command to find %util lasts 100%, or run the `top` command to find many -## iowaits) when writing (importing) a lot of data while the CPU resources are adequate, you can -## compress level-0 and level-1 and exchange CPU resources for I/O resources. If you use the -## compression mode of "no:no:lz4:lz4:lz4:zstd:zstd" and you find the I/O pressure of the system is -## not big when writing a lot of data, but CPU resources are inadequate. Then run the `top` command -## and choose the `-H` option. If you find a lot of bg threads (namely the compression thread of -## RocksDB) are running, you can exchange I/O resources for CPU resources and change the compression -## mode to "no:no:no:lz4:lz4:zstd:zstd". In a word, it aims at making full use of the existing -## resources of the system and improving TiKV performance in terms of the current resources. -# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] - -## The data block size. RocksDB compresses data based on the unit of block. -## Similar to page in other databases, block is the smallest unit cached in block-cache. Note that -## the block size specified here corresponds to uncompressed data. -# block-size = "64KB" - -## If you're doing point lookups you definitely want to turn bloom filters on. We use bloom filters -## to avoid unnecessary disk reads. Default bits_per_key is 10, which yields ~1% false positive -## rate. Larger `bloom-filter-bits-per-key` values will reduce false positive rate, but increase -## memory usage and space amplification. -# bloom-filter-bits-per-key = 10 - -## `false` means one SST file one bloom filter, `true` means every block has a corresponding bloom -## filter. -# block-based-bloom-filter = false - -# level0-file-num-compaction-trigger = 4 - -## Soft limit on number of level-0 files. -## When the number of SST files of level-0 reaches the limit of `level0-slowdown-writes-trigger`, -## RocksDB tries to slow down the write operation, because too many SST files of level-0 can cause -## higher read pressure of RocksDB. -# level0-slowdown-writes-trigger = 20 - -## Maximum number of level-0 files. -## When the number of SST files of level-0 reaches the limit of `level0-stop-writes-trigger`, -## RocksDB stalls the new write operation. -# level0-stop-writes-trigger = 36 - -## Amount of data to build up in memory (backed by an unsorted log on disk) before converting to a -## sorted on-disk file. It is the RocksDB MemTable size. -# write-buffer-size = "128MB" - -## The maximum number of the MemTables. The data written into RocksDB is first recorded in the WAL -## log, and then inserted into MemTables. When the MemTable reaches the size limit of -## `write-buffer-size`, it turns into read only and generates a new MemTable receiving new write -## operations. The flush threads of RocksDB will flush the read only MemTable to the disks to become -## an SST file of level0. `max-background-flushes` controls the maximum number of flush threads. -## When the flush threads are busy, resulting in the number of the MemTables waiting to be flushed -## to the disks reaching the limit of `max-write-buffer-number`, RocksDB stalls the new operation. -## "Stall" is a flow control mechanism of RocksDB. When importing data, you can set the -## `max-write-buffer-number` value higher, like 10. -# max-write-buffer-number = 5 - -## The minimum number of write buffers that will be merged together before writing to storage. -# min-write-buffer-number-to-merge = 1 - -## Control maximum total data size for base level (level 1). -## When the level-1 data size reaches the limit value of `max-bytes-for-level-base`, the SST files -## of level-1 and their overlap SST files of level-2 will be compacted. The golden rule: the first -## reference principle of setting `max-bytes-for-level-base` is guaranteeing that the -## `max-bytes-for-level-base` value is roughly equal to the data volume of level-0. Thus -## unnecessary compaction is reduced. For example, if the compression mode is -## "no:no:lz4:lz4:lz4:lz4:lz4", the `max-bytes-for-level-base` value can be `write-buffer-size * 4`, -## because there is no compression of level-0 and level-1 and the trigger condition of compaction -## for level-0 is that the number of the SST files reaches 4 (the default value). When both level-0 -## and level-1 adopt compaction, it is necessary to analyze RocksDB logs to know the size of an SST -## file compressed from a MemTable. For example, if the file size is 32MB, the proposed value of -## `max-bytes-for-level-base` is 32MB * 4 = 128MB. -# max-bytes-for-level-base = "512MB" - -## Target file size for compaction. -## The SST file size of level-0 is influenced by the compaction algorithm of `write-buffer-size` -## and level0. `target-file-size-base` is used to control the size of a single SST file of level1 to -## level6. -# target-file-size-base = "8MB" - -## Max bytes for `compaction.max_compaction_bytes`. -# max-compaction-bytes = "2GB" - -## There are four different compaction priorities. -## 0 : ByCompensatedSize -## 1 : OldestLargestSeqFirst -## 2 : OldestSmallestSeqFirst -## 3 : MinOverlappingRatio -# compaction-pri = 3 - -## Block cache used to cache uncompressed blocks. -## Big block-cache can speed up read. Normally it should be tuned to 30%-50% system's total memory. -## When the config is not set, TiKV sets the value to 40% of the system memory size. -## To deploy multiple TiKV nodes on one physical machine, configure this parameter explicitly. -## Otherwise, the OOM problem might occur in TiKV. -# block-cache-size = "1GB" - -## Indicating if we'd put index/filter blocks to the block cache. -## If not specified, each "table reader" object will pre-load index/filter block during table -## initialization. -# cache-index-and-filter-blocks = true - -## Pin level-0 filter and index blocks in cache. -# pin-l0-filter-and-index-blocks = true - -## Enable read amplification statistics. -## value => memory usage (percentage of loaded blocks memory) -## 1 => 12.50 % -## 2 => 06.25 % -## 4 => 03.12 % -## 8 => 01.56 % -## 16 => 00.78 % -# read-amp-bytes-per-bit = 0 - -## Pick target size of each level dynamically. -# dynamic-level-bytes = true - -## Optimizes bloom filters. If true, RocksDB won't create bloom filters for the max level of -## the LSM to reduce metadata that should fit in RAM. -## This value is setted to true for `default` cf by default because its kv data could be determined -## whether really exists by upper logic instead of bloom filters. But we suggest to set it to false -## while using `Raw` mode. -# optimize-filters-for-hits = true - -## Options for "Default" Column Family for `Titan`. -[rocksdb.defaultcf.titan] -## The smallest value to store in blob files. Value smaller than -## this threshold will be inlined in base DB. -## default: 1KB -# min-blob-size = "1KB" - -## The compression algorithm used to compress data in blob files. -## Compression method. -## no: kNoCompression -## snappy: kSnappyCompression -## zlib: kZlibCompression -## bzip2: kBZip2Compression -## lz4: kLZ4Compression -## lz4hc: kLZ4HCCompression -## zstd: kZSTD -# default: lz4 -# blob-file-compression = "lz4" - -## Specifics cache size for blob records -# default: 0 -# blob-cache-size = "0GB" - -## The minimum batch size of one gc job. The total blob file size -## of one gc job cannot smaller than this threshold. -## default: 16MB -# min-gc-batch-size = "16MB" - -## The maximum batch size of one gc job. The total blob file size -## of one gc job cannot exceed this threshold. -# max-gc-batch-size = "64MB" - -## If the ratio of discardable size of a blob file is larger than -## this threshold, the blob file will be GCed out. -# default: 0.5 -# discardable-ratio = 0.5 - -## The gc job will sample the target blob files to see if its -## discardable ratio is smaller than discardable-ratio metioned -## above before gc start, if so the blob file will be exclude. -# sample-ratio = 0.1 - -## If the size of the blob file is smaller than this threshold, -## the blob file will be merge. -# default: 8MB -# merge-small-file-threshold = "8MB" - -## Options for "Write" Column Family, which stores MVCC commit information -[rocksdb.writecf] -## Recommend to set it the same as `rocksdb.defaultcf.compression-per-level`. -# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] -# block-size = "64KB" - -## Recommend to set it the same as `rocksdb.defaultcf.write-buffer-size`. -# write-buffer-size = "128MB" -# max-write-buffer-number = 5 -# min-write-buffer-number-to-merge = 1 - -## Recommend to set it the same as `rocksdb.defaultcf.max-bytes-for-level-base`. -# max-bytes-for-level-base = "512MB" -# target-file-size-base = "8MB" - -## In normal cases, this config should be tuned to 10%-30% of the system's total memory. -## When this config is not set, TiKV sets it to 15% of the system memory size. -## To deploy multiple TiKV nodes on a single physical machine, configure this parameter explicitly. -## The related data of the version information (MVCC) and the index-related data are recorded in -## Write CF. In scenarios that include many single table indexes, set this parameter value higher. -# block-cache-size = "256MB" - -# level0-file-num-compaction-trigger = 4 -# level0-slowdown-writes-trigger = 20 -# level0-stop-writes-trigger = 36 -# cache-index-and-filter-blocks = true -# pin-l0-filter-and-index-blocks = true -# compaction-pri = 3 -# read-amp-bytes-per-bit = 0 -# dynamic-level-bytes = true -# optimize-filters-for-hits = false - -[rocksdb.lockcf] -# compression-per-level = ["no", "no", "no", "no", "no", "no", "no"] -# block-size = "16KB" -# write-buffer-size = "128MB" -# max-write-buffer-number = 5 -# min-write-buffer-number-to-merge = 1 -# max-bytes-for-level-base = "128MB" -# target-file-size-base = "8MB" -# block-cache-size = "256MB" -# level0-file-num-compaction-trigger = 1 -# level0-slowdown-writes-trigger = 20 -# level0-stop-writes-trigger = 36 -# cache-index-and-filter-blocks = true -# pin-l0-filter-and-index-blocks = true -# compaction-pri = 0 -# read-amp-bytes-per-bit = 0 -# dynamic-level-bytes = true -# optimize-filters-for-hits = false - -[raftdb] -# max-background-jobs = 2 -# max-sub-compactions = 1 -max-open-files = 1024 -# max-manifest-file-size = "20MB" -# create-if-missing = true - -# enable-statistics = true -# stats-dump-period = "10m" - -# compaction-readahead-size = 0 -# writable-file-max-buffer-size = "1MB" -# use-direct-io-for-flush-and-compaction = false -# enable-pipelined-write = true -# allow-concurrent-memtable-write = false -# bytes-per-sync = "1MB" -# wal-bytes-per-sync = "512KB" - -# info-log-max-size = "1GB" -# info-log-roll-time = "0" -# info-log-keep-log-file-num = 10 -# info-log-dir = "" -# optimize-filters-for-hits = true - -[raftdb.defaultcf] -## Recommend to set it the same as `rocksdb.defaultcf.compression-per-level`. -# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] -# block-size = "64KB" - -## Recommend to set it the same as `rocksdb.defaultcf.write-buffer-size`. -# write-buffer-size = "128MB" -# max-write-buffer-number = 5 -# min-write-buffer-number-to-merge = 1 - -## Recommend to set it the same as `rocksdb.defaultcf.max-bytes-for-level-base`. -# max-bytes-for-level-base = "512MB" -# target-file-size-base = "8MB" - -## Generally, you can set it from 256MB to 2GB. In most cases, you can use the default value. But -## if the system resources are adequate, you can set it higher. -# block-cache-size = "256MB" - -# level0-file-num-compaction-trigger = 4 -# level0-slowdown-writes-trigger = 20 -# level0-stop-writes-trigger = 36 -# cache-index-and-filter-blocks = true -# pin-l0-filter-and-index-blocks = true -# compaction-pri = 0 -# read-amp-bytes-per-bit = 0 -# dynamic-level-bytes = true -# optimize-filters-for-hits = true - -[security] -## The path for TLS certificates. Empty string means disabling secure connections. -# ca-path = "" -# cert-path = "" -# key-path = "" - -[import] -## Number of threads to handle RPC requests. -# num-threads = 8 - -## Stream channel window size, stream will be blocked on channel full. -# stream-channel-window = 128 diff --git a/go-ycsb/tool/docker/mariadb.yml b/go-ycsb/tool/docker/mariadb.yml deleted file mode 100644 index d8f0dada4..000000000 --- a/go-ycsb/tool/docker/mariadb.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '2.1' - -services: - mariadb: - image: mariadb - ports: - - "3308:3306" - volumes: - - ./data/mariadb:/var/lib/mysql - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - MYSQL_DATABASE: "test" - restart: on-failure - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/mysql.yml b/go-ycsb/tool/docker/mysql.yml deleted file mode 100644 index bd0a2012e..000000000 --- a/go-ycsb/tool/docker/mysql.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '2.1' - -services: - mysql: - image: mysql:5.7 - ports: - - "3306:3306" - volumes: - - ./data/mysql:/var/lib/mysql - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - MYSQL_DATABASE: "test" - restart: on-failure - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/mysql8.yml b/go-ycsb/tool/docker/mysql8.yml deleted file mode 100644 index 5116614f5..000000000 --- a/go-ycsb/tool/docker/mysql8.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '2.1' - -services: - mysql: - image: mysql:8.0 - ports: - - "3307:3306" - volumes: - - ./data/mysql8:/var/lib/mysql - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - MYSQL_DATABASE: "test" - restart: on-failure - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/pg.yml b/go-ycsb/tool/docker/pg.yml deleted file mode 100644 index b2691b1bb..000000000 --- a/go-ycsb/tool/docker/pg.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '2.1' - -services: - pg: - image: postgres:latest - ports: - - "5432:5432" - volumes: - - ./data/pg:/var/lib/postgresql/data - environment: - POSTGRES_USER: "root" - POSTGRES_PASSWORD: "" - POSTGRES_DB: "test" - restart: on-failure - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/raw.yml b/go-ycsb/tool/docker/raw.yml deleted file mode 100644 index f97af7b02..000000000 --- a/go-ycsb/tool/docker/raw.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '2.1' - -services: - pd: - image: pingcap/pd:latest - ports: - - "2379" - volumes: - - ./config/pd.toml:/pd.toml:ro - - ./data/txn:/data - - ./logs/txn:/logs - command: - - --name=pd - - --client-urls=http://0.0.0.0:2379 - - --peer-urls=http://0.0.0.0:2380 - - --advertise-client-urls=http://pd:2379 - - --advertise-peer-urls=http://pd:2380 - - --initial-cluster=pd=http://pd:2380 - - --data-dir=/data/pd - - --config=/pd.toml - - --log-file=/logs/pd.log - restart: on-failure - - tikv: - image: pingcap/tikv:latest - volumes: - - ./config/tikv.toml:/tikv.toml:ro - - ./data/txn:/data - - ./logs/txn:/logs - command: - - --addr=0.0.0.0:20160 - - --advertise-addr=tikv:20160 - - --data-dir=/data/tikv - - --pd=pd:2379 - - --config=/tikv.toml - - --log-file=/logs/tikv.log - depends_on: - - "pd" - restart: on-failure - - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/run_bench.sh b/go-ycsb/tool/docker/run_bench.sh deleted file mode 100755 index 05f879858..000000000 --- a/go-ycsb/tool/docker/run_bench.sh +++ /dev/null @@ -1,10 +0,0 @@ -rm -rf ./data -rm -rf ./logs - -for db in pg cockroach mysql mysql8 mariadb tidb tikv raw cassandra scylla -do - ./bench.sh load ${db} - ./bench.sh run ${db} -done - -./clear.sh diff --git a/go-ycsb/tool/docker/scylla.yml b/go-ycsb/tool/docker/scylla.yml deleted file mode 100644 index 6aa08b843..000000000 --- a/go-ycsb/tool/docker/scylla.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: '2.1' - -services: - scylla: - image: scylladb/scylla:latest - volumes: - - ./data/scylla:/var/lib/scylla - ports: - - "9042:9042" - restart: on-failure - - db-init: - image: scylladb/scylla:latest - volumes: - - ./config/init_cassandra.sh:/init_cassandra.sh - entrypoint: - - "/bin/bash" - command: /init_cassandra.sh scylla - depends_on: - - "scylla" - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/sqlite.yml b/go-ycsb/tool/docker/sqlite.yml deleted file mode 100644 index f6e32c11c..000000000 --- a/go-ycsb/tool/docker/sqlite.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '2.1' - -services: - ycsb: - image: pingcap/go-ycsb - volumes: - - ./data/sqlite:/data - diff --git a/go-ycsb/tool/docker/tidb.yml b/go-ycsb/tool/docker/tidb.yml deleted file mode 100644 index 8f1b08bbf..000000000 --- a/go-ycsb/tool/docker/tidb.yml +++ /dev/null @@ -1,60 +0,0 @@ -version: '2.1' - -services: - pd: - image: pingcap/pd:latest - ports: - - "2379" - volumes: - - ./config/pd.toml:/pd.toml:ro - - ./data/tidb:/data - - ./logs/tidb:/logs - command: - - --name=pd - - --client-urls=http://0.0.0.0:2379 - - --peer-urls=http://0.0.0.0:2380 - - --advertise-client-urls=http://pd:2379 - - --advertise-peer-urls=http://pd:2380 - - --initial-cluster=pd=http://pd:2380 - - --data-dir=/data/pd - - --config=/pd.toml - - --log-file=/logs/pd.log - restart: on-failure - - tikv: - image: pingcap/tikv:latest - volumes: - - ./config/tikv.toml:/tikv.toml:ro - - ./data/tidb:/data - - ./logs/tidb:/logs - command: - - --addr=0.0.0.0:20160 - - --advertise-addr=tikv:20160 - - --data-dir=/data/tikv - - --pd=pd:2379 - - --config=/tikv.toml - - --log-file=/logs/tikv.log - depends_on: - - "pd" - restart: on-failure - - tidb: - image: pingcap/tidb:latest - ports: - - "4000:4000" - - "10080:10080" - volumes: - - ./config/tidb.toml:/tidb.toml:ro - - ./logs/tidb:/logs - command: - - --store=tikv - - --path=pd:2379 - - --config=/tidb.toml - - --log-file=/logs/tidb.log - depends_on: - - "tikv" - restart: on-failure - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/docker/tikv.yml b/go-ycsb/tool/docker/tikv.yml deleted file mode 100644 index 1c98b81e0..000000000 --- a/go-ycsb/tool/docker/tikv.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '2.1' - -services: - pd: - image: pingcap/pd:latest - ports: - - "2379" - volumes: - - ./config/pd.toml:/pd.toml:ro - - ./data/tikv:/data - - ./logs/tikv:/logs - command: - - --name=pd - - --client-urls=http://0.0.0.0:2379 - - --peer-urls=http://0.0.0.0:2380 - - --advertise-client-urls=http://pd:2379 - - --advertise-peer-urls=http://pd:2380 - - --initial-cluster=pd=http://pd:2380 - - --data-dir=/data/pd - - --config=/pd.toml - - --log-file=/logs/pd.log - restart: on-failure - - tikv: - image: pingcap/tikv:latest - volumes: - - ./config/tikv.toml:/tikv.toml:ro - - ./data/tikv:/data - - ./logs/tikv:/logs - command: - - --addr=0.0.0.0:20160 - - --advertise-addr=tikv:20160 - - --data-dir=/data/tikv - - --pd=pd:2379 - - --config=/tikv.toml - - --log-file=/logs/tikv.log - depends_on: - - "pd" - restart: on-failure - - - # monitors - ycsb: - image: pingcap/go-ycsb diff --git a/go-ycsb/tool/report.go b/go-ycsb/tool/report.go deleted file mode 100644 index f17499ca8..000000000 --- a/go-ycsb/tool/report.go +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bufio" - "fmt" - "io/ioutil" - "os" - "path" - "sort" - "strconv" - "strings" -) - -var ( - logsPath = "./logs" - operations = map[string]struct{}{ - "INSERT": struct{}{}, - "READ": struct{}{}, - "UPDATE": struct{}{}, - "SCAN": struct{}{}, - "READ_MODIFY_WRITE": struct{}{}, - "DELETE": struct{}{}, - } - workloads = map[string]struct{}{ - "load": struct{}{}, - "workloada": struct{}{}, - "workloadb": struct{}{}, - "workloadc": struct{}{}, - "workloadd": struct{}{}, - "workloade": struct{}{}, - "workloadf": struct{}{}, - } -) - -type stat struct { - OPS float64 - P99 float64 -} - -func statFieldFunc(c rune) bool { - return c == ':' || c == ',' -} - -func newStat(line string) (*stat, error) { - kvs := strings.FieldsFunc(line, statFieldFunc) - s := stat{} - if len(kvs)%2 != 0 { - println(line) - } - for i := 0; i < len(kvs); i += 2 { - v, err := strconv.ParseFloat(strings.TrimSpace(kvs[i+1]), 64) - if err != nil { - return nil, err - } - switch strings.TrimSpace(kvs[i]) { - case "OPS": - s.OPS = v - case "99th(us)": - s.P99 = v - default: - } - } - return &s, nil -} - -type dbStat struct { - db string - workload string - summary map[string]*stat - // progress map[string][]*stat -} - -func parseDBStat(pathName string) (*dbStat, error) { - // check db and workload from file name, the name format is: - // 1. db_load.log - // 2. db_run_workloadx.log - s := new(dbStat) - s.summary = make(map[string]*stat, 1) - // s.progress = make(map[string][]*stat, 1) - - fileName := path.Base(pathName) - seps := strings.Split(fileName, "_") - - if len(seps) != 2 { - return nil, nil - } - - s.db = seps[0] - workload := strings.TrimSuffix(seps[1], ".log") - if _, ok := workloads[workload]; !ok { - return nil, nil - } - - s.workload = workload - file, err := os.Open(pathName) - if err != nil { - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - handleSummary := false - - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "Run finished") { - handleSummary = true - continue - } - - seps := strings.Split(line, "-") - op := strings.TrimSpace(seps[0]) - if _, ok := operations[op]; !ok { - continue - } - - stat, err := newStat(strings.TrimSpace(seps[1])) - if err != nil { - return nil, err - } - - if handleSummary { - // handle summary logs - s.summary[op] = stat - } - - // TODO handle progress logs - // s.progress[op] = append(s.progress[op], stat) - } - - if err := scanner.Err(); err != nil { - return nil, err - } - return s, nil -} - -type dbStats []*dbStat - -func (a dbStats) Len() int { return len(a) } -func (a dbStats) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a dbStats) Less(i, j int) bool { return a[i].db < a[j].db } - -func reportDBStats(logsPath string, workload string, stats dbStats) error { - dir := path.Join(logsPath, "report") - os.MkdirAll(dir, 0755) - - fileName := path.Join(dir, fmt.Sprintf("%s_summary.csv", workload)) - sort.Sort(stats) - - file, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return err - } - defer file.Close() - - var fields []string - switch workload { - case "load": - fields = []string{"INSERT"} - case "workloada": - fields = []string{"READ", "UPDATE"} - case "workloadb": - fields = []string{"READ", "UPDATE"} - case "workloadc": - fields = []string{"READ"} - case "workloadd": - fields = []string{"READ", "INSERT"} - case "workloade": - fields = []string{"SCAN", "INSERT"} - case "workloadf": - fields = []string{"READ_MODIFY_WRITE"} - default: - } - - fmt.Fprintf(file, "DB") - for _, field := range fields { - fmt.Fprintf(file, ",%s OPS,%s P99(us)", field, field) - } - fmt.Fprint(file, "\n") - - for _, stat := range stats { - fmt.Fprintf(file, "%s", stat.db) - for _, field := range fields { - s := stat.summary[field] - if s == nil { - fmt.Fprintf(file, ",0.0") - } else { - fmt.Fprintf(file, ",%.f,%.f", s.OPS, s.P99) - } - } - fmt.Fprintf(file, "\n") - } - - return nil -} - -func main() { - if len(os.Args) >= 2 { - logsPath = os.Args[1] - } - - files, err := ioutil.ReadDir(logsPath) - if err != nil { - println(err.Error()) - return - } - - stats := make(map[string][]*dbStat) - for _, file := range files { - if file.IsDir() { - continue - } - - s, err := parseDBStat(path.Join(logsPath, file.Name())) - - if err != nil { - fmt.Printf("parse %s failed %v\n", file.Name(), err.Error()) - } - - if s == nil { - continue - } - - stats[s.workload] = append(stats[s.workload], s) - } - - for workload, s := range stats { - if err := reportDBStats(logsPath, workload, s); err != nil { - fmt.Printf("report %s failed %v\n", workload, err) - } - } -} diff --git a/go-ycsb/workloads/minio b/go-ycsb/workloads/minio deleted file mode 100644 index c4017fbec..000000000 --- a/go-ycsb/workloads/minio +++ /dev/null @@ -1,4 +0,0 @@ -minio.access-key=minio -minio.secret-key=myminio -minio.endpoint=127.0.0.1:9000 -minio.secure=false diff --git a/go-ycsb/workloads/workload_template b/go-ycsb/workloads/workload_template deleted file mode 100644 index 51aa1f2b0..000000000 --- a/go-ycsb/workloads/workload_template +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) 2012-2016 YCSB contributors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload Template: Default Values -# -# File contains all properties that can be set to define a -# YCSB session. All properties are set to their default -# value if one exists. If not, the property is commented -# out. When a property has a finite number of settings, -# the default is enabled and the alternates are shown in -# comments below it. -# -# Use of most explained through comments in Client.java or -# CoreWorkload.java or on the YCSB wiki page: -# https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties - -# The name of the workload class to use -workload=core - -# There is no default setting for recordcount but it is -# required to be set. -# The number of records in the table to be inserted in -# the load phase or the number of records already in the -# table before the run phase. -recordcount=1000000 - -# There is no default setting for operationcount but it is -# required to be set. -# The number of operations to use during the run phase. -operationcount=3000000 - -# The number of thread. -threadcount=500 - -# The number of insertions to do, if different from recordcount. -# Used with insertstart to grow an existing table. -#insertcount= - -# The offset of the first insertion -insertstart=0 - -# The number of fields in a record -fieldcount=10 - -# The size of each field (in bytes) -fieldlength=100 - -# Should read all fields -readallfields=true - -# Should write all fields on update -writeallfields=false - -# The distribution used to choose the length of a field -fieldlengthdistribution=constant -#fieldlengthdistribution=uniform -#fieldlengthdistribution=zipfian - -# What proportion of operations are reads -readproportion=0.95 - -# What proportion of operations are updates -updateproportion=0.05 - -# What proportion of operations are inserts -insertproportion=0 - -# What proportion of operations read then modify a record -readmodifywriteproportion=0 - -# What proportion of operations are scans -scanproportion=0 - -# On a single scan, the maximum number of records to access -maxscanlength=1000 - -# The distribution used to choose the number of records to access on a scan -scanlengthdistribution=uniform -#scanlengthdistribution=zipfian - -# Should records be inserted in order or pseudo-randomly -insertorder=hashed -#insertorder=ordered - -# The distribution of requests across the keyspace -requestdistribution=zipfian -#requestdistribution=uniform -#requestdistribution=latest - -# Percentage of data items that constitute the hot set -hotspotdatafraction=0.2 - -# Percentage of operations that access the hot set -hotspotopnfraction=0.8 - -# Maximum execution time in seconds -#maxexecutiontime= - -# The name of the database table to run queries against -table=usertable - -# The column family of fields (required by some databases) -#columnfamily= - -# How the latency measurements are presented -measurementtype=histogram -#measurementtype=timeseries -#measurementtype=raw -# When measurementtype is set to raw, measurements will be output -# as RAW datapoints in the following csv format: -# "operation, timestamp of the measurement, latency in us" -# -# Raw datapoints are collected in-memory while the test is running. Each -# data point consumes about 50 bytes (including java object overhead). -# For a typical run of 1 million to 10 million operations, this should -# fit into memory most of the time. If you plan to do 100s of millions of -# operations per run, consider provisioning a machine with larger RAM when using -# the RAW measurement type, or split the run into multiple runs. -# -# Optionally, you can specify an output file to save raw datapoints. -# Otherwise, raw datapoints will be written to stdout. -# The output file will be appended to if it already exists, otherwise -# a new output file will be created. -#measurement.raw.output_file = /tmp/your_output_file_for_this_run - -# JVM Reporting. -# -# Measure JVM information over time including GC counts, max and min memory -# used, max and min thread counts, max and min system load and others. This -# setting must be enabled in conjunction with the "-s" flag to run the status -# thread. Every "status.interval", the status thread will capture JVM -# statistics and record the results. At the end of the run, max and mins will -# be recorded. -# measurement.trackjvm = false - -# The range of latencies to track in the histogram (milliseconds) -histogram.buckets=1000 - -# Granularity for time series (in milliseconds) -timeseries.granularity=1000 - -# Latency reporting. -# -# YCSB records latency of failed operations separately from successful ones. -# Latency of all OK operations will be reported under their operation name, -# such as [READ], [UPDATE], etc. -# -# For failed operations: -# By default we don't track latency numbers of specific error status. -# We just report latency of all failed operation under one measurement name -# such as [READ-FAILED]. But optionally, user can configure to have either: -# 1. Record and report latency for each and every error status code by -# setting reportLatencyForEachError to true, or -# 2. Record and report latency for a select set of error status codes by -# providing a CSV list of Status codes via the "latencytrackederrors" -# property. -# reportlatencyforeacherror=false -# latencytrackederrors="" - -# Insertion error retry for the core workload. -# -# By default, the YCSB core workload does not retry any operations. -# However, during the load process, if any insertion fails, the entire -# load process is terminated. -# If a user desires to have more robust behavior during this phase, they can -# enable retry for insertion by setting the following property to a positive -# number. -# core_workload_insertion_retry_limit = 0 -# -# the following number controls the interval between retries (in seconds): -# core_workload_insertion_retry_interval = 3 - -# Distributed Tracing via Apache HTrace (http://htrace.incubator.apache.org/) -# -# Defaults to blank / no tracing -# Below sends to a local file, sampling at 0.1% -# -# htrace.sampler.classes=ProbabilitySampler -# htrace.sampler.fraction=0.001 -# htrace.span.receiver.classes=org.apache.htrace.core.LocalFileSpanReceiver -# htrace.local.file.span.receiver.path=/some/path/to/local/file -# -# To capture all spans, use the AlwaysSampler -# -# htrace.sampler.classes=AlwaysSampler -# -# To send spans to an HTraced receiver, use the below and ensure -# your classpath contains the htrace-htraced jar (i.e. when invoking the ycsb -# command add -cp /path/to/htrace-htraced.jar) -# -# htrace.span.receiver.classes=org.apache.htrace.impl.HTracedSpanReceiver -# htrace.htraced.receiver.address=example.com:9075 -# htrace.htraced.error.log.period.ms=10000 diff --git a/go-ycsb/workloads/workloada b/go-ycsb/workloads/workloada deleted file mode 100644 index ce71eb7cc..000000000 --- a/go-ycsb/workloads/workloada +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - - -# Yahoo! Cloud System Benchmark -# Workload A: Update heavy workload -# Application example: Session store recording recent actions -# -# Read/update ratio: 50/50 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -recordcount=1000 -operationcount=1000 -workload=core - -readallfields=true - -readproportion=0.5 -updateproportion=0.5 -scanproportion=0 -insertproportion=0 - -requestdistribution=uniform - diff --git a/go-ycsb/workloads/workloadb b/go-ycsb/workloads/workloadb deleted file mode 100644 index 9294450b6..000000000 --- a/go-ycsb/workloads/workloadb +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload B: Read mostly workload -# Application example: photo tagging; add a tag is an update, but most operations are to read tags -# -# Read/update ratio: 95/5 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -recordcount=1000 -operationcount=1000 -workload=core - -readallfields=true - -readproportion=0.95 -updateproportion=0.05 -scanproportion=0 -insertproportion=0 - -requestdistribution=uniform - diff --git a/go-ycsb/workloads/workloadc b/go-ycsb/workloads/workloadc deleted file mode 100644 index df49bccbc..000000000 --- a/go-ycsb/workloads/workloadc +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload C: Read only -# Application example: user profile cache, where profiles are constructed elsewhere (e.g., Hadoop) -# -# Read/update ratio: 100/0 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -recordcount=1000 -operationcount=1000 -workload=core - -readallfields=true - -readproportion=1 -updateproportion=0 -scanproportion=0 -insertproportion=0 - -requestdistribution=uniform - - - diff --git a/go-ycsb/workloads/workloadd b/go-ycsb/workloads/workloadd deleted file mode 100644 index bc09b75bd..000000000 --- a/go-ycsb/workloads/workloadd +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload D: Read latest workload -# Application example: user status updates; people want to read the latest -# -# Read/update/insert ratio: 95/0/5 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: latest - -# The insert order for this is hashed, not ordered. The "latest" items may be -# scattered around the keyspace if they are keyed by userid.timestamp. A workload -# which orders items purely by time, and demands the latest, is very different than -# workload here (which we believe is more typical of how people build systems.) - -recordcount=1000 -operationcount=1000 -workload=core - -readallfields=true - -readproportion=0.95 -updateproportion=0 -scanproportion=0 -insertproportion=0.05 - -requestdistribution=latest - diff --git a/go-ycsb/workloads/workloade b/go-ycsb/workloads/workloade deleted file mode 100644 index bfae98833..000000000 --- a/go-ycsb/workloads/workloade +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload E: Short ranges -# Application example: threaded conversations, where each scan is for the posts in a given thread (assumed to be clustered by thread id) -# -# Scan/insert ratio: 95/5 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -# The insert order is hashed, not ordered. Although the scans are ordered, it does not necessarily -# follow that the data is inserted in order. For example, posts for thread 342 may not be inserted contiguously, but -# instead interspersed with posts from lots of other threads. The way the YCSB client works is that it will pick a start -# key, and then request a number of records; this works fine even for hashed insertion. - -recordcount=1000 -operationcount=1000 -workload=core - -readallfields=true - -readproportion=0 -updateproportion=0 -scanproportion=0.95 -insertproportion=0.05 - -requestdistribution=uniform - -maxscanlength=1 - -scanlengthdistribution=uniform - - diff --git a/go-ycsb/workloads/workloadf b/go-ycsb/workloads/workloadf deleted file mode 100644 index e754d1be0..000000000 --- a/go-ycsb/workloads/workloadf +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2010 Yahoo! Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you -# may not use this file except in compliance with the License. You -# may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. See accompanying -# LICENSE file. - -# Yahoo! Cloud System Benchmark -# Workload F: Read-modify-write workload -# Application example: user database, where user records are read and modified by the user or to record user activity. -# -# Read/read-modify-write ratio: 50/50 -# Default data size: 1 KB records (10 fields, 100 bytes each, plus key) -# Request distribution: zipfian - -recordcount=1000 -operationcount=1000 -workload=core - -readallfields=true - -readproportion=0.5 -updateproportion=0 -scanproportion=0 -insertproportion=0 -readmodifywriteproportion=0.5 - -requestdistribution=uniform - From 7828f4c796dd12ff804419c4f954596c70497caf Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Wed, 22 Mar 2023 22:30:15 +0800 Subject: [PATCH 12/22] add submodules --- .gitmodules | 3 +++ go-ycsb | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 go-ycsb diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..9b6e3b701 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "go-ycsb"] + path = go-ycsb + url = git@github.com:QingyangZ/go-ycsb.git diff --git a/go-ycsb b/go-ycsb new file mode 160000 index 000000000..060a9db20 --- /dev/null +++ b/go-ycsb @@ -0,0 +1 @@ +Subproject commit 060a9db207f9f56eadea145f68e4e06296487a29 From c11b568d7e6ca37182b2573cdbf6141d7a523184 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Mon, 22 May 2023 21:16:48 +0800 Subject: [PATCH 13/22] add leveldb --- .gitmodules | 3 + doc/tinydb/Home.md | 26 ++++----- doc/tinydb/project1-LSMTree.md | 55 +++++++++++++++++++ ...eparation.md => project2-KV Separation.md} | 0 leveldb | 1 + 5 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 doc/tinydb/project1-LSMTree.md rename doc/tinydb/{project1-KV Separation.md => project2-KV Separation.md} (100%) create mode 160000 leveldb diff --git a/.gitmodules b/.gitmodules index 9b6e3b701..4c7099ad2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "go-ycsb"] path = go-ycsb url = git@github.com:QingyangZ/go-ycsb.git +[submodule "leveldb"] + path = leveldb + url = https://github.com/QingyangZ/leveldb.git diff --git a/doc/tinydb/Home.md b/doc/tinydb/Home.md index dbb25562e..1b0cd2ed9 100644 --- a/doc/tinydb/Home.md +++ b/doc/tinydb/Home.md @@ -1,19 +1,19 @@ -# Welcome to the TinyDB! -TinyDB is a course designed to help you quickly familiarize yourself with the underlying storage engine of [TiKV Project](https://github.com/tikv/tikv). +# Welcome to the TinyEngine! +TinyEngine is a course designed to help you quickly familiarize yourself with the underlying storage engine of [TiKV Project](https://github.com/tikv/tikv). -After completing this course, you will have a better understanding of LSM-tree based KV stores with high throughput, high scalability and high space utilization. +After completing this course, you will have a better understanding of LSM-tree-based KV stores with high throughput, high scalability and high space utilization. ## Course Introduction -TinyDB is forked from open source [TinyKV](https://github.com/talent-plan/tinykv), a key-value storage system with the Raft consensus algorithm. TinyKV focuses on the storage layer of a distributed database system, which uses [badger](https://github.com/dgraph-io/badger), a Go library to store keys and values, as its storage engine. In order to get closer to the actual implementation of TiKV, TinyDB plans to replace the original storage engine badger with [LevelDB](https://github.com/google/leveldb)/[RocksDB](https://github.com/facebook/rocksdb) wrapped by Golang. Therefore, please modify your implementation of project1 to use the interface of levigo(a wrapper of LevelDB) or gorocksdb(a wrapper of RocksDB) rather than badger. +TinyEngine is forked from open source [TinyKV](https://github.com/talent-plan/tinykv), a key-value storage system with the Raft consensus algorithm. TinyKV focuses on the storage layer of a distributed database system, which uses [badger](https://github.com/dgraph-io/badger), a Go library to store keys and values, as its storage engine. In order to get closer to the actual implementation of TiKV, TinyEngine plans to replace the original storage engine badger with [LevelDB](https://github.com/google/leveldb)/[RocksDB](https://github.com/facebook/rocksdb) wrapped by Golang. Therefore, please modify your implementation of project1 to use the interface of levigo(a wrapper of LevelDB) or gorocksdb(a wrapper of RocksDB) rather than badger. -In this course, you need to implement an existing optimization method on LevelDB/RocksDB. We provide several projects in this folder, which introduce some classic and generally accepted optimization ideas presented in recent famous paper. Please choose one and implement it. +In this course, you need to finish the implementation of LevelDB and then implement an existing optimization method on LevelDB/RocksDB. We provide several projects in this folder, which introduce some classic and generally accepted optimization ideas presented in recent famous paper. Please choose one and implement it. After completing the implementation, you need to test and evaluate your optimization. We provide go-ycsb, which can be used to evaluate database performance. If you successfully implement a project, you will get better performance in reading or writing or some other dimension. Finally, you need to chart your evaluation results and submit a report and source code. The experts will give you an appropriate score depending on your optimization results and report. ### LevelDB/RocksDB [LevelDB](https://github.com/google/leveldb)/[RocksDB](https://github.com/facebook/rocksdb) is a storage engine for server workloads on various storage media, with the initial focus on fast storage (especially Flash storage). It is a C++ library to store key-value pairs. It supports both point lookups and range scans, and provides different types of ACID guarantees. -RocksDB borrows significant code from the open source [leveldb](https://code.google.com/google/leveldb/) project and does a lot of performance optimization. It performs better than LevelDB under many real workloads and TiKV uses it as storage engine. However, RocksDB has a higher amount of code and is more difficult to learn. As a beginner's course for KV storage, TinyDB requires you to complete the course with LevelDB. Besides, if you already have the knowledge of LSM-tree or enough coding ability, TinyDB also provide code based on the RocksDB. +RocksDB borrows significant code from the open source [Leveldb](https://code.google.com/google/leveldb/) project and does a lot of performance optimization. It performs better than LevelDB under many real workloads and TiKV uses it as storage engine. However, RocksDB has a higher amount of code and is more difficult to learn. As a beginner's course for KV storage, TinyEngine requires you to complete the course with LevelDB. Besides, if you already have the knowledge of LSM-tree or enough coding ability, TinyEngine also provide code based on the RocksDB. ## Evaluation & Report After finishing the project, you need to present evaluation results that demonstrate the benefits of your optimization and write a report. After submitting, you will receive a score based on the optimization results and your report. @@ -21,14 +21,14 @@ After finishing the project, you need to present evaluation results that demonst ### Go-ycsb Go-ycsb is a Go port of YCSB. It fully supports all YCSB generators and the Core workload so we can do the basic CRUD benchmarks with Go. Please follow [go-ycsb](https://github.com/pingcap/go-ycsb/blob/master/README.md) to understand it. There are different workloads can be used in the go-ycsb/workloads folder. The specific parameters can be changed if needed. -## Build TinyDB from Source +## Build TinyEngine from Source ### Prerequisites -* `git`: The source code of TinyDB is hosted on GitHub as a git repository. To work with git repository, please [install `git`](https://git-scm.com/downloads). -* `go`: TinyDB is a Go project. To build TinyDB from source, please [install `go`](https://golang.org/doc/install) with version greater or equal to 1.13. -* `leveldb`: LevelDB is a storage engine of TinyDB, please [install `leveldb`](https://github.com/google/leveldb) with version greater or equal to 1.7. -* `rocksdb`: RocksDB is also a storage engine of TinyDB, please [install `rocksdb`](https://github.com/facebook/rocksdb) with version greater or equal to 5.16. +* `git`: The source code of TinyEngine is hosted on GitHub as a git repository. To work with git repository, please [install `git`](https://git-scm.com/downloads). +* `go`: TinyEngine is a Go project. To build TinyEngine from source, please [install `go`](https://golang.org/doc/install) with version greater or equal to 1.13. +* `leveldb`: LevelDB is a storage engine of TinyEngine, please [install `leveldb`](https://github.com/google/leveldb) with version greater or equal to 1.7. +* `rocksdb`: RocksDB is also a storage engine of TinyEngine, please [install `rocksdb`](https://github.com/facebook/rocksdb) with version greater or equal to 5.16. ### Clone @@ -40,7 +40,7 @@ git clone https://github.com/QingyangZ/tinydb ### Build -Build TinyDB from the source code. +Build TinyEngine from the source code. ```bash cd tinydb @@ -49,7 +49,7 @@ make ### Go-ycsb Test -Build go-ycsb and test TinyDB +Build go-ycsb and test TinyEngine ```bash cd go-ycsb diff --git a/doc/tinydb/project1-LSMTree.md b/doc/tinydb/project1-LSMTree.md new file mode 100644 index 000000000..fc3b220a8 --- /dev/null +++ b/doc/tinydb/project1-LSMTree.md @@ -0,0 +1,55 @@ +# Project1 LSM Tree +In this project, you will learn the basic architecture of LSM tree and finish the implementation of LevelDB, a representative LSM-tree-based key/value storage engine. + +The log-structured merge-tree (also known as LSM tree, or LSMT) is a data structure with performance characteristics that make it attractive for providing indexed access to files with high insert volume, such as transactional log data. LSM trees, like other search trees, maintain key-value pairs. LSM trees maintain data in two or more separate structures, each of which is optimized for its respective underlying storage medium; data is synchronized between the two structures efficiently, in batches. + +LSM tree is widely used in mainstream persistent KV storage systems like LevelDB and RocksDB. RocksDB, which is used as TiKV's storage engine, borrows significant code from the open source [Leveldb](https://code.google.com/google/leveldb/) project and does a lot of performance optimization. However, RocksDB has a higher amount of code and is more difficult to learn. As a beginner's course for KV storage, this project is based on LevelDB. + +### Architecture +![lsmtree](imgs/lsmtree.png) + +A brief introduction to the architecture of LSM tree is provided later in this article. For more details, you can also read our collection of documents at . + + +#### Memtable +`Memtable` is an in-memory component in which the data incoming from client requests is first written, which is basically a skip-list, an ordered data structure. After the skip-list has got enough entries and has hit its threshold memory it is transformed into an immutable Memtable, and then it waits to be flushed to the Disk, sorted in form of `SSTable`. + +#### SSTable +`SSTable` or sorted string table as the name explains contains the entry in sorted format on keys on disk. When the sorted data from RAM is flushed to disk it is stored in form of `SSTable`. SSTables are divided into different levels, with lower levels storing newer entries and higher levels storing older entries. The SSTable within each level are ordered(except Level0) and the SSTables between the different levels are disordered. After the memtable has reach its threshold size, it is first flushed to Level0. After a while the entries will be moved to a higher level by `compaction`. + +#### Log +`Log` in LevelDB is a write ahead log. As mentioned earlier, entries written by LevelDB is first saved to MemTable. To prevent data loss due to downtime, data is persisted to the log file before being written to MemTable. After Memtable is flushed to the disk, related data can be deleted from the log. + +### The Code +In this part, you will finish the implementation of LevelDB, which involves three important operations in LSM tree: Get/Compaction/Scan. It maintains a simple database of key/value pairs. Keys and values are strings. `Get` queries the database and fetches the newest value for a key. There may be multiple versions of the same key in the database because of the append nature of the LSM tree. `Compaction` merges some files into the next layer and does garbage collection. `Scan` fetches the current value for a series of keys. + +#### 1. Get +The code you need to implement is in `db/db_impl.cc` and `db/version_set.cc`. +- Inside `db/db_impl.cc`, you need to complete the function `DBImpl::Get`(Black1.1), which searches Memtable, immutable Memtable, and SSTable until the key is found and returns the value. +- Inside `db/version_set.cc`, you need to complete the function `Version::ForEachOverlapping`(Black1.2) and the function `Version::Get`(Black1.3). + +#### 2. Compaction +In this part, the code you need to implement is in `db/db_impl.cc` and `db/version_set.cc`. +- Inside `db/db_impl.cc`, you need to complete the function `DBImpl::BackgroundCompaction`(Black2.1) and the function `DBImpl::InstallCompactionResults`(Black2.2). +- Inside `db/version_set.cc`, you need to complete the function `VersionSet::Finalize`(Black2.3) and the function `VersionSet::PickCompaction`(Black2.4) + +#### 3. Scan +In this part, the code you need to implement is in `db/version_set.cc` and `table/merger.cc`. +- Inside `db/version_set.cc`, you need to complete the function `Version::AddIterators`(Black3.1). +- Inside `table/merger.cc`, you need to complete the function `Seek`(Black3.2). + +### Test + +Make: +```bash +cd leveldb +mkdir -p build && cd build +cmake -DCMAKE_BUILD_TYPE=Release .. && cmake --build . +``` +Please see the CMake documentation and `CMakeLists.txt` for more advanced usage. + +Test: +```bash +./db_bench +``` +Please see the CMake documentation and `leveldb/benchmarks/db_bench.cc` for more advanced usage. \ No newline at end of file diff --git a/doc/tinydb/project1-KV Separation.md b/doc/tinydb/project2-KV Separation.md similarity index 100% rename from doc/tinydb/project1-KV Separation.md rename to doc/tinydb/project2-KV Separation.md diff --git a/leveldb b/leveldb new file mode 160000 index 000000000..68d28d054 --- /dev/null +++ b/leveldb @@ -0,0 +1 @@ +Subproject commit 68d28d054c60a6ba58852bc352de3fd2898d3b4d From d56e530e200cbcc9d49255a7052659c65c331d40 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 15:04:37 +0800 Subject: [PATCH 14/22] 0 --- kv/config/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kv/config/config.go b/kv/config/config.go index 3301ca54d..ba54229b6 100644 --- a/kv/config/config.go +++ b/kv/config/config.go @@ -105,6 +105,10 @@ func NewTestConfig() *Config { SchedulerStoreHeartbeatTickInterval: 500 * time.Millisecond, RegionMaxSize: 144 * MB, RegionSplitSize: 96 * MB, +<<<<<<< HEAD DBPath: "./dbtest_rdb", +======= + DBPath: "/tmp/badger", +>>>>>>> parent of 44e0d89... modify db_path } } From af1547e9a18c8888460d7f0ab1ca2dbe665c11c7 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 15:08:50 +0800 Subject: [PATCH 15/22] 0 --- kv/config/config.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/kv/config/config.go b/kv/config/config.go index ba54229b6..3301ca54d 100644 --- a/kv/config/config.go +++ b/kv/config/config.go @@ -105,10 +105,6 @@ func NewTestConfig() *Config { SchedulerStoreHeartbeatTickInterval: 500 * time.Millisecond, RegionMaxSize: 144 * MB, RegionSplitSize: 96 * MB, -<<<<<<< HEAD DBPath: "./dbtest_rdb", -======= - DBPath: "/tmp/badger", ->>>>>>> parent of 44e0d89... modify db_path } } From e5abdd8555824fbfad8522117e0d78464f53fb72 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 17:16:30 +0800 Subject: [PATCH 16/22] checkout --- .gitmodules | 3 + kv/server/raw_api.go | 103 +++++++++++++++++- .../standalone_storage/standalone_storage.go | 52 ++++++++- levigo/levigo.sh | 10 +- snappy | 1 + 5 files changed, 158 insertions(+), 11 deletions(-) create mode 160000 snappy diff --git a/.gitmodules b/.gitmodules index 4c7099ad2..01c3b20ea 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "leveldb"] path = leveldb url = https://github.com/QingyangZ/leveldb.git +[submodule "snappy"] + path = snappy + url = https://github.com/google/snappy.git diff --git a/kv/server/raw_api.go b/kv/server/raw_api.go index e9a9e6983..7b4574093 100644 --- a/kv/server/raw_api.go +++ b/kv/server/raw_api.go @@ -14,26 +14,121 @@ import ( // RawGet return the corresponding Get response based on RawGetRequest's CF and Key fields func (server *Server) RawGet(_ context.Context, req *kvrpcpb.RawGetRequest) (*kvrpcpb.RawGetResponse, error) { // Your Code Here (1). - return nil, nil + resp := &kvrpcpb.RawGetResponse{} + reader, err := server.storage.Reader(req.Context) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + defer reader.Close() + val, err := reader.GetCF(req.Cf, req.Key) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + resp.Value = val + if val == nil { + resp.NotFound = true + } + return resp, nil } // RawPut puts the target data into storage and returns the corresponding response func (server *Server) RawPut(_ context.Context, req *kvrpcpb.RawPutRequest) (*kvrpcpb.RawPutResponse, error) { // Your Code Here (1). // Hint: Consider using Storage.Modify to store data to be modified - return nil, nil + resp := &kvrpcpb.RawPutResponse{} + batch := []storage.Modify{ + { + Data: storage.Put{ + Cf: req.Cf, + Key: req.Key, + Value: req.Value, + }, + }} + err := server.storage.Write(req.Context, batch) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + return resp, nil } // RawDelete delete the target data from storage and returns the corresponding response func (server *Server) RawDelete(_ context.Context, req *kvrpcpb.RawDeleteRequest) (*kvrpcpb.RawDeleteResponse, error) { // Your Code Here (1). // Hint: Consider using Storage.Modify to store data to be deleted - return nil, nil + resp := &kvrpcpb.RawDeleteResponse{} + batch := []storage.Modify{ + { + Data: storage.Delete{ + Cf: req.Cf, + Key: req.Key, + }, + }} + err := server.storage.Write(req.Context, batch) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + return resp, nil } // RawScan scan the data starting from the start key up to limit. and return the corresponding result func (server *Server) RawScan(_ context.Context, req *kvrpcpb.RawScanRequest) (*kvrpcpb.RawScanResponse, error) { // Your Code Here (1). // Hint: Consider using reader.IterCF - return nil, nil + resp := &kvrpcpb.RawScanResponse{} + if req.Limit == 0 { + return resp, nil + } + reader, err := server.storage.Reader(req.Context) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + defer reader.Close() + iter := reader.IterCF(req.Cf) + if err != nil { + if regionErr, ok := err.(*raft_storage.RegionError); ok { + resp.RegionError = regionErr.RequestErr + return resp, nil + } + return nil, err + } + defer iter.Close() + var pairs []*kvrpcpb.KvPair + n := req.Limit + for iter.Seek(req.StartKey); iter.Valid(); iter.Next() { + item := iter.Item() + val, err := item.ValueCopy(nil) + if err != nil { + return nil, err + } + pairs = append(pairs, &kvrpcpb.KvPair{ + Key: item.KeyCopy(nil), + Value: val, + }) + n-- + if n == 0 { + break + } + } + resp.Kvs = pairs + return resp, nil } diff --git a/kv/storage/standalone_storage/standalone_storage.go b/kv/storage/standalone_storage/standalone_storage.go index 80c793f9a..5ba48c658 100644 --- a/kv/storage/standalone_storage/standalone_storage.go +++ b/kv/storage/standalone_storage/standalone_storage.go @@ -14,11 +14,22 @@ import ( // communicate with other nodes and all data is stored locally. type StandAloneStorage struct { // Your Data Here (1). + db *levigo.DB + roptions *levigo.ReadOptions + woptions *levigo.WriteOptions } func NewStandAloneStorage(conf *config.Config) *StandAloneStorage { // Your Code Here (1). - return nil + dbName := conf.DBPath + opt := levigo.NewOptions() + ropt := levigo.NewReadOptions() + wopt := levigo.NewWriteOptions() + opt.SetCreateIfMissing(true) + opt.SetWriteBufferSize(67108864) + policy := levigo.NewBloomFilter(10) + opt.SetFilterPolicy(policy) + db, err := levigo.Open(dbName, opt) } func (s *StandAloneStorage) Start() error { @@ -28,15 +39,52 @@ func (s *StandAloneStorage) Start() error { func (s *StandAloneStorage) Stop() error { // Your Code Here (1). + s.db.Close() return nil } func (s *StandAloneStorage) Reader(ctx *kvrpcpb.Context) (storage.StorageReader, error) { // Your Code Here (1). - return nil, nil + return &StandAloneStorageReader{ + s.db, + s.roptions, + }, nil } func (s *StandAloneStorage) Write(ctx *kvrpcpb.Context, batch []storage.Modify) error { // Your Code Here (1). + for _, m := range batch { + switch m.Data.(type) { + case storage.Put: + put := m.Data.(storage.Put) + if err := s.db.Put(s.woptions, engine_util.KeyWithCF(put.Cf, put.Key), put.Value); err != nil { + return err + } + case storage.Delete: + del := m.Data.(storage.Delete) + if err := s.db.Delete(s.woptions, engine_util.KeyWithCF(del.Cf, del.Key)); err != nil { + return err + } + } + } return nil } + +type StandAloneStorageReader struct { + db *levigo.DB + roptions *levigo.ReadOptions +} + +func (sReader *StandAloneStorageReader) Close() { + return +} + +func (sReader *StandAloneStorageReader) GetCF(cf string, key []byte) ([]byte, error) { + val, err := sReader.db.Get(sReader.roptions, engine_util.KeyWithCF(cf, key)) + + return val, err +} + +func (sReader *StandAloneStorageReader) IterCF(cf string) engine_util.DBIterator { + return engine_util.NewLDBIterator(cf, sReader.db, sReader.roptions) +} \ No newline at end of file diff --git a/levigo/levigo.sh b/levigo/levigo.sh index 9008d5fcd..b25fabd68 100755 --- a/levigo/levigo.sh +++ b/levigo/levigo.sh @@ -2,11 +2,11 @@ #refer https://github.com/norton/lets/blob/master/c_src/build_deps.sh #你必须在这里设置实际的snappy以及leveldb源代码地址 -SNAPPY_SRC=/home/kvgroup/QingyangZ/kv/snappy -LEVELDB_SRC=/home/kvgroup/QingyangZ/kv/leveldb +SNAPPY_SRC=../snappy +LEVELDB_SRC=../leveldb -SNAPPY_DIR=/home/kvgroup/QingyangZ/kv/snappy/build -LEVELDB_DIR=/home/kvgroup/QingyangZ/kv/leveldb/build +SNAPPY_DIR=../snappy/build +LEVELDB_DIR=../leveldb/build if [ ! -f $SNAPPY_DIR/libsnappy.a ]; then (cd $SNAPPY_SRC && \ @@ -17,7 +17,7 @@ else echo "skip install snappy" fi -if [ ! -f $LEVELDB_DIR/libleveldb.so ]; then +if [ ! -f $LEVELDB_DIR/libleveldb.a ]; then (cd $LEVELDB_SRC && \ echo "echo \"PLATFORM_CFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform && echo "echo \"PLATFORM_CXXFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform && diff --git a/snappy b/snappy new file mode 160000 index 000000000..c9f9edf6d --- /dev/null +++ b/snappy @@ -0,0 +1 @@ +Subproject commit c9f9edf6d75bb065fa47468bf035e051a57bec7c From 12932e174a4704ed97d4b5525af762ef9f7992bb Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 17:19:20 +0800 Subject: [PATCH 17/22] correction --- doc/tinydb/project1-LSMTree.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/tinydb/project1-LSMTree.md b/doc/tinydb/project1-LSMTree.md index fc3b220a8..04af9f07a 100644 --- a/doc/tinydb/project1-LSMTree.md +++ b/doc/tinydb/project1-LSMTree.md @@ -25,18 +25,18 @@ In this part, you will finish the implementation of LevelDB, which involves thre #### 1. Get The code you need to implement is in `db/db_impl.cc` and `db/version_set.cc`. -- Inside `db/db_impl.cc`, you need to complete the function `DBImpl::Get`(Black1.1), which searches Memtable, immutable Memtable, and SSTable until the key is found and returns the value. -- Inside `db/version_set.cc`, you need to complete the function `Version::ForEachOverlapping`(Black1.2) and the function `Version::Get`(Black1.3). +- Inside `db/db_impl.cc`, you need to complete the function `DBImpl::Get`(Blank1.1), which searches Memtable, immutable Memtable, and SSTable until the key is found and returns the value. +- Inside `db/version_set.cc`, you need to complete the function `Version::ForEachOverlapping`(Blank1.2) and the function `Version::Get`(Blank1.3). #### 2. Compaction In this part, the code you need to implement is in `db/db_impl.cc` and `db/version_set.cc`. -- Inside `db/db_impl.cc`, you need to complete the function `DBImpl::BackgroundCompaction`(Black2.1) and the function `DBImpl::InstallCompactionResults`(Black2.2). -- Inside `db/version_set.cc`, you need to complete the function `VersionSet::Finalize`(Black2.3) and the function `VersionSet::PickCompaction`(Black2.4) +- Inside `db/db_impl.cc`, you need to complete the function `DBImpl::BackgroundCompaction`(Blank2.1) and the function `DBImpl::InstallCompactionResults`(Blank2.2). +- Inside `db/version_set.cc`, you need to complete the function `VersionSet::Finalize`(Blank2.3) and the function `VersionSet::PickCompaction`(Blank2.4) #### 3. Scan In this part, the code you need to implement is in `db/version_set.cc` and `table/merger.cc`. -- Inside `db/version_set.cc`, you need to complete the function `Version::AddIterators`(Black3.1). -- Inside `table/merger.cc`, you need to complete the function `Seek`(Black3.2). +- Inside `db/version_set.cc`, you need to complete the function `Version::AddIterators`(Blank3.1). +- Inside `table/merger.cc`, you need to complete the function `Seek`(Blank3.2). ### Test From 9879945f660b7ea5ed7b88c018b4db336e8d2c57 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 19:42:19 +0800 Subject: [PATCH 18/22] leveldb --- leveldb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leveldb b/leveldb index 68d28d054..da7123637 160000 --- a/leveldb +++ b/leveldb @@ -1 +1 @@ -Subproject commit 68d28d054c60a6ba58852bc352de3fd2898d3b4d +Subproject commit da7123637f2b4d8d7cf3b8be85f54d59469a1e36 From 3ab639438829f48e00b54cd6e610a93aae0bd304 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 19:58:36 +0800 Subject: [PATCH 19/22] remove a submodule --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 01c3b20ea..855b4b701 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "go-ycsb"] path = go-ycsb url = git@github.com:QingyangZ/go-ycsb.git -[submodule "leveldb"] - path = leveldb - url = https://github.com/QingyangZ/leveldb.git [submodule "snappy"] path = snappy url = https://github.com/google/snappy.git From cbcd2841db2f31f885daf1aa6f0295894e21e4a0 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 20:02:58 +0800 Subject: [PATCH 20/22] 0 --- leveldb | 1 - 1 file changed, 1 deletion(-) delete mode 160000 leveldb diff --git a/leveldb b/leveldb deleted file mode 160000 index da7123637..000000000 --- a/leveldb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit da7123637f2b4d8d7cf3b8be85f54d59469a1e36 From 7f0620cb37fb4afa3d78ee18c97ceece62ec8e95 Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 20:03:44 +0800 Subject: [PATCH 21/22] 1 --- leveldb/.clang-format | 18 + leveldb/.github/workflows/build.yml | 102 + leveldb/.gitignore | 8 + leveldb/.gitmodules | 6 + leveldb/AUTHORS | 12 + leveldb/CMakeLists.txt | 519 +++++ leveldb/CONTRIBUTING.md | 31 + leveldb/LICENSE | 27 + leveldb/NEWS | 17 + leveldb/README.md | 255 +++ leveldb/TODO | 14 + leveldb/benchmarks/db_bench.cc | 1159 +++++++++++ leveldb/benchmarks/db_bench_log.cc | 92 + leveldb/benchmarks/db_bench_sqlite3.cc | 726 +++++++ leveldb/benchmarks/db_bench_tree_db.cc | 531 +++++ leveldb/cmake/leveldbConfig.cmake.in | 9 + leveldb/db/autocompact_test.cc | 110 ++ leveldb/db/builder.cc | 82 + leveldb/db/builder.h | 30 + leveldb/db/c.cc | 565 ++++++ leveldb/db/c_test.c | 384 ++++ leveldb/db/corruption_test.cc | 362 ++++ leveldb/db/db_impl.cc | 1579 +++++++++++++++ leveldb/db/db_impl.h | 217 +++ leveldb/db/db_iter.cc | 318 +++ leveldb/db/db_iter.h | 26 + leveldb/db/db_test.cc | 2360 +++++++++++++++++++++++ leveldb/db/dbformat.cc | 136 ++ leveldb/db/dbformat.h | 224 +++ leveldb/db/dbformat_test.cc | 128 ++ leveldb/db/dumpfile.cc | 232 +++ leveldb/db/fault_injection_test.cc | 550 ++++++ leveldb/db/filename.cc | 141 ++ leveldb/db/filename.h | 83 + leveldb/db/filename_test.cc | 127 ++ leveldb/db/leveldbutil.cc | 64 + leveldb/db/log_format.h | 35 + leveldb/db/log_reader.cc | 274 +++ leveldb/db/log_reader.h | 112 ++ leveldb/db/log_test.cc | 558 ++++++ leveldb/db/log_writer.cc | 111 ++ leveldb/db/log_writer.h | 54 + leveldb/db/memtable.cc | 138 ++ leveldb/db/memtable.h | 87 + leveldb/db/recovery_test.cc | 339 ++++ leveldb/db/repair.cc | 451 +++++ leveldb/db/skiplist.h | 380 ++++ leveldb/db/skiplist_test.cc | 368 ++++ leveldb/db/snapshot.h | 95 + leveldb/db/table_cache.cc | 120 ++ leveldb/db/table_cache.h | 61 + leveldb/db/version_edit.cc | 258 +++ leveldb/db/version_edit.h | 106 + leveldb/db/version_edit_test.cc | 41 + leveldb/db/version_set.cc | 1573 +++++++++++++++ leveldb/db/version_set.h | 393 ++++ leveldb/db/version_set_test.cc | 331 ++++ leveldb/db/write_batch.cc | 150 ++ leveldb/db/write_batch_internal.h | 45 + leveldb/db/write_batch_test.cc | 132 ++ leveldb/doc/benchmark.html | 459 +++++ leveldb/doc/imgs/lsmtree.png | Bin 0 -> 131549 bytes leveldb/doc/impl.md | 172 ++ leveldb/doc/index.md | 524 +++++ leveldb/doc/log_format.md | 75 + leveldb/doc/project1-LSMTree.md | 54 + leveldb/doc/table_format.md | 107 + leveldb/helpers/memenv/memenv.cc | 390 ++++ leveldb/helpers/memenv/memenv.h | 22 + leveldb/helpers/memenv/memenv_test.cc | 259 +++ leveldb/include/leveldb/c.h | 270 +++ leveldb/include/leveldb/cache.h | 103 + leveldb/include/leveldb/comparator.h | 64 + leveldb/include/leveldb/db.h | 167 ++ leveldb/include/leveldb/dumpfile.h | 28 + leveldb/include/leveldb/env.h | 417 ++++ leveldb/include/leveldb/export.h | 33 + leveldb/include/leveldb/filter_policy.h | 72 + leveldb/include/leveldb/iterator.h | 112 ++ leveldb/include/leveldb/options.h | 186 ++ leveldb/include/leveldb/slice.h | 114 ++ leveldb/include/leveldb/status.h | 122 ++ leveldb/include/leveldb/table.h | 84 + leveldb/include/leveldb/table_builder.h | 93 + leveldb/include/leveldb/write_batch.h | 83 + leveldb/issues/issue178_test.cc | 85 + leveldb/issues/issue200_test.cc | 54 + leveldb/issues/issue320_test.cc | 126 ++ leveldb/port/README.md | 10 + leveldb/port/port.h | 19 + leveldb/port/port_config.h.in | 38 + leveldb/port/port_example.h | 119 ++ leveldb/port/port_stdcxx.h | 218 +++ leveldb/port/thread_annotations.h | 108 ++ leveldb/table/block.cc | 292 +++ leveldb/table/block.h | 44 + leveldb/table/block_builder.cc | 107 + leveldb/table/block_builder.h | 54 + leveldb/table/filter_block.cc | 106 + leveldb/table/filter_block.h | 68 + leveldb/table/filter_block_test.cc | 122 ++ leveldb/table/format.cc | 160 ++ leveldb/table/format.h | 99 + leveldb/table/iterator.cc | 76 + leveldb/table/iterator_wrapper.h | 92 + leveldb/table/merger.cc | 189 ++ leveldb/table/merger.h | 26 + leveldb/table/table.cc | 271 +++ leveldb/table/table_builder.cc | 279 +++ leveldb/table/table_test.cc | 842 ++++++++ leveldb/table/two_level_iterator.cc | 171 ++ leveldb/table/two_level_iterator.h | 31 + leveldb/util/arena.cc | 66 + leveldb/util/arena.h | 71 + leveldb/util/arena_test.cc | 61 + leveldb/util/bloom.cc | 92 + leveldb/util/bloom_test.cc | 154 ++ leveldb/util/cache.cc | 401 ++++ leveldb/util/cache_test.cc | 224 +++ leveldb/util/coding.cc | 156 ++ leveldb/util/coding.h | 122 ++ leveldb/util/coding_test.cc | 193 ++ leveldb/util/comparator.cc | 75 + leveldb/util/crc32c.cc | 380 ++++ leveldb/util/crc32c.h | 43 + leveldb/util/crc32c_test.cc | 56 + leveldb/util/env.cc | 108 ++ leveldb/util/env_posix.cc | 926 +++++++++ leveldb/util/env_posix_test.cc | 353 ++++ leveldb/util/env_posix_test_helper.h | 28 + leveldb/util/env_test.cc | 248 +++ leveldb/util/env_windows.cc | 816 ++++++++ leveldb/util/env_windows_test.cc | 65 + leveldb/util/env_windows_test_helper.h | 25 + leveldb/util/filter_policy.cc | 11 + leveldb/util/hash.cc | 55 + leveldb/util/hash.h | 19 + leveldb/util/hash_test.cc | 41 + leveldb/util/histogram.cc | 272 +++ leveldb/util/histogram.h | 44 + leveldb/util/logging.cc | 82 + leveldb/util/logging.h | 44 + leveldb/util/logging_test.cc | 140 ++ leveldb/util/mutexlock.h | 39 + leveldb/util/no_destructor.h | 46 + leveldb/util/no_destructor_test.cc | 44 + leveldb/util/options.cc | 14 + leveldb/util/posix_logger.h | 130 ++ leveldb/util/random.h | 63 + leveldb/util/status.cc | 77 + leveldb/util/status_test.cc | 39 + leveldb/util/testutil.cc | 51 + leveldb/util/testutil.h | 82 + leveldb/util/windows_logger.h | 124 ++ 154 files changed, 31352 insertions(+) create mode 100644 leveldb/.clang-format create mode 100644 leveldb/.github/workflows/build.yml create mode 100644 leveldb/.gitignore create mode 100644 leveldb/.gitmodules create mode 100644 leveldb/AUTHORS create mode 100644 leveldb/CMakeLists.txt create mode 100644 leveldb/CONTRIBUTING.md create mode 100644 leveldb/LICENSE create mode 100644 leveldb/NEWS create mode 100644 leveldb/README.md create mode 100644 leveldb/TODO create mode 100644 leveldb/benchmarks/db_bench.cc create mode 100644 leveldb/benchmarks/db_bench_log.cc create mode 100644 leveldb/benchmarks/db_bench_sqlite3.cc create mode 100644 leveldb/benchmarks/db_bench_tree_db.cc create mode 100644 leveldb/cmake/leveldbConfig.cmake.in create mode 100644 leveldb/db/autocompact_test.cc create mode 100644 leveldb/db/builder.cc create mode 100644 leveldb/db/builder.h create mode 100644 leveldb/db/c.cc create mode 100644 leveldb/db/c_test.c create mode 100644 leveldb/db/corruption_test.cc create mode 100644 leveldb/db/db_impl.cc create mode 100644 leveldb/db/db_impl.h create mode 100644 leveldb/db/db_iter.cc create mode 100644 leveldb/db/db_iter.h create mode 100644 leveldb/db/db_test.cc create mode 100644 leveldb/db/dbformat.cc create mode 100644 leveldb/db/dbformat.h create mode 100644 leveldb/db/dbformat_test.cc create mode 100644 leveldb/db/dumpfile.cc create mode 100644 leveldb/db/fault_injection_test.cc create mode 100644 leveldb/db/filename.cc create mode 100644 leveldb/db/filename.h create mode 100644 leveldb/db/filename_test.cc create mode 100644 leveldb/db/leveldbutil.cc create mode 100644 leveldb/db/log_format.h create mode 100644 leveldb/db/log_reader.cc create mode 100644 leveldb/db/log_reader.h create mode 100644 leveldb/db/log_test.cc create mode 100644 leveldb/db/log_writer.cc create mode 100644 leveldb/db/log_writer.h create mode 100644 leveldb/db/memtable.cc create mode 100644 leveldb/db/memtable.h create mode 100644 leveldb/db/recovery_test.cc create mode 100644 leveldb/db/repair.cc create mode 100644 leveldb/db/skiplist.h create mode 100644 leveldb/db/skiplist_test.cc create mode 100644 leveldb/db/snapshot.h create mode 100644 leveldb/db/table_cache.cc create mode 100644 leveldb/db/table_cache.h create mode 100644 leveldb/db/version_edit.cc create mode 100644 leveldb/db/version_edit.h create mode 100644 leveldb/db/version_edit_test.cc create mode 100644 leveldb/db/version_set.cc create mode 100644 leveldb/db/version_set.h create mode 100644 leveldb/db/version_set_test.cc create mode 100644 leveldb/db/write_batch.cc create mode 100644 leveldb/db/write_batch_internal.h create mode 100644 leveldb/db/write_batch_test.cc create mode 100644 leveldb/doc/benchmark.html create mode 100644 leveldb/doc/imgs/lsmtree.png create mode 100644 leveldb/doc/impl.md create mode 100644 leveldb/doc/index.md create mode 100644 leveldb/doc/log_format.md create mode 100644 leveldb/doc/project1-LSMTree.md create mode 100644 leveldb/doc/table_format.md create mode 100644 leveldb/helpers/memenv/memenv.cc create mode 100644 leveldb/helpers/memenv/memenv.h create mode 100644 leveldb/helpers/memenv/memenv_test.cc create mode 100644 leveldb/include/leveldb/c.h create mode 100644 leveldb/include/leveldb/cache.h create mode 100644 leveldb/include/leveldb/comparator.h create mode 100644 leveldb/include/leveldb/db.h create mode 100644 leveldb/include/leveldb/dumpfile.h create mode 100644 leveldb/include/leveldb/env.h create mode 100644 leveldb/include/leveldb/export.h create mode 100644 leveldb/include/leveldb/filter_policy.h create mode 100644 leveldb/include/leveldb/iterator.h create mode 100644 leveldb/include/leveldb/options.h create mode 100644 leveldb/include/leveldb/slice.h create mode 100644 leveldb/include/leveldb/status.h create mode 100644 leveldb/include/leveldb/table.h create mode 100644 leveldb/include/leveldb/table_builder.h create mode 100644 leveldb/include/leveldb/write_batch.h create mode 100644 leveldb/issues/issue178_test.cc create mode 100644 leveldb/issues/issue200_test.cc create mode 100644 leveldb/issues/issue320_test.cc create mode 100644 leveldb/port/README.md create mode 100644 leveldb/port/port.h create mode 100644 leveldb/port/port_config.h.in create mode 100644 leveldb/port/port_example.h create mode 100644 leveldb/port/port_stdcxx.h create mode 100644 leveldb/port/thread_annotations.h create mode 100644 leveldb/table/block.cc create mode 100644 leveldb/table/block.h create mode 100644 leveldb/table/block_builder.cc create mode 100644 leveldb/table/block_builder.h create mode 100644 leveldb/table/filter_block.cc create mode 100644 leveldb/table/filter_block.h create mode 100644 leveldb/table/filter_block_test.cc create mode 100644 leveldb/table/format.cc create mode 100644 leveldb/table/format.h create mode 100644 leveldb/table/iterator.cc create mode 100644 leveldb/table/iterator_wrapper.h create mode 100644 leveldb/table/merger.cc create mode 100644 leveldb/table/merger.h create mode 100644 leveldb/table/table.cc create mode 100644 leveldb/table/table_builder.cc create mode 100644 leveldb/table/table_test.cc create mode 100644 leveldb/table/two_level_iterator.cc create mode 100644 leveldb/table/two_level_iterator.h create mode 100644 leveldb/util/arena.cc create mode 100644 leveldb/util/arena.h create mode 100644 leveldb/util/arena_test.cc create mode 100644 leveldb/util/bloom.cc create mode 100644 leveldb/util/bloom_test.cc create mode 100644 leveldb/util/cache.cc create mode 100644 leveldb/util/cache_test.cc create mode 100644 leveldb/util/coding.cc create mode 100644 leveldb/util/coding.h create mode 100644 leveldb/util/coding_test.cc create mode 100644 leveldb/util/comparator.cc create mode 100644 leveldb/util/crc32c.cc create mode 100644 leveldb/util/crc32c.h create mode 100644 leveldb/util/crc32c_test.cc create mode 100644 leveldb/util/env.cc create mode 100644 leveldb/util/env_posix.cc create mode 100644 leveldb/util/env_posix_test.cc create mode 100644 leveldb/util/env_posix_test_helper.h create mode 100644 leveldb/util/env_test.cc create mode 100644 leveldb/util/env_windows.cc create mode 100644 leveldb/util/env_windows_test.cc create mode 100644 leveldb/util/env_windows_test_helper.h create mode 100644 leveldb/util/filter_policy.cc create mode 100644 leveldb/util/hash.cc create mode 100644 leveldb/util/hash.h create mode 100644 leveldb/util/hash_test.cc create mode 100644 leveldb/util/histogram.cc create mode 100644 leveldb/util/histogram.h create mode 100644 leveldb/util/logging.cc create mode 100644 leveldb/util/logging.h create mode 100644 leveldb/util/logging_test.cc create mode 100644 leveldb/util/mutexlock.h create mode 100644 leveldb/util/no_destructor.h create mode 100644 leveldb/util/no_destructor_test.cc create mode 100644 leveldb/util/options.cc create mode 100644 leveldb/util/posix_logger.h create mode 100644 leveldb/util/random.h create mode 100644 leveldb/util/status.cc create mode 100644 leveldb/util/status_test.cc create mode 100644 leveldb/util/testutil.cc create mode 100644 leveldb/util/testutil.h create mode 100644 leveldb/util/windows_logger.h diff --git a/leveldb/.clang-format b/leveldb/.clang-format new file mode 100644 index 000000000..f493f7538 --- /dev/null +++ b/leveldb/.clang-format @@ -0,0 +1,18 @@ +# Run manually to reformat a file: +# clang-format -i --style=file +# find . -iname '*.cc' -o -iname '*.h' -o -iname '*.h.in' | xargs clang-format -i --style=file +BasedOnStyle: Google +DerivePointerAlignment: false + +# Public headers are in a different location in the internal Google repository. +# Order them so that when imported to the authoritative repository they will be +# in correct alphabetical order. +IncludeCategories: + - Regex: '^(<|"(benchmarks|db|helpers)/)' + Priority: 1 + - Regex: '^"(leveldb)/' + Priority: 2 + - Regex: '^(<|"(issues|port|table|third_party|util)/)' + Priority: 3 + - Regex: '.*' + Priority: 4 diff --git a/leveldb/.github/workflows/build.yml b/leveldb/.github/workflows/build.yml new file mode 100644 index 000000000..d28902e5b --- /dev/null +++ b/leveldb/.github/workflows/build.yml @@ -0,0 +1,102 @@ +# Copyright 2021 The LevelDB Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. See the AUTHORS file for names of contributors. + +name: ci +on: [push, pull_request] + +permissions: + contents: read + +jobs: + build-and-test: + name: >- + CI + ${{ matrix.os }} + ${{ matrix.compiler }} + ${{ matrix.optimized && 'release' || 'debug' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + compiler: [clang, gcc, msvc] + os: [ubuntu-latest, macos-latest, windows-latest] + optimized: [true, false] + exclude: + # MSVC only works on Windows. + - os: ubuntu-latest + compiler: msvc + - os: macos-latest + compiler: msvc + # Not testing with GCC on macOS. + - os: macos-latest + compiler: gcc + # Only testing with MSVC on Windows. + - os: windows-latest + compiler: clang + - os: windows-latest + compiler: gcc + include: + - compiler: clang + CC: clang + CXX: clang++ + - compiler: gcc + CC: gcc + CXX: g++ + - compiler: msvc + CC: + CXX: + + env: + CMAKE_BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_TYPE: ${{ matrix.optimized && 'RelWithDebInfo' || 'Debug' }} + CC: ${{ matrix.CC }} + CXX: ${{ matrix.CXX }} + BINARY_SUFFIX: ${{ startsWith(matrix.os, 'windows') && '.exe' || '' }} + BINARY_PATH: >- + ${{ format( + startsWith(matrix.os, 'windows') && '{0}\build\{1}\' || '{0}/build/', + github.workspace, + matrix.optimized && 'RelWithDebInfo' || 'Debug') }} + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Install dependencies on Linux + if: ${{ runner.os == 'Linux' }} + # libgoogle-perftools-dev is temporarily removed from the package list + # because it is currently broken on GitHub's Ubuntu 22.04. + run: | + sudo apt-get update + sudo apt-get install libkyotocabinet-dev libsnappy-dev libsqlite3-dev + + - name: Generate build config + run: >- + cmake -S "${{ github.workspace }}" -B "${{ env.CMAKE_BUILD_DIR }}" + -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} + -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/install_test/ + + - name: Build + run: >- + cmake --build "${{ env.CMAKE_BUILD_DIR }}" + --config "${{ env.CMAKE_BUILD_TYPE }}" + + - name: Run Tests + working-directory: ${{ github.workspace }}/build + run: ctest -C "${{ env.CMAKE_BUILD_TYPE }}" --verbose + + - name: Run LevelDB Benchmarks + run: ${{ env.BINARY_PATH }}db_bench${{ env.BINARY_SUFFIX }} + + - name: Run SQLite Benchmarks + if: ${{ runner.os != 'Windows' }} + run: ${{ env.BINARY_PATH }}db_bench_sqlite3${{ env.BINARY_SUFFIX }} + + - name: Run Kyoto Cabinet Benchmarks + if: ${{ runner.os == 'Linux' && matrix.compiler == 'clang' }} + run: ${{ env.BINARY_PATH }}db_bench_tree_db${{ env.BINARY_SUFFIX }} + + - name: Test CMake installation + run: cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target install diff --git a/leveldb/.gitignore b/leveldb/.gitignore new file mode 100644 index 000000000..c4b242534 --- /dev/null +++ b/leveldb/.gitignore @@ -0,0 +1,8 @@ +# Editors. +*.sw* +.vscode +.DS_Store + +# Build directory. +build/ +out/ diff --git a/leveldb/.gitmodules b/leveldb/.gitmodules new file mode 100644 index 000000000..6e6d3f0d7 --- /dev/null +++ b/leveldb/.gitmodules @@ -0,0 +1,6 @@ +[submodule "third_party/googletest"] + path = third_party/googletest + url = https://github.com/google/googletest.git +[submodule "third_party/benchmark"] + path = third_party/benchmark + url = https://github.com/google/benchmark diff --git a/leveldb/AUTHORS b/leveldb/AUTHORS new file mode 100644 index 000000000..2439d7a45 --- /dev/null +++ b/leveldb/AUTHORS @@ -0,0 +1,12 @@ +# Names should be added to this file like so: +# Name or Organization + +Google Inc. + +# Initial version authors: +Jeffrey Dean +Sanjay Ghemawat + +# Partial list of contributors: +Kevin Regan +Johan Bilien diff --git a/leveldb/CMakeLists.txt b/leveldb/CMakeLists.txt new file mode 100644 index 000000000..fda9e01bb --- /dev/null +++ b/leveldb/CMakeLists.txt @@ -0,0 +1,519 @@ +# Copyright 2017 The LevelDB Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. See the AUTHORS file for names of contributors. + +cmake_minimum_required(VERSION 3.9) +# Keep the version below in sync with the one in db.h +project(leveldb VERSION 1.23.0 LANGUAGES C CXX) + +# C standard can be overridden when this is used as a sub-project. +if(NOT CMAKE_C_STANDARD) + # This project can use C11, but will gracefully decay down to C89. + set(CMAKE_C_STANDARD 11) + set(CMAKE_C_STANDARD_REQUIRED OFF) + set(CMAKE_C_EXTENSIONS OFF) +endif(NOT CMAKE_C_STANDARD) + +# C++ standard can be overridden when this is used as a sub-project. +if(NOT CMAKE_CXX_STANDARD) + # This project requires C++11. + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) +endif(NOT CMAKE_CXX_STANDARD) + +if (WIN32) + set(LEVELDB_PLATFORM_NAME LEVELDB_PLATFORM_WINDOWS) + # TODO(cmumford): Make UNICODE configurable for Windows. + add_definitions(-D_UNICODE -DUNICODE) +else (WIN32) + set(LEVELDB_PLATFORM_NAME LEVELDB_PLATFORM_POSIX) +endif (WIN32) + +option(LEVELDB_BUILD_TESTS "Build LevelDB's unit tests" ON) +option(LEVELDB_BUILD_BENCHMARKS "Build LevelDB's benchmarks" ON) +option(LEVELDB_INSTALL "Install LevelDB's header and library" ON) + +include(CheckIncludeFile) +check_include_file("unistd.h" HAVE_UNISTD_H) + +include(CheckLibraryExists) +check_library_exists(crc32c crc32c_value "" HAVE_CRC32C) +check_library_exists(snappy snappy_compress "" HAVE_SNAPPY) +check_library_exists(zstd zstd_compress "" HAVE_ZSTD) +check_library_exists(tcmalloc malloc "" HAVE_TCMALLOC) + +include(CheckCXXSymbolExists) +# Using check_cxx_symbol_exists() instead of check_c_symbol_exists() because +# we're including the header from C++, and feature detection should use the same +# compiler language that the project will use later. Principles aside, some +# versions of do not expose fdatasync() in in standard C mode +# (-std=c11), but do expose the function in standard C++ mode (-std=c++11). +check_cxx_symbol_exists(fdatasync "unistd.h" HAVE_FDATASYNC) +check_cxx_symbol_exists(F_FULLFSYNC "fcntl.h" HAVE_FULLFSYNC) +check_cxx_symbol_exists(O_CLOEXEC "fcntl.h" HAVE_O_CLOEXEC) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + # Disable C++ exceptions. + string(REGEX REPLACE "/EH[a-z]+" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-") + add_definitions(-D_HAS_EXCEPTIONS=0) + + # Disable RTTI. + string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-") +else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + # Enable strict prototype warnings for C code in clang and gcc. + if(NOT CMAKE_C_FLAGS MATCHES "-Wstrict-prototypes") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes") + endif(NOT CMAKE_C_FLAGS MATCHES "-Wstrict-prototypes") + + # Disable C++ exceptions. + string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions") + + # Disable RTTI. + string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") +endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + +# Test whether -Wthread-safety is available. See +# https://clang.llvm.org/docs/ThreadSafetyAnalysis.html +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag(-Wthread-safety HAVE_CLANG_THREAD_SAFETY) + +# Used by googletest. +check_cxx_compiler_flag(-Wno-missing-field-initializers + LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS) + +include(CheckCXXSourceCompiles) + +# Test whether C++17 __has_include is available. +check_cxx_source_compiles(" +#if defined(__has_include) && __has_include() +#include +#endif +int main() { std::string str; return 0; } +" HAVE_CXX17_HAS_INCLUDE) + +set(LEVELDB_PUBLIC_INCLUDE_DIR "include/leveldb") +set(LEVELDB_PORT_CONFIG_DIR "include/port") + +configure_file( + "port/port_config.h.in" + "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" +) + +include_directories( + "${PROJECT_BINARY_DIR}/include" + "." +) + +if(BUILD_SHARED_LIBS) + # Only export LEVELDB_EXPORT symbols from the shared library. + add_compile_options(-fvisibility=hidden) +endif(BUILD_SHARED_LIBS) + +# Must be included before CMAKE_INSTALL_INCLUDEDIR is used. +include(GNUInstallDirs) + +add_library(leveldb "") +target_sources(leveldb + PRIVATE + "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" + "db/builder.cc" + "db/builder.h" + "db/c.cc" + "db/db_impl.cc" + "db/db_impl.h" + "db/db_iter.cc" + "db/db_iter.h" + "db/dbformat.cc" + "db/dbformat.h" + "db/dumpfile.cc" + "db/filename.cc" + "db/filename.h" + "db/log_format.h" + "db/log_reader.cc" + "db/log_reader.h" + "db/log_writer.cc" + "db/log_writer.h" + "db/memtable.cc" + "db/memtable.h" + "db/repair.cc" + "db/skiplist.h" + "db/snapshot.h" + "db/table_cache.cc" + "db/table_cache.h" + "db/version_edit.cc" + "db/version_edit.h" + "db/version_set.cc" + "db/version_set.h" + "db/write_batch_internal.h" + "db/write_batch.cc" + "port/port_stdcxx.h" + "port/port.h" + "port/thread_annotations.h" + "table/block_builder.cc" + "table/block_builder.h" + "table/block.cc" + "table/block.h" + "table/filter_block.cc" + "table/filter_block.h" + "table/format.cc" + "table/format.h" + "table/iterator_wrapper.h" + "table/iterator.cc" + "table/merger.cc" + "table/merger.h" + "table/table_builder.cc" + "table/table.cc" + "table/two_level_iterator.cc" + "table/two_level_iterator.h" + "util/arena.cc" + "util/arena.h" + "util/bloom.cc" + "util/cache.cc" + "util/coding.cc" + "util/coding.h" + "util/comparator.cc" + "util/crc32c.cc" + "util/crc32c.h" + "util/env.cc" + "util/filter_policy.cc" + "util/hash.cc" + "util/hash.h" + "util/logging.cc" + "util/logging.h" + "util/mutexlock.h" + "util/no_destructor.h" + "util/options.cc" + "util/random.h" + "util/status.cc" + + # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install". + $<$:PUBLIC> + "${LEVELDB_PUBLIC_INCLUDE_DIR}/c.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/cache.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/comparator.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/db.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/dumpfile.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/env.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/export.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/filter_policy.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/iterator.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/options.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/slice.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/status.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/table_builder.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/table.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h" +) + +if (WIN32) + target_sources(leveldb + PRIVATE + "util/env_windows.cc" + "util/windows_logger.h" + ) +else (WIN32) + target_sources(leveldb + PRIVATE + "util/env_posix.cc" + "util/posix_logger.h" + ) +endif (WIN32) + +# MemEnv is not part of the interface and could be pulled to a separate library. +target_sources(leveldb + PRIVATE + "helpers/memenv/memenv.cc" + "helpers/memenv/memenv.h" +) + +target_include_directories(leveldb + PUBLIC + $ + $ +) + +set_target_properties(leveldb + PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR}) + +target_compile_definitions(leveldb + PRIVATE + # Used by include/export.h when building shared libraries. + LEVELDB_COMPILE_LIBRARY + # Used by port/port.h. + ${LEVELDB_PLATFORM_NAME}=1 +) +if (NOT HAVE_CXX17_HAS_INCLUDE) + target_compile_definitions(leveldb + PRIVATE + LEVELDB_HAS_PORT_CONFIG_H=1 + ) +endif(NOT HAVE_CXX17_HAS_INCLUDE) + +if(BUILD_SHARED_LIBS) + target_compile_definitions(leveldb + PUBLIC + # Used by include/export.h. + LEVELDB_SHARED_LIBRARY + ) +endif(BUILD_SHARED_LIBS) + +if(HAVE_CLANG_THREAD_SAFETY) + target_compile_options(leveldb + PUBLIC + -Werror -Wthread-safety) +endif(HAVE_CLANG_THREAD_SAFETY) + +if(HAVE_CRC32C) + target_link_libraries(leveldb crc32c) +endif(HAVE_CRC32C) +if(HAVE_SNAPPY) + target_link_libraries(leveldb snappy) +endif(HAVE_SNAPPY) +if(HAVE_ZSTD) + target_link_libraries(leveldb zstd) +endif(HAVE_ZSTD) +if(HAVE_TCMALLOC) + target_link_libraries(leveldb tcmalloc) +endif(HAVE_TCMALLOC) + +# Needed by port_stdcxx.h +find_package(Threads REQUIRED) +target_link_libraries(leveldb Threads::Threads) + +add_executable(leveldbutil + "db/leveldbutil.cc" +) +target_link_libraries(leveldbutil leveldb) + +if(LEVELDB_BUILD_TESTS) + enable_testing() + + # Prevent overriding the parent project's compiler/linker settings on Windows. + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + set(install_gtest OFF) + set(install_gmock OFF) + set(build_gmock ON) + + # This project is tested using GoogleTest. + add_subdirectory("third_party/googletest") + + # GoogleTest triggers a missing field initializers warning. + if(LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS) + set_property(TARGET gtest + APPEND PROPERTY COMPILE_OPTIONS -Wno-missing-field-initializers) + set_property(TARGET gmock + APPEND PROPERTY COMPILE_OPTIONS -Wno-missing-field-initializers) + endif(LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS) + + add_executable(leveldb_tests "") + target_sources(leveldb_tests + PRIVATE + # "db/fault_injection_test.cc" + # "issues/issue178_test.cc" + # "issues/issue200_test.cc" + # "issues/issue320_test.cc" + "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" + # "util/env_test.cc" + "util/status_test.cc" + "util/no_destructor_test.cc" + "util/testutil.cc" + "util/testutil.h" + ) + if(NOT BUILD_SHARED_LIBS) + target_sources(leveldb_tests + PRIVATE + "db/autocompact_test.cc" + "db/corruption_test.cc" + "db/db_test.cc" + "db/dbformat_test.cc" + "db/filename_test.cc" + "db/log_test.cc" + "db/recovery_test.cc" + "db/skiplist_test.cc" + "db/version_edit_test.cc" + "db/version_set_test.cc" + "db/write_batch_test.cc" + "helpers/memenv/memenv_test.cc" + "table/filter_block_test.cc" + "table/table_test.cc" + "util/arena_test.cc" + "util/bloom_test.cc" + "util/cache_test.cc" + "util/coding_test.cc" + "util/crc32c_test.cc" + "util/hash_test.cc" + "util/logging_test.cc" + ) + endif(NOT BUILD_SHARED_LIBS) + target_link_libraries(leveldb_tests leveldb gmock gtest gtest_main) + target_compile_definitions(leveldb_tests + PRIVATE + ${LEVELDB_PLATFORM_NAME}=1 + ) + if (NOT HAVE_CXX17_HAS_INCLUDE) + target_compile_definitions(leveldb_tests + PRIVATE + LEVELDB_HAS_PORT_CONFIG_H=1 + ) + endif(NOT HAVE_CXX17_HAS_INCLUDE) + + add_test(NAME "leveldb_tests" COMMAND "leveldb_tests") + + function(leveldb_test test_file) + get_filename_component(test_target_name "${test_file}" NAME_WE) + + add_executable("${test_target_name}" "") + target_sources("${test_target_name}" + PRIVATE + "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" + "util/testutil.cc" + "util/testutil.h" + + "${test_file}" + ) + target_link_libraries("${test_target_name}" leveldb gmock gtest) + target_compile_definitions("${test_target_name}" + PRIVATE + ${LEVELDB_PLATFORM_NAME}=1 + ) + if (NOT HAVE_CXX17_HAS_INCLUDE) + target_compile_definitions("${test_target_name}" + PRIVATE + LEVELDB_HAS_PORT_CONFIG_H=1 + ) + endif(NOT HAVE_CXX17_HAS_INCLUDE) + + add_test(NAME "${test_target_name}" COMMAND "${test_target_name}") + endfunction(leveldb_test) + + leveldb_test("db/c_test.c") + + if(NOT BUILD_SHARED_LIBS) + # TODO(costan): This test also uses + # "util/env_{posix|windows}_test_helper.h" + if (WIN32) + leveldb_test("util/env_windows_test.cc") + else (WIN32) + leveldb_test("util/env_posix_test.cc") + endif (WIN32) + endif(NOT BUILD_SHARED_LIBS) +endif(LEVELDB_BUILD_TESTS) + +if(LEVELDB_BUILD_BENCHMARKS) + # This project uses Google benchmark for benchmarking. + set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) + set(BENCHMARK_ENABLE_EXCEPTIONS OFF CACHE BOOL "" FORCE) + add_subdirectory("third_party/benchmark") + + function(leveldb_benchmark bench_file) + get_filename_component(bench_target_name "${bench_file}" NAME_WE) + + add_executable("${bench_target_name}" "") + target_sources("${bench_target_name}" + PRIVATE + "${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h" + "util/histogram.cc" + "util/histogram.h" + "util/testutil.cc" + "util/testutil.h" + + "${bench_file}" + ) + target_link_libraries("${bench_target_name}" leveldb gmock gtest benchmark) + target_compile_definitions("${bench_target_name}" + PRIVATE + ${LEVELDB_PLATFORM_NAME}=1 + ) + if (NOT HAVE_CXX17_HAS_INCLUDE) + target_compile_definitions("${bench_target_name}" + PRIVATE + LEVELDB_HAS_PORT_CONFIG_H=1 + ) + endif(NOT HAVE_CXX17_HAS_INCLUDE) + endfunction(leveldb_benchmark) + + if(NOT BUILD_SHARED_LIBS) + leveldb_benchmark("benchmarks/db_bench.cc") + endif(NOT BUILD_SHARED_LIBS) + + check_library_exists(sqlite3 sqlite3_open "" HAVE_SQLITE3) + if(HAVE_SQLITE3) + leveldb_benchmark("benchmarks/db_bench_sqlite3.cc") + target_link_libraries(db_bench_sqlite3 sqlite3) + endif(HAVE_SQLITE3) + + # check_library_exists is insufficient here because the library names have + # different manglings when compiled with clang or gcc, at least when installed + # with Homebrew on Mac. + set(OLD_CMAKE_REQURED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}) + list(APPEND CMAKE_REQUIRED_LIBRARIES kyotocabinet) + check_cxx_source_compiles(" +#include + +int main() { + kyotocabinet::TreeDB* db = new kyotocabinet::TreeDB(); + delete db; + return 0; +} + " HAVE_KYOTOCABINET) + set(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQURED_LIBRARIES}) + if(HAVE_KYOTOCABINET) + leveldb_benchmark("benchmarks/db_bench_tree_db.cc") + target_link_libraries(db_bench_tree_db kyotocabinet) + endif(HAVE_KYOTOCABINET) +endif(LEVELDB_BUILD_BENCHMARKS) + +if(LEVELDB_INSTALL) + install(TARGETS leveldb + EXPORT leveldbTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + install( + FILES + "${LEVELDB_PUBLIC_INCLUDE_DIR}/c.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/cache.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/comparator.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/db.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/dumpfile.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/env.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/export.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/filter_policy.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/iterator.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/options.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/slice.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/status.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/table_builder.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/table.h" + "${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/leveldb" + ) + + include(CMakePackageConfigHelpers) + configure_package_config_file( + "cmake/${PROJECT_NAME}Config.cmake.in" + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) + write_basic_package_version_file( + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake" + COMPATIBILITY SameMajorVersion + ) + install( + EXPORT leveldbTargets + NAMESPACE leveldb:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) + install( + FILES + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake" + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) +endif(LEVELDB_INSTALL) diff --git a/leveldb/CONTRIBUTING.md b/leveldb/CONTRIBUTING.md new file mode 100644 index 000000000..3cf27bb40 --- /dev/null +++ b/leveldb/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code Reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +See [the README](README.md#contributing-to-the-leveldb-project) for areas +where we are likely to accept external contributions. + +## Community Guidelines + +This project follows [Google's Open Source Community +Guidelines](https://opensource.google/conduct/). \ No newline at end of file diff --git a/leveldb/LICENSE b/leveldb/LICENSE new file mode 100644 index 000000000..8e80208cd --- /dev/null +++ b/leveldb/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/leveldb/NEWS b/leveldb/NEWS new file mode 100644 index 000000000..3fd99242d --- /dev/null +++ b/leveldb/NEWS @@ -0,0 +1,17 @@ +Release 1.2 2011-05-16 +---------------------- + +Fixes for larger databases (tested up to one billion 100-byte entries, +i.e., ~100GB). + +(1) Place hard limit on number of level-0 files. This fixes errors +of the form "too many open files". + +(2) Fixed memtable management. Before the fix, a heavy write burst +could cause unbounded memory usage. + +A fix for a logging bug where the reader would incorrectly complain +about corruption. + +Allow public access to WriteBatch contents so that users can easily +wrap a DB. diff --git a/leveldb/README.md b/leveldb/README.md new file mode 100644 index 000000000..fde440f1b --- /dev/null +++ b/leveldb/README.md @@ -0,0 +1,255 @@ + +# GET experiment for talent plan + + + + + + + +LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values. + +> **This repository is receiving very limited maintenance. We will only review the following types of changes.** +> +> * Fixes for critical bugs, such as data loss or memory corruption +> * Changes absolutely needed by internally supported leveldb clients. These typically fix breakage introduced by a language/standard library/OS update + +[![ci](https://github.com/google/leveldb/actions/workflows/build.yml/badge.svg)](https://github.com/google/leveldb/actions/workflows/build.yml) + +Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) + +# Features + + * Keys and values are arbitrary byte arrays. + * Data is stored sorted by key. + * Callers can provide a custom comparison function to override the sort order. + * The basic operations are `Put(key,value)`, `Get(key)`, `Delete(key)`. + * Multiple changes can be made in one atomic batch. + * Users can create a transient snapshot to get a consistent view of data. + * Forward and backward iteration is supported over the data. + * Data is automatically compressed using the [Snappy compression library](https://google.github.io/snappy/), but [Zstd compression](https://facebook.github.io/zstd/) is also supported. + * External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions. + +# Documentation + + [LevelDB library documentation](https://github.com/google/leveldb/blob/main/doc/index.md) is online and bundled with the source code. + +# Limitations + + * This is not a SQL database. It does not have a relational data model, it does not support SQL queries, and it has no support for indexes. + * Only a single process (possibly multi-threaded) can access a particular database at a time. + * There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library. + +# Getting the Source + +```bash +git clone --recurse-submodules https://github.com/google/leveldb.git +``` + +# Building + +This project supports [CMake](https://cmake.org/) out of the box. + +### Build for POSIX + +Quick start: + +```bash +mkdir -p build && cd build +cmake -DCMAKE_BUILD_TYPE=Release .. && cmake --build . +``` + +### Building for Windows + +First generate the Visual Studio 2017 project/solution files: + +```cmd +mkdir build +cd build +cmake -G "Visual Studio 15" .. +``` +The default default will build for x86. For 64-bit run: + +```cmd +cmake -G "Visual Studio 15 Win64" .. +``` + +To compile the Windows solution from the command-line: + +```cmd +devenv /build Debug leveldb.sln +``` + +or open leveldb.sln in Visual Studio and build from within. + +Please see the CMake documentation and `CMakeLists.txt` for more advanced usage. + +# Contributing to the leveldb Project + +> **This repository is receiving very limited maintenance. We will only review the following types of changes.** +> +> * Bug fixes +> * Changes absolutely needed by internally supported leveldb clients. These typically fix breakage introduced by a language/standard library/OS update + +The leveldb project welcomes contributions. leveldb's primary goal is to be +a reliable and fast key/value store. Changes that are in line with the +features/limitations outlined above, and meet the requirements below, +will be considered. + +Contribution requirements: + +1. **Tested platforms only**. We _generally_ will only accept changes for + platforms that are compiled and tested. This means POSIX (for Linux and + macOS) or Windows. Very small changes will sometimes be accepted, but + consider that more of an exception than the rule. + +2. **Stable API**. We strive very hard to maintain a stable API. Changes that + require changes for projects using leveldb _might_ be rejected without + sufficient benefit to the project. + +3. **Tests**: All changes must be accompanied by a new (or changed) test, or + a sufficient explanation as to why a new (or changed) test is not required. + +4. **Consistent Style**: This project conforms to the + [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html). + To ensure your changes are properly formatted please run: + + ``` + clang-format -i --style=file + ``` + +We are unlikely to accept contributions to the build configuration files, such +as `CMakeLists.txt`. We are focused on maintaining a build configuration that +allows us to test that the project works in a few supported configurations +inside Google. We are not currently interested in supporting other requirements, +such as different operating systems, compilers, or build systems. + +## Submitting a Pull Request + +Before any pull request will be accepted the author must first sign a +Contributor License Agreement (CLA) at https://cla.developers.google.com/. + +In order to keep the commit timeline linear +[squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits) +your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase) +on google/leveldb/main. This keeps the commit timeline linear and more easily sync'ed +with the internal repository at Google. More information at GitHub's +[About Git rebase](https://help.github.com/articles/about-git-rebase/) page. + +# Performance + +Here is a performance report (with explanations) from the run of the +included db_bench program. The results are somewhat noisy, but should +be enough to get a ballpark performance estimate. + +## Setup + +We use a database with a million entries. Each entry has a 16 byte +key, and a 100 byte value. Values used by the benchmark compress to +about half their original size. + + LevelDB: version 1.1 + Date: Sun May 1 12:11:26 2011 + CPU: 4 x Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz + CPUCache: 4096 KB + Keys: 16 bytes each + Values: 100 bytes each (50 bytes after compression) + Entries: 1000000 + Raw Size: 110.6 MB (estimated) + File Size: 62.9 MB (estimated) + +## Write performance + +The "fill" benchmarks create a brand new database, in either +sequential, or random order. The "fillsync" benchmark flushes data +from the operating system to the disk after every operation; the other +write operations leave the data sitting in the operating system buffer +cache for a while. The "overwrite" benchmark does random writes that +update existing keys in the database. + + fillseq : 1.765 micros/op; 62.7 MB/s + fillsync : 268.409 micros/op; 0.4 MB/s (10000 ops) + fillrandom : 2.460 micros/op; 45.0 MB/s + overwrite : 2.380 micros/op; 46.5 MB/s + +Each "op" above corresponds to a write of a single key/value pair. +I.e., a random write benchmark goes at approximately 400,000 writes per second. + +Each "fillsync" operation costs much less (0.3 millisecond) +than a disk seek (typically 10 milliseconds). We suspect that this is +because the hard disk itself is buffering the update in its memory and +responding before the data has been written to the platter. This may +or may not be safe based on whether or not the hard disk has enough +power to save its memory in the event of a power failure. + +## Read performance + +We list the performance of reading sequentially in both the forward +and reverse direction, and also the performance of a random lookup. +Note that the database created by the benchmark is quite small. +Therefore the report characterizes the performance of leveldb when the +working set fits in memory. The cost of reading a piece of data that +is not present in the operating system buffer cache will be dominated +by the one or two disk seeks needed to fetch the data from disk. +Write performance will be mostly unaffected by whether or not the +working set fits in memory. + + readrandom : 16.677 micros/op; (approximately 60,000 reads per second) + readseq : 0.476 micros/op; 232.3 MB/s + readreverse : 0.724 micros/op; 152.9 MB/s + +LevelDB compacts its underlying storage data in the background to +improve read performance. The results listed above were done +immediately after a lot of random writes. The results after +compactions (which are usually triggered automatically) are better. + + readrandom : 11.602 micros/op; (approximately 85,000 reads per second) + readseq : 0.423 micros/op; 261.8 MB/s + readreverse : 0.663 micros/op; 166.9 MB/s + +Some of the high cost of reads comes from repeated decompression of blocks +read from disk. If we supply enough cache to the leveldb so it can hold the +uncompressed blocks in memory, the read performance improves again: + + readrandom : 9.775 micros/op; (approximately 100,000 reads per second before compaction) + readrandom : 5.215 micros/op; (approximately 190,000 reads per second after compaction) + +## Repository contents + +See [doc/index.md](doc/index.md) for more explanation. See +[doc/impl.md](doc/impl.md) for a brief overview of the implementation. + +The public interface is in include/leveldb/*.h. Callers should not include or +rely on the details of any other header files in this package. Those +internal APIs may be changed without warning. + +Guide to header files: + +* **include/leveldb/db.h**: Main interface to the DB: Start here. + +* **include/leveldb/options.h**: Control over the behavior of an entire database, +and also control over the behavior of individual reads and writes. + +* **include/leveldb/comparator.h**: Abstraction for user-specified comparison function. +If you want just bytewise comparison of keys, you can use the default +comparator, but clients can write their own comparator implementations if they +want custom ordering (e.g. to handle different character encodings, etc.). + +* **include/leveldb/iterator.h**: Interface for iterating over data. You can get +an iterator from a DB object. + +* **include/leveldb/write_batch.h**: Interface for atomically applying multiple +updates to a database. + +* **include/leveldb/slice.h**: A simple module for maintaining a pointer and a +length into some other byte array. + +* **include/leveldb/status.h**: Status is returned from many of the public interfaces +and is used to report success and various kinds of errors. + +* **include/leveldb/env.h**: +Abstraction of the OS environment. A posix implementation of this interface is +in util/env_posix.cc. + +* **include/leveldb/table.h, include/leveldb/table_builder.h**: Lower-level modules that most +clients probably won't use directly. diff --git a/leveldb/TODO b/leveldb/TODO new file mode 100644 index 000000000..e603c0713 --- /dev/null +++ b/leveldb/TODO @@ -0,0 +1,14 @@ +ss +- Stats + +db +- Maybe implement DB::BulkDeleteForRange(start_key, end_key) + that would blow away files whose ranges are entirely contained + within [start_key..end_key]? For Chrome, deletion of obsolete + object stores, etc. can be done in the background anyway, so + probably not that important. +- There have been requests for MultiGet. + +After a range is completely deleted, what gets rid of the +corresponding files if we do no future changes to that range. Make +the conditions for triggering compactions fire in more situations? diff --git a/leveldb/benchmarks/db_bench.cc b/leveldb/benchmarks/db_bench.cc new file mode 100644 index 000000000..12ea7d566 --- /dev/null +++ b/leveldb/benchmarks/db_bench.cc @@ -0,0 +1,1159 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include +#include +#include + +#include "leveldb/cache.h" +#include "leveldb/comparator.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/filter_policy.h" +#include "leveldb/write_batch.h" +#include "port/port.h" +#include "util/crc32c.h" +#include "util/histogram.h" +#include "util/mutexlock.h" +#include "util/random.h" +#include "util/testutil.h" + +// Comma-separated list of operations to run in the specified order +// Actual benchmarks: +// fillseq -- write N values in sequential key order in async mode +// fillrandom -- write N values in random key order in async mode +// overwrite -- overwrite N values in random key order in async mode +// fillsync -- write N/100 values in random key order in sync mode +// fill100K -- write N/1000 100K values in random order in async mode +// deleteseq -- delete N keys in sequential order +// deleterandom -- delete N keys in random order +// readseq -- read N times sequentially +// readreverse -- read N times in reverse order +// readrandom -- read N times in random order +// readmissing -- read N missing keys in random order +// readhot -- read N times in random order from 1% section of DB +// seekrandom -- N random seeks +// seekordered -- N ordered seeks +// open -- cost of opening a DB +// crc32c -- repeated crc32c of 4K of data +// Meta operations: +// compact -- Compact the entire DB +// stats -- Print DB stats +// sstables -- Print sstable info +// heapprofile -- Dump a heap profile (if supported by this port) +static const char* FLAGS_benchmarks = + "fillseq," + "fillsync," + "fillrandom," + "overwrite," + "readrandom," + "readrandom," // Extra run to allow previous compactions to quiesce + "readseq," + "readreverse," + "compact," + "readrandom," + "readseq," + "readreverse," + "fill100K," + "crc32c," + "snappycomp," + "snappyuncomp," + "zstdcomp," + "zstduncomp,"; + +// Number of key/values to place in database +static int FLAGS_num = 1000000; + +// Number of read operations to do. If negative, do FLAGS_num reads. +static int FLAGS_reads = -1; + +// Number of each scan operations. If negative, do 1000. +static int FLAGS_scanLength = -1; + +// Number of concurrent threads to run. +static int FLAGS_threads = 1; + +// Size of each value +static int FLAGS_value_size = 100; + +// Arrange to generate values that shrink to this fraction of +// their original size after compression +static double FLAGS_compression_ratio = 0.5; + +// Print histogram of operation timings +static bool FLAGS_histogram = false; + +// Count the number of string comparisons performed +static bool FLAGS_comparisons = false; + +// Number of bytes to buffer in memtable before compacting +// (initialized to default value by "main") +static int FLAGS_write_buffer_size = 0; + +// Number of bytes written to each file. +// (initialized to default value by "main") +static int FLAGS_max_file_size = 0; + +// Approximate size of user data packed per block (before compression. +// (initialized to default value by "main") +static int FLAGS_block_size = 0; + +// Number of bytes to use as a cache of uncompressed data. +// Negative means use default settings. +static int FLAGS_cache_size = -1; + +// Maximum number of files to keep open at the same time (use default if == 0) +static int FLAGS_open_files = 0; + +// Bloom filter bits per key. +// Negative means use default settings. +static int FLAGS_bloom_bits = -1; + +// Common key prefix length. +static int FLAGS_key_prefix = 0; + +// If true, do not destroy the existing database. If you set this +// flag and also specify a benchmark that wants a fresh database, that +// benchmark will fail. +static bool FLAGS_use_existing_db = false; + +// If true, reuse existing log/MANIFEST files when re-opening a database. +static bool FLAGS_reuse_logs = false; + +// If true, use compression. +static bool FLAGS_compression = true; + +// Use the db with the following name. +static const char* FLAGS_db = nullptr; + +namespace leveldb { + +namespace { +leveldb::Env* g_env = nullptr; + +class CountComparator : public Comparator { + public: + CountComparator(const Comparator* wrapped) : wrapped_(wrapped) {} + ~CountComparator() override {} + int Compare(const Slice& a, const Slice& b) const override { + count_.fetch_add(1, std::memory_order_relaxed); + return wrapped_->Compare(a, b); + } + const char* Name() const override { return wrapped_->Name(); } + void FindShortestSeparator(std::string* start, + const Slice& limit) const override { + wrapped_->FindShortestSeparator(start, limit); + } + + void FindShortSuccessor(std::string* key) const override { + return wrapped_->FindShortSuccessor(key); + } + + size_t comparisons() const { return count_.load(std::memory_order_relaxed); } + + void reset() { count_.store(0, std::memory_order_relaxed); } + + private: + mutable std::atomic count_{0}; + const Comparator* const wrapped_; +}; + +// Helper for quickly generating random data. +class RandomGenerator { + private: + std::string data_; + int pos_; + + public: + RandomGenerator() { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + // large enough to serve all typical value sizes we want to write. + Random rnd(301); + std::string piece; + while (data_.size() < 1048576) { + // Add a short fragment that is as compressible as specified + // by FLAGS_compression_ratio. + test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); + data_.append(piece); + } + pos_ = 0; + } + + Slice Generate(size_t len) { + if (pos_ + len > data_.size()) { + pos_ = 0; + assert(len < data_.size()); + } + pos_ += len; + return Slice(data_.data() + pos_ - len, len); + } +}; + +class KeyBuffer { + public: + KeyBuffer() { + assert(FLAGS_key_prefix < sizeof(buffer_)); + memset(buffer_, 'a', FLAGS_key_prefix); + } + KeyBuffer& operator=(KeyBuffer& other) = delete; + KeyBuffer(KeyBuffer& other) = delete; + + void Set(int k) { + std::snprintf(buffer_ + FLAGS_key_prefix, + sizeof(buffer_) - FLAGS_key_prefix, "%016d", k); + } + + Slice slice() const { return Slice(buffer_, FLAGS_key_prefix + 16); } + + private: + char buffer_[1024]; +}; + +#if defined(__linux) +static Slice TrimSpace(Slice s) { + size_t start = 0; + while (start < s.size() && isspace(s[start])) { + start++; + } + size_t limit = s.size(); + while (limit > start && isspace(s[limit - 1])) { + limit--; + } + return Slice(s.data() + start, limit - start); +} +#endif + +static void AppendWithSpace(std::string* str, Slice msg) { + if (msg.empty()) return; + if (!str->empty()) { + str->push_back(' '); + } + str->append(msg.data(), msg.size()); +} + +class Stats { + private: + double start_; + double finish_; + double seconds_; + int done_; + int next_report_; + int64_t bytes_; + double last_op_finish_; + Histogram hist_; + std::string message_; + + public: + Stats() { Start(); } + + void Start() { + next_report_ = 100; + hist_.Clear(); + done_ = 0; + bytes_ = 0; + seconds_ = 0; + message_.clear(); + start_ = finish_ = last_op_finish_ = g_env->NowMicros(); + } + + void Merge(const Stats& other) { + hist_.Merge(other.hist_); + done_ += other.done_; + bytes_ += other.bytes_; + seconds_ += other.seconds_; + if (other.start_ < start_) start_ = other.start_; + if (other.finish_ > finish_) finish_ = other.finish_; + + // Just keep the messages from one thread + if (message_.empty()) message_ = other.message_; + } + + void Stop() { + finish_ = g_env->NowMicros(); + seconds_ = (finish_ - start_) * 1e-6; + } + + void AddMessage(Slice msg) { AppendWithSpace(&message_, msg); } + + void FinishedSingleOp() { + if (FLAGS_histogram) { + double now = g_env->NowMicros(); + double micros = now - last_op_finish_; + hist_.Add(micros); + if (micros > 20000) { + std::fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); + std::fflush(stderr); + } + last_op_finish_ = now; + } + + done_++; + if (done_ >= next_report_) { + if (next_report_ < 1000) + next_report_ += 100; + else if (next_report_ < 5000) + next_report_ += 500; + else if (next_report_ < 10000) + next_report_ += 1000; + else if (next_report_ < 50000) + next_report_ += 5000; + else if (next_report_ < 100000) + next_report_ += 10000; + else if (next_report_ < 500000) + next_report_ += 50000; + else + next_report_ += 100000; + std::fprintf(stderr, "... finished %d ops%30s\r", done_, ""); + std::fflush(stderr); + } + } + + void AddBytes(int64_t n) { bytes_ += n; } + + void Report(const Slice& name) { + // Pretend at least one op was done in case we are running a benchmark + // that does not call FinishedSingleOp(). + if (done_ < 1) done_ = 1; + + std::string extra; + if (bytes_ > 0) { + // Rate is computed on actual elapsed time, not the sum of per-thread + // elapsed times. + double elapsed = (finish_ - start_) * 1e-6; + char rate[100]; + std::snprintf(rate, sizeof(rate), "%6.1f MB/s", + (bytes_ / 1048576.0) / elapsed); + extra = rate; + } + AppendWithSpace(&extra, message_); + + std::fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", + name.ToString().c_str(), seconds_ * 1e6 / done_, + (extra.empty() ? "" : " "), extra.c_str()); + if (FLAGS_histogram) { + std::fprintf(stdout, "Microseconds per op:\n%s\n", + hist_.ToString().c_str()); + } + std::fflush(stdout); + } +}; + +// State shared by all concurrent executions of the same benchmark. +struct SharedState { + port::Mutex mu; + port::CondVar cv GUARDED_BY(mu); + int total GUARDED_BY(mu); + + // Each thread goes through the following states: + // (1) initializing + // (2) waiting for others to be initialized + // (3) running + // (4) done + + int num_initialized GUARDED_BY(mu); + int num_done GUARDED_BY(mu); + bool start GUARDED_BY(mu); + + SharedState(int total) + : cv(&mu), total(total), num_initialized(0), num_done(0), start(false) {} +}; + +// Per-thread state for concurrent executions of the same benchmark. +struct ThreadState { + int tid; // 0..n-1 when running in n threads + Random rand; // Has different seeds for different threads + Stats stats; + SharedState* shared; + + ThreadState(int index, int seed) : tid(index), rand(seed), shared(nullptr) {} +}; + +void Compress( + ThreadState* thread, std::string name, + std::function compress_func) { + RandomGenerator gen; + Slice input = gen.Generate(Options().block_size); + int64_t bytes = 0; + int64_t produced = 0; + bool ok = true; + std::string compressed; + while (ok && bytes < 1024 * 1048576) { // Compress 1G + ok = compress_func(input.data(), input.size(), &compressed); + produced += compressed.size(); + bytes += input.size(); + thread->stats.FinishedSingleOp(); + } + + if (!ok) { + thread->stats.AddMessage("(" + name + " failure)"); + } else { + char buf[100]; + std::snprintf(buf, sizeof(buf), "(output: %.1f%%)", + (produced * 100.0) / bytes); + thread->stats.AddMessage(buf); + thread->stats.AddBytes(bytes); + } +} + +void Uncompress( + ThreadState* thread, std::string name, + std::function compress_func, + std::function uncompress_func) { + RandomGenerator gen; + Slice input = gen.Generate(Options().block_size); + std::string compressed; + bool ok = compress_func(input.data(), input.size(), &compressed); + int64_t bytes = 0; + char* uncompressed = new char[input.size()]; + while (ok && bytes < 1024 * 1048576) { // Compress 1G + ok = uncompress_func(compressed.data(), compressed.size(), uncompressed); + bytes += input.size(); + thread->stats.FinishedSingleOp(); + } + delete[] uncompressed; + + if (!ok) { + thread->stats.AddMessage("(" + name + " failure)"); + } else { + thread->stats.AddBytes(bytes); + } +} + +} // namespace + +class Benchmark { + private: + Cache* cache_; + const FilterPolicy* filter_policy_; + DB* db_; + int num_; + int value_size_; + int entries_per_batch_; + WriteOptions write_options_; + int reads_; + int scanLength_; //Number of each scan operations + int heap_counter_; + CountComparator count_comparator_; + int total_thread_count_; + + void PrintHeader() { + const int kKeySize = 16 + FLAGS_key_prefix; + PrintEnvironment(); + std::fprintf(stdout, "Keys: %d bytes each\n", kKeySize); + std::fprintf( + stdout, "Values: %d bytes each (%d bytes after compression)\n", + FLAGS_value_size, + static_cast(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); + std::fprintf(stdout, "Entries: %d\n", num_); + std::fprintf(stdout, "RawSize: %.1f MB (estimated)\n", + ((static_cast(kKeySize + FLAGS_value_size) * num_) / + 1048576.0)); + std::fprintf( + stdout, "FileSize: %.1f MB (estimated)\n", + (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) / + 1048576.0)); + PrintWarnings(); + std::fprintf(stdout, "------------------------------------------------\n"); + } + + void PrintWarnings() { +#if defined(__GNUC__) && !defined(__OPTIMIZE__) + std::fprintf( + stdout, + "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"); +#endif +#ifndef NDEBUG + std::fprintf( + stdout, + "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); +#endif + + // See if snappy is working by attempting to compress a compressible string + const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"; + std::string compressed; + if (!port::Snappy_Compress(text, sizeof(text), &compressed)) { + std::fprintf(stdout, "WARNING: Snappy compression is not enabled\n"); + } else if (compressed.size() >= sizeof(text)) { + std::fprintf(stdout, "WARNING: Snappy compression is not effective\n"); + } + } + + void PrintEnvironment() { + std::fprintf(stderr, "LevelDB: version %d.%d\n", kMajorVersion, + kMinorVersion); + +#if defined(__linux) + time_t now = time(nullptr); + std::fprintf(stderr, "Date: %s", + ctime(&now)); // ctime() adds newline + + FILE* cpuinfo = std::fopen("/proc/cpuinfo", "r"); + if (cpuinfo != nullptr) { + char line[1000]; + int num_cpus = 0; + std::string cpu_type; + std::string cache_size; + while (fgets(line, sizeof(line), cpuinfo) != nullptr) { + const char* sep = strchr(line, ':'); + if (sep == nullptr) { + continue; + } + Slice key = TrimSpace(Slice(line, sep - 1 - line)); + Slice val = TrimSpace(Slice(sep + 1)); + if (key == "model name") { + ++num_cpus; + cpu_type = val.ToString(); + } else if (key == "cache size") { + cache_size = val.ToString(); + } + } + std::fclose(cpuinfo); + std::fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str()); + std::fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); + } +#endif + } + + public: + Benchmark() + : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : nullptr), + filter_policy_(FLAGS_bloom_bits >= 0 + ? NewBloomFilterPolicy(FLAGS_bloom_bits) + : nullptr), + db_(nullptr), + num_(FLAGS_num), + value_size_(FLAGS_value_size), + entries_per_batch_(1), + reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), + scanLength_(FLAGS_scanLength < 0 ? 1000 : FLAGS_scanLength), + heap_counter_(0), + count_comparator_(BytewiseComparator()), + total_thread_count_(0) { + std::vector files; + g_env->GetChildren(FLAGS_db, &files); + for (size_t i = 0; i < files.size(); i++) { + if (Slice(files[i]).starts_with("heap-")) { + g_env->RemoveFile(std::string(FLAGS_db) + "/" + files[i]); + } + } + if (!FLAGS_use_existing_db) { + DestroyDB(FLAGS_db, Options()); + } + } + + ~Benchmark() { + delete db_; + delete cache_; + delete filter_policy_; + } + + void Run() { + PrintHeader(); + Open(); + + const char* benchmarks = FLAGS_benchmarks; + while (benchmarks != nullptr) { + const char* sep = strchr(benchmarks, ','); + Slice name; + if (sep == nullptr) { + name = benchmarks; + benchmarks = nullptr; + } else { + name = Slice(benchmarks, sep - benchmarks); + benchmarks = sep + 1; + } + + // Reset parameters that may be overridden below + num_ = FLAGS_num; + reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); + value_size_ = FLAGS_value_size; + entries_per_batch_ = 1; + write_options_ = WriteOptions(); + + void (Benchmark::*method)(ThreadState*) = nullptr; + bool fresh_db = false; + int num_threads = FLAGS_threads; + + if (name == Slice("open")) { + method = &Benchmark::OpenBench; + num_ /= 10000; + if (num_ < 1) num_ = 1; + } else if (name == Slice("fillseq")) { + fresh_db = true; + method = &Benchmark::WriteSeq; + } else if (name == Slice("fillbatch")) { + fresh_db = true; + entries_per_batch_ = 1000; + method = &Benchmark::WriteSeq; + } else if (name == Slice("fillrandom")) { + fresh_db = true; + method = &Benchmark::WriteRandom; + } else if (name == Slice("overwrite")) { + fresh_db = false; + method = &Benchmark::WriteRandom; + } else if (name == Slice("fillsync")) { + fresh_db = true; + num_ /= 1000; + write_options_.sync = true; + method = &Benchmark::WriteRandom; + } else if (name == Slice("fill100K")) { + fresh_db = true; + num_ /= 1000; + value_size_ = 100 * 1000; + method = &Benchmark::WriteRandom; + } else if (name == Slice("readseq")) { + method = &Benchmark::ReadSequential; + } else if (name == Slice("readreverse")) { + method = &Benchmark::ReadReverse; + } else if (name == Slice("readrandom")) { + method = &Benchmark::ReadRandom; + } else if (name == Slice("readmissing")) { + method = &Benchmark::ReadMissing; + } else if (name == Slice("scan")) { + method = &Benchmark::Scan; + //std::cout << "scan length = " << scanLength_ << std::endl; + } else if (name == Slice("seekrandom")) { + method = &Benchmark::SeekRandom; + } else if (name == Slice("seekordered")) { + method = &Benchmark::SeekOrdered; + } else if (name == Slice("readhot")) { + method = &Benchmark::ReadHot; + } else if (name == Slice("readrandomsmall")) { + reads_ /= 1000; + method = &Benchmark::ReadRandom; + } else if (name == Slice("deleteseq")) { + method = &Benchmark::DeleteSeq; + } else if (name == Slice("deleterandom")) { + method = &Benchmark::DeleteRandom; + } else if (name == Slice("readwhilewriting")) { + num_threads++; // Add extra thread for writing + method = &Benchmark::ReadWhileWriting; + } else if (name == Slice("compact")) { + method = &Benchmark::Compact; + } else if (name == Slice("crc32c")) { + method = &Benchmark::Crc32c; + } else if (name == Slice("snappycomp")) { + method = &Benchmark::SnappyCompress; + } else if (name == Slice("snappyuncomp")) { + method = &Benchmark::SnappyUncompress; + } else if (name == Slice("zstdcomp")) { + method = &Benchmark::ZstdCompress; + } else if (name == Slice("zstduncomp")) { + method = &Benchmark::ZstdUncompress; + } else if (name == Slice("heapprofile")) { + HeapProfile(); + } else if (name == Slice("stats")) { + PrintStats("leveldb.stats"); + } else if (name == Slice("sstables")) { + PrintStats("leveldb.sstables"); + } else { + if (!name.empty()) { // No error message for empty name + std::fprintf(stderr, "unknown benchmark '%s'\n", + name.ToString().c_str()); + } + } + + if (fresh_db) { + if (FLAGS_use_existing_db) { + std::fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n", + name.ToString().c_str()); + method = nullptr; + } else { + delete db_; + db_ = nullptr; + DestroyDB(FLAGS_db, Options()); + Open(); + } + } + + if (method != nullptr) { + RunBenchmark(num_threads, name, method); + } + } + } + + private: + struct ThreadArg { + Benchmark* bm; + SharedState* shared; + ThreadState* thread; + void (Benchmark::*method)(ThreadState*); + }; + + static void ThreadBody(void* v) { + ThreadArg* arg = reinterpret_cast(v); + SharedState* shared = arg->shared; + ThreadState* thread = arg->thread; + { + MutexLock l(&shared->mu); + shared->num_initialized++; + if (shared->num_initialized >= shared->total) { + shared->cv.SignalAll(); + } + while (!shared->start) { + shared->cv.Wait(); + } + } + + thread->stats.Start(); + (arg->bm->*(arg->method))(thread); + thread->stats.Stop(); + + { + MutexLock l(&shared->mu); + shared->num_done++; + if (shared->num_done >= shared->total) { + shared->cv.SignalAll(); + } + } + } + + void RunBenchmark(int n, Slice name, + void (Benchmark::*method)(ThreadState*)) { + SharedState shared(n); + + ThreadArg* arg = new ThreadArg[n]; + for (int i = 0; i < n; i++) { + arg[i].bm = this; + arg[i].method = method; + arg[i].shared = &shared; + ++total_thread_count_; + // Seed the thread's random state deterministically based upon thread + // creation across all benchmarks. This ensures that the seeds are unique + // but reproducible when rerunning the same set of benchmarks. + arg[i].thread = new ThreadState(i, /*seed=*/1000 + total_thread_count_); + arg[i].thread->shared = &shared; + g_env->StartThread(ThreadBody, &arg[i]); + } + + shared.mu.Lock(); + while (shared.num_initialized < n) { + shared.cv.Wait(); + } + + shared.start = true; + shared.cv.SignalAll(); + while (shared.num_done < n) { + shared.cv.Wait(); + } + shared.mu.Unlock(); + + for (int i = 1; i < n; i++) { + arg[0].thread->stats.Merge(arg[i].thread->stats); + } + arg[0].thread->stats.Report(name); + if (FLAGS_comparisons) { + fprintf(stdout, "Comparisons: %zu\n", count_comparator_.comparisons()); + count_comparator_.reset(); + fflush(stdout); + } + + for (int i = 0; i < n; i++) { + delete arg[i].thread; + } + delete[] arg; + } + + void Crc32c(ThreadState* thread) { + // Checksum about 500MB of data total + const int size = 4096; + const char* label = "(4K per op)"; + std::string data(size, 'x'); + int64_t bytes = 0; + uint32_t crc = 0; + while (bytes < 500 * 1048576) { + crc = crc32c::Value(data.data(), size); + thread->stats.FinishedSingleOp(); + bytes += size; + } + // Print so result is not dead + std::fprintf(stderr, "... crc=0x%x\r", static_cast(crc)); + + thread->stats.AddBytes(bytes); + thread->stats.AddMessage(label); + } + + void SnappyCompress(ThreadState* thread) { + Compress(thread, "snappy", &port::Snappy_Compress); + } + + void SnappyUncompress(ThreadState* thread) { + Uncompress(thread, "snappy", &port::Snappy_Compress, + &port::Snappy_Uncompress); + } + + void ZstdCompress(ThreadState* thread) { + Compress(thread, "zstd", &port::Zstd_Compress); + } + + void ZstdUncompress(ThreadState* thread) { + Uncompress(thread, "zstd", &port::Zstd_Compress, &port::Zstd_Uncompress); + } + + void Open() { + assert(db_ == nullptr); + Options options; + options.env = g_env; + options.create_if_missing = !FLAGS_use_existing_db; + options.block_cache = cache_; + options.write_buffer_size = FLAGS_write_buffer_size; + options.max_file_size = FLAGS_max_file_size; + options.block_size = FLAGS_block_size; + if (FLAGS_comparisons) { + options.comparator = &count_comparator_; + } + options.max_open_files = FLAGS_open_files; + options.filter_policy = filter_policy_; + options.reuse_logs = FLAGS_reuse_logs; + options.compression = + FLAGS_compression ? kSnappyCompression : kNoCompression; + Status s = DB::Open(options, FLAGS_db, &db_); + if (!s.ok()) { + std::fprintf(stderr, "open error: %s\n", s.ToString().c_str()); + std::exit(1); + } + } + + void OpenBench(ThreadState* thread) { + for (int i = 0; i < num_; i++) { + delete db_; + Open(); + thread->stats.FinishedSingleOp(); + } + } + + void WriteSeq(ThreadState* thread) { DoWrite(thread, true); } + + void WriteRandom(ThreadState* thread) { DoWrite(thread, false); } + + void DoWrite(ThreadState* thread, bool seq) { + if (num_ != FLAGS_num) { + char msg[100]; + std::snprintf(msg, sizeof(msg), "(%d ops)", num_); + thread->stats.AddMessage(msg); + } + + RandomGenerator gen; + WriteBatch batch; + Status s; + int64_t bytes = 0; + KeyBuffer key; + for (int i = 0; i < num_; i += entries_per_batch_) { + batch.Clear(); + for (int j = 0; j < entries_per_batch_; j++) { + const int k = seq ? i + j : thread->rand.Uniform(FLAGS_num); + key.Set(k); + batch.Put(key.slice(), gen.Generate(value_size_)); + bytes += value_size_ + key.slice().size(); + thread->stats.FinishedSingleOp(); + } + s = db_->Write(write_options_, &batch); + if (!s.ok()) { + std::fprintf(stderr, "put error: %s\n", s.ToString().c_str()); + std::exit(1); + } + } + thread->stats.AddBytes(bytes); + } + + void ReadSequential(ThreadState* thread) { + Iterator* iter = db_->NewIterator(ReadOptions()); + int i = 0; + int64_t bytes = 0; + for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) { + bytes += iter->key().size() + iter->value().size(); + thread->stats.FinishedSingleOp(); + ++i; + } + delete iter; + thread->stats.AddBytes(bytes); + } + + void ReadReverse(ThreadState* thread) { + Iterator* iter = db_->NewIterator(ReadOptions()); + int i = 0; + int64_t bytes = 0; + for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) { + bytes += iter->key().size() + iter->value().size(); + thread->stats.FinishedSingleOp(); + ++i; + } + delete iter; + thread->stats.AddBytes(bytes); + } + + void ReadRandom(ThreadState* thread) { + ReadOptions options; + std::string value; + int found = 0; + KeyBuffer key; + for (int i = 0; i < reads_; i++) { + const int k = thread->rand.Uniform(FLAGS_num); + key.Set(k); + if (db_->Get(options, key.slice(), &value).ok()) { + found++; + } + thread->stats.FinishedSingleOp(); + } + char msg[100]; + std::snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); + thread->stats.AddMessage(msg); + } + + void ReadMissing(ThreadState* thread) { + ReadOptions options; + std::string value; + KeyBuffer key; + for (int i = 0; i < reads_; i++) { + const int k = thread->rand.Uniform(FLAGS_num); + key.Set(k); + Slice s = Slice(key.slice().data(), key.slice().size() - 1); + db_->Get(options, s, &value); + thread->stats.FinishedSingleOp(); + } + } + + void ReadHot(ThreadState* thread) { + ReadOptions options; + std::string value; + const int range = (FLAGS_num + 99) / 100; + KeyBuffer key; + for (int i = 0; i < reads_; i++) { + const int k = thread->rand.Uniform(range); + key.Set(k); + db_->Get(options, key.slice(), &value); + thread->stats.FinishedSingleOp(); + } + } + + // scan interface function + void Scan(ThreadState* thread) { + ReadOptions options; + int found = 0; + KeyBuffer key; + int64_t bytes = 0; + for (int i = 0; i < reads_; i++) { + Iterator* iter = db_->NewIterator(options); + const int k = thread->rand.Uniform(FLAGS_num); + key.Set(k); + iter->Seek(key.slice()); + if (iter->Valid() && iter->key() == key.slice()) found++; + for (int j = 0; j < scanLength_ && iter->Valid(); j++, iter->Next()) { + bytes += iter->key().size() + iter->value().size(); + } + delete iter; + thread->stats.FinishedSingleOp(); + } + thread->stats.AddBytes(bytes); + char msg[100]; + snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); + thread->stats.AddMessage(msg); + } + + void SeekRandom(ThreadState* thread) { + ReadOptions options; + int found = 0; + KeyBuffer key; + for (int i = 0; i < reads_; i++) { + Iterator* iter = db_->NewIterator(options); + const int k = thread->rand.Uniform(FLAGS_num); + key.Set(k); + iter->Seek(key.slice()); + if (iter->Valid() && iter->key() == key.slice()) found++; + delete iter; + thread->stats.FinishedSingleOp(); + } + char msg[100]; + snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); + thread->stats.AddMessage(msg); + } + + void SeekOrdered(ThreadState* thread) { + ReadOptions options; + Iterator* iter = db_->NewIterator(options); + int found = 0; + int k = 0; + KeyBuffer key; + for (int i = 0; i < reads_; i++) { + k = (k + (thread->rand.Uniform(100))) % FLAGS_num; + key.Set(k); + iter->Seek(key.slice()); + if (iter->Valid() && iter->key() == key.slice()) found++; + thread->stats.FinishedSingleOp(); + } + delete iter; + char msg[100]; + std::snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); + thread->stats.AddMessage(msg); + } + + void DoDelete(ThreadState* thread, bool seq) { + RandomGenerator gen; + WriteBatch batch; + Status s; + KeyBuffer key; + for (int i = 0; i < num_; i += entries_per_batch_) { + batch.Clear(); + for (int j = 0; j < entries_per_batch_; j++) { + const int k = seq ? i + j : (thread->rand.Uniform(FLAGS_num)); + key.Set(k); + batch.Delete(key.slice()); + thread->stats.FinishedSingleOp(); + } + s = db_->Write(write_options_, &batch); + if (!s.ok()) { + std::fprintf(stderr, "del error: %s\n", s.ToString().c_str()); + std::exit(1); + } + } + } + + void DeleteSeq(ThreadState* thread) { DoDelete(thread, true); } + + void DeleteRandom(ThreadState* thread) { DoDelete(thread, false); } + + void ReadWhileWriting(ThreadState* thread) { + if (thread->tid > 0) { + ReadRandom(thread); + } else { + // Special thread that keeps writing until other threads are done. + RandomGenerator gen; + KeyBuffer key; + while (true) { + { + MutexLock l(&thread->shared->mu); + if (thread->shared->num_done + 1 >= thread->shared->num_initialized) { + // Other threads have finished + break; + } + } + + const int k = thread->rand.Uniform(FLAGS_num); + key.Set(k); + Status s = + db_->Put(write_options_, key.slice(), gen.Generate(value_size_)); + if (!s.ok()) { + std::fprintf(stderr, "put error: %s\n", s.ToString().c_str()); + std::exit(1); + } + } + + // Do not count any of the preceding work/delay in stats. + thread->stats.Start(); + } + } + + void Compact(ThreadState* thread) { db_->CompactRange(nullptr, nullptr); } + + void PrintStats(const char* key) { + std::string stats; + if (!db_->GetProperty(key, &stats)) { + stats = "(failed)"; + } + std::fprintf(stdout, "\n%s\n", stats.c_str()); + } + + static void WriteToFile(void* arg, const char* buf, int n) { + reinterpret_cast(arg)->Append(Slice(buf, n)); + } + + void HeapProfile() { + char fname[100]; + std::snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, + ++heap_counter_); + WritableFile* file; + Status s = g_env->NewWritableFile(fname, &file); + if (!s.ok()) { + std::fprintf(stderr, "%s\n", s.ToString().c_str()); + return; + } + bool ok = port::GetHeapProfile(WriteToFile, file); + delete file; + if (!ok) { + std::fprintf(stderr, "heap profiling not supported\n"); + g_env->RemoveFile(fname); + } + } +}; + +} // namespace leveldb + +int main(int argc, char** argv) { + FLAGS_write_buffer_size = leveldb::Options().write_buffer_size; + FLAGS_max_file_size = leveldb::Options().max_file_size; + FLAGS_block_size = leveldb::Options().block_size; + FLAGS_open_files = leveldb::Options().max_open_files; + std::string default_db_path; + + for (int i = 1; i < argc; i++) { + double d; + int n; + char junk; + if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { + FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); + } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { + FLAGS_compression_ratio = d; + } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_histogram = n; + } else if (sscanf(argv[i], "--comparisons=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_comparisons = n; + } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_use_existing_db = n; + } else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_reuse_logs = n; + } else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_compression = n; + } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { + FLAGS_num = n; + } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { + FLAGS_reads = n; + } else if (sscanf(argv[i], "--scan_length=%d%c", &n, &junk) == 1) { + FLAGS_scanLength = n; + } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) { + FLAGS_threads = n; + } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { + FLAGS_value_size = n; + } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) { + FLAGS_write_buffer_size = n; + } else if (sscanf(argv[i], "--max_file_size=%d%c", &n, &junk) == 1) { + FLAGS_max_file_size = n; + } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) { + FLAGS_block_size = n; + } else if (sscanf(argv[i], "--key_prefix=%d%c", &n, &junk) == 1) { + FLAGS_key_prefix = n; + } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) { + FLAGS_cache_size = n; + } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) { + FLAGS_bloom_bits = n; + } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) { + FLAGS_open_files = n; + } else if (strncmp(argv[i], "--db=", 5) == 0) { + FLAGS_db = argv[i] + 5; + } else { + std::fprintf(stderr, "Invalid flag '%s'\n", argv[i]); + std::exit(1); + } + } + + leveldb::g_env = leveldb::Env::Default(); + + // Choose a location for the test database if none given with --db= + if (FLAGS_db == nullptr) { + leveldb::g_env->GetTestDirectory(&default_db_path); + default_db_path += "/dbbench"; + FLAGS_db = default_db_path.c_str(); + } + + leveldb::Benchmark benchmark; + benchmark.Run(); + return 0; +} diff --git a/leveldb/benchmarks/db_bench_log.cc b/leveldb/benchmarks/db_bench_log.cc new file mode 100644 index 000000000..a1845bf14 --- /dev/null +++ b/leveldb/benchmarks/db_bench_log.cc @@ -0,0 +1,92 @@ +// Copyright (c) 2019 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include + +#include "gtest/gtest.h" +#include "benchmark/benchmark.h" +#include "db/version_set.h" +#include "leveldb/comparator.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/options.h" +#include "port/port.h" +#include "util/mutexlock.h" +#include "util/testutil.h" + +namespace leveldb { + +namespace { + +std::string MakeKey(unsigned int num) { + char buf[30]; + std::snprintf(buf, sizeof(buf), "%016u", num); + return std::string(buf); +} + +void BM_LogAndApply(benchmark::State& state) { + const int num_base_files = state.range(0); + + std::string dbname = testing::TempDir() + "leveldb_test_benchmark"; + DestroyDB(dbname, Options()); + + DB* db = nullptr; + Options opts; + opts.create_if_missing = true; + Status s = DB::Open(opts, dbname, &db); + ASSERT_LEVELDB_OK(s); + ASSERT_TRUE(db != nullptr); + + delete db; + db = nullptr; + + Env* env = Env::Default(); + + port::Mutex mu; + MutexLock l(&mu); + + InternalKeyComparator cmp(BytewiseComparator()); + Options options; + VersionSet vset(dbname, &options, nullptr, &cmp); + bool save_manifest; + ASSERT_LEVELDB_OK(vset.Recover(&save_manifest)); + VersionEdit vbase; + uint64_t fnum = 1; + for (int i = 0; i < num_base_files; i++) { + InternalKey start(MakeKey(2 * fnum), 1, kTypeValue); + InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion); + vbase.AddFile(2, fnum++, 1 /* file size */, start, limit); + } + ASSERT_LEVELDB_OK(vset.LogAndApply(&vbase, &mu)); + + uint64_t start_micros = env->NowMicros(); + + for (auto st : state) { + VersionEdit vedit; + vedit.RemoveFile(2, fnum); + InternalKey start(MakeKey(2 * fnum), 1, kTypeValue); + InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion); + vedit.AddFile(2, fnum++, 1 /* file size */, start, limit); + vset.LogAndApply(&vedit, &mu); + } + + uint64_t stop_micros = env->NowMicros(); + unsigned int us = stop_micros - start_micros; + char buf[16]; + std::snprintf(buf, sizeof(buf), "%d", num_base_files); + std::fprintf(stderr, + "BM_LogAndApply/%-6s %8" PRIu64 + " iters : %9u us (%7.0f us / iter)\n", + buf, state.iterations(), us, ((float)us) / state.iterations()); +} + +BENCHMARK(BM_LogAndApply)->Arg(1)->Arg(100)->Arg(10000)->Arg(100000); + +} // namespace + +} // namespace leveldb + +BENCHMARK_MAIN(); diff --git a/leveldb/benchmarks/db_bench_sqlite3.cc b/leveldb/benchmarks/db_bench_sqlite3.cc new file mode 100644 index 000000000..c9be652ad --- /dev/null +++ b/leveldb/benchmarks/db_bench_sqlite3.cc @@ -0,0 +1,726 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include +#include + +#include "util/histogram.h" +#include "util/random.h" +#include "util/testutil.h" + +// Comma-separated list of operations to run in the specified order +// Actual benchmarks: +// +// fillseq -- write N values in sequential key order in async mode +// fillseqsync -- write N/100 values in sequential key order in sync mode +// fillseqbatch -- batch write N values in sequential key order in async mode +// fillrandom -- write N values in random key order in async mode +// fillrandsync -- write N/100 values in random key order in sync mode +// fillrandbatch -- batch write N values in sequential key order in async mode +// overwrite -- overwrite N values in random key order in async mode +// fillrand100K -- write N/1000 100K values in random order in async mode +// fillseq100K -- write N/1000 100K values in sequential order in async mode +// readseq -- read N times sequentially +// readrandom -- read N times in random order +// readrand100K -- read N/1000 100K values in sequential order in async mode +static const char* FLAGS_benchmarks = + "fillseq," + "fillseqsync," + "fillseqbatch," + "fillrandom," + "fillrandsync," + "fillrandbatch," + "overwrite," + "overwritebatch," + "readrandom," + "readseq," + "fillrand100K," + "fillseq100K," + "readseq," + "readrand100K,"; + +// Number of key/values to place in database +static int FLAGS_num = 1000000; + +// Number of read operations to do. If negative, do FLAGS_num reads. +static int FLAGS_reads = -1; + +// Size of each value +static int FLAGS_value_size = 100; + +// Print histogram of operation timings +static bool FLAGS_histogram = false; + +// Arrange to generate values that shrink to this fraction of +// their original size after compression +static double FLAGS_compression_ratio = 0.5; + +// Page size. Default 1 KB. +static int FLAGS_page_size = 1024; + +// Number of pages. +// Default cache size = FLAGS_page_size * FLAGS_num_pages = 4 MB. +static int FLAGS_num_pages = 4096; + +// If true, do not destroy the existing database. If you set this +// flag and also specify a benchmark that wants a fresh database, that +// benchmark will fail. +static bool FLAGS_use_existing_db = false; + +// If true, the SQLite table has ROWIDs. +static bool FLAGS_use_rowids = false; + +// If true, we allow batch writes to occur +static bool FLAGS_transaction = true; + +// If true, we enable Write-Ahead Logging +static bool FLAGS_WAL_enabled = true; + +// Use the db with the following name. +static const char* FLAGS_db = nullptr; + +inline static void ExecErrorCheck(int status, char* err_msg) { + if (status != SQLITE_OK) { + std::fprintf(stderr, "SQL error: %s\n", err_msg); + sqlite3_free(err_msg); + std::exit(1); + } +} + +inline static void StepErrorCheck(int status) { + if (status != SQLITE_DONE) { + std::fprintf(stderr, "SQL step error: status = %d\n", status); + std::exit(1); + } +} + +inline static void ErrorCheck(int status) { + if (status != SQLITE_OK) { + std::fprintf(stderr, "sqlite3 error: status = %d\n", status); + std::exit(1); + } +} + +inline static void WalCheckpoint(sqlite3* db_) { + // Flush all writes to disk + if (FLAGS_WAL_enabled) { + sqlite3_wal_checkpoint_v2(db_, nullptr, SQLITE_CHECKPOINT_FULL, nullptr, + nullptr); + } +} + +namespace leveldb { + +// Helper for quickly generating random data. +namespace { +class RandomGenerator { + private: + std::string data_; + int pos_; + + public: + RandomGenerator() { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + // large enough to serve all typical value sizes we want to write. + Random rnd(301); + std::string piece; + while (data_.size() < 1048576) { + // Add a short fragment that is as compressible as specified + // by FLAGS_compression_ratio. + test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); + data_.append(piece); + } + pos_ = 0; + } + + Slice Generate(int len) { + if (pos_ + len > data_.size()) { + pos_ = 0; + assert(len < data_.size()); + } + pos_ += len; + return Slice(data_.data() + pos_ - len, len); + } +}; + +static Slice TrimSpace(Slice s) { + int start = 0; + while (start < s.size() && isspace(s[start])) { + start++; + } + int limit = s.size(); + while (limit > start && isspace(s[limit - 1])) { + limit--; + } + return Slice(s.data() + start, limit - start); +} + +} // namespace + +class Benchmark { + private: + sqlite3* db_; + int db_num_; + int num_; + int reads_; + double start_; + double last_op_finish_; + int64_t bytes_; + std::string message_; + Histogram hist_; + RandomGenerator gen_; + Random rand_; + + // State kept for progress messages + int done_; + int next_report_; // When to report next + + void PrintHeader() { + const int kKeySize = 16; + PrintEnvironment(); + std::fprintf(stdout, "Keys: %d bytes each\n", kKeySize); + std::fprintf(stdout, "Values: %d bytes each\n", FLAGS_value_size); + std::fprintf(stdout, "Entries: %d\n", num_); + std::fprintf(stdout, "RawSize: %.1f MB (estimated)\n", + ((static_cast(kKeySize + FLAGS_value_size) * num_) / + 1048576.0)); + PrintWarnings(); + std::fprintf(stdout, "------------------------------------------------\n"); + } + + void PrintWarnings() { +#if defined(__GNUC__) && !defined(__OPTIMIZE__) + std::fprintf( + stdout, + "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"); +#endif +#ifndef NDEBUG + std::fprintf( + stdout, + "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); +#endif + } + + void PrintEnvironment() { + std::fprintf(stderr, "SQLite: version %s\n", SQLITE_VERSION); + +#if defined(__linux) + time_t now = time(nullptr); + std::fprintf(stderr, "Date: %s", + ctime(&now)); // ctime() adds newline + + FILE* cpuinfo = std::fopen("/proc/cpuinfo", "r"); + if (cpuinfo != nullptr) { + char line[1000]; + int num_cpus = 0; + std::string cpu_type; + std::string cache_size; + while (fgets(line, sizeof(line), cpuinfo) != nullptr) { + const char* sep = strchr(line, ':'); + if (sep == nullptr) { + continue; + } + Slice key = TrimSpace(Slice(line, sep - 1 - line)); + Slice val = TrimSpace(Slice(sep + 1)); + if (key == "model name") { + ++num_cpus; + cpu_type = val.ToString(); + } else if (key == "cache size") { + cache_size = val.ToString(); + } + } + std::fclose(cpuinfo); + std::fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str()); + std::fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); + } +#endif + } + + void Start() { + start_ = Env::Default()->NowMicros() * 1e-6; + bytes_ = 0; + message_.clear(); + last_op_finish_ = start_; + hist_.Clear(); + done_ = 0; + next_report_ = 100; + } + + void FinishedSingleOp() { + if (FLAGS_histogram) { + double now = Env::Default()->NowMicros() * 1e-6; + double micros = (now - last_op_finish_) * 1e6; + hist_.Add(micros); + if (micros > 20000) { + std::fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); + std::fflush(stderr); + } + last_op_finish_ = now; + } + + done_++; + if (done_ >= next_report_) { + if (next_report_ < 1000) + next_report_ += 100; + else if (next_report_ < 5000) + next_report_ += 500; + else if (next_report_ < 10000) + next_report_ += 1000; + else if (next_report_ < 50000) + next_report_ += 5000; + else if (next_report_ < 100000) + next_report_ += 10000; + else if (next_report_ < 500000) + next_report_ += 50000; + else + next_report_ += 100000; + std::fprintf(stderr, "... finished %d ops%30s\r", done_, ""); + std::fflush(stderr); + } + } + + void Stop(const Slice& name) { + double finish = Env::Default()->NowMicros() * 1e-6; + + // Pretend at least one op was done in case we are running a benchmark + // that does not call FinishedSingleOp(). + if (done_ < 1) done_ = 1; + + if (bytes_ > 0) { + char rate[100]; + std::snprintf(rate, sizeof(rate), "%6.1f MB/s", + (bytes_ / 1048576.0) / (finish - start_)); + if (!message_.empty()) { + message_ = std::string(rate) + " " + message_; + } else { + message_ = rate; + } + } + + std::fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", + name.ToString().c_str(), (finish - start_) * 1e6 / done_, + (message_.empty() ? "" : " "), message_.c_str()); + if (FLAGS_histogram) { + std::fprintf(stdout, "Microseconds per op:\n%s\n", + hist_.ToString().c_str()); + } + std::fflush(stdout); + } + + public: + enum Order { SEQUENTIAL, RANDOM }; + enum DBState { FRESH, EXISTING }; + + Benchmark() + : db_(nullptr), + db_num_(0), + num_(FLAGS_num), + reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), + bytes_(0), + rand_(301) { + std::vector files; + std::string test_dir; + Env::Default()->GetTestDirectory(&test_dir); + Env::Default()->GetChildren(test_dir, &files); + if (!FLAGS_use_existing_db) { + for (int i = 0; i < files.size(); i++) { + if (Slice(files[i]).starts_with("dbbench_sqlite3")) { + std::string file_name(test_dir); + file_name += "/"; + file_name += files[i]; + Env::Default()->RemoveFile(file_name.c_str()); + } + } + } + } + + ~Benchmark() { + int status = sqlite3_close(db_); + ErrorCheck(status); + } + + void Run() { + PrintHeader(); + Open(); + + const char* benchmarks = FLAGS_benchmarks; + while (benchmarks != nullptr) { + const char* sep = strchr(benchmarks, ','); + Slice name; + if (sep == nullptr) { + name = benchmarks; + benchmarks = nullptr; + } else { + name = Slice(benchmarks, sep - benchmarks); + benchmarks = sep + 1; + } + + bytes_ = 0; + Start(); + + bool known = true; + bool write_sync = false; + if (name == Slice("fillseq")) { + Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillseqbatch")) { + Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1000); + WalCheckpoint(db_); + } else if (name == Slice("fillrandom")) { + Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillrandbatch")) { + Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1000); + WalCheckpoint(db_); + } else if (name == Slice("overwrite")) { + Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("overwritebatch")) { + Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1000); + WalCheckpoint(db_); + } else if (name == Slice("fillrandsync")) { + write_sync = true; + Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillseqsync")) { + write_sync = true; + Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillrand100K")) { + Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1); + WalCheckpoint(db_); + } else if (name == Slice("fillseq100K")) { + Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1); + WalCheckpoint(db_); + } else if (name == Slice("readseq")) { + ReadSequential(); + } else if (name == Slice("readrandom")) { + Read(RANDOM, 1); + } else if (name == Slice("readrand100K")) { + int n = reads_; + reads_ /= 1000; + Read(RANDOM, 1); + reads_ = n; + } else { + known = false; + if (name != Slice()) { // No error message for empty name + std::fprintf(stderr, "unknown benchmark '%s'\n", + name.ToString().c_str()); + } + } + if (known) { + Stop(name); + } + } + } + + void Open() { + assert(db_ == nullptr); + + int status; + char file_name[100]; + char* err_msg = nullptr; + db_num_++; + + // Open database + std::string tmp_dir; + Env::Default()->GetTestDirectory(&tmp_dir); + std::snprintf(file_name, sizeof(file_name), "%s/dbbench_sqlite3-%d.db", + tmp_dir.c_str(), db_num_); + status = sqlite3_open(file_name, &db_); + if (status) { + std::fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_)); + std::exit(1); + } + + // Change SQLite cache size + char cache_size[100]; + std::snprintf(cache_size, sizeof(cache_size), "PRAGMA cache_size = %d", + FLAGS_num_pages); + status = sqlite3_exec(db_, cache_size, nullptr, nullptr, &err_msg); + ExecErrorCheck(status, err_msg); + + // FLAGS_page_size is defaulted to 1024 + if (FLAGS_page_size != 1024) { + char page_size[100]; + std::snprintf(page_size, sizeof(page_size), "PRAGMA page_size = %d", + FLAGS_page_size); + status = sqlite3_exec(db_, page_size, nullptr, nullptr, &err_msg); + ExecErrorCheck(status, err_msg); + } + + // Change journal mode to WAL if WAL enabled flag is on + if (FLAGS_WAL_enabled) { + std::string WAL_stmt = "PRAGMA journal_mode = WAL"; + + // LevelDB's default cache size is a combined 4 MB + std::string WAL_checkpoint = "PRAGMA wal_autocheckpoint = 4096"; + status = sqlite3_exec(db_, WAL_stmt.c_str(), nullptr, nullptr, &err_msg); + ExecErrorCheck(status, err_msg); + status = + sqlite3_exec(db_, WAL_checkpoint.c_str(), nullptr, nullptr, &err_msg); + ExecErrorCheck(status, err_msg); + } + + // Change locking mode to exclusive and create tables/index for database + std::string locking_stmt = "PRAGMA locking_mode = EXCLUSIVE"; + std::string create_stmt = + "CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))"; + if (!FLAGS_use_rowids) create_stmt += " WITHOUT ROWID"; + std::string stmt_array[] = {locking_stmt, create_stmt}; + int stmt_array_length = sizeof(stmt_array) / sizeof(std::string); + for (int i = 0; i < stmt_array_length; i++) { + status = + sqlite3_exec(db_, stmt_array[i].c_str(), nullptr, nullptr, &err_msg); + ExecErrorCheck(status, err_msg); + } + } + + void Write(bool write_sync, Order order, DBState state, int num_entries, + int value_size, int entries_per_batch) { + // Create new database if state == FRESH + if (state == FRESH) { + if (FLAGS_use_existing_db) { + message_ = "skipping (--use_existing_db is true)"; + return; + } + sqlite3_close(db_); + db_ = nullptr; + Open(); + Start(); + } + + if (num_entries != num_) { + char msg[100]; + std::snprintf(msg, sizeof(msg), "(%d ops)", num_entries); + message_ = msg; + } + + char* err_msg = nullptr; + int status; + + sqlite3_stmt *replace_stmt, *begin_trans_stmt, *end_trans_stmt; + std::string replace_str = "REPLACE INTO test (key, value) VALUES (?, ?)"; + std::string begin_trans_str = "BEGIN TRANSACTION;"; + std::string end_trans_str = "END TRANSACTION;"; + + // Check for synchronous flag in options + std::string sync_stmt = + (write_sync) ? "PRAGMA synchronous = FULL" : "PRAGMA synchronous = OFF"; + status = sqlite3_exec(db_, sync_stmt.c_str(), nullptr, nullptr, &err_msg); + ExecErrorCheck(status, err_msg); + + // Preparing sqlite3 statements + status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1, &replace_stmt, + nullptr); + ErrorCheck(status); + status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1, + &begin_trans_stmt, nullptr); + ErrorCheck(status); + status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, &end_trans_stmt, + nullptr); + ErrorCheck(status); + + bool transaction = (entries_per_batch > 1); + for (int i = 0; i < num_entries; i += entries_per_batch) { + // Begin write transaction + if (FLAGS_transaction && transaction) { + status = sqlite3_step(begin_trans_stmt); + StepErrorCheck(status); + status = sqlite3_reset(begin_trans_stmt); + ErrorCheck(status); + } + + // Create and execute SQL statements + for (int j = 0; j < entries_per_batch; j++) { + const char* value = gen_.Generate(value_size).data(); + + // Create values for key-value pair + const int k = + (order == SEQUENTIAL) ? i + j : (rand_.Next() % num_entries); + char key[100]; + std::snprintf(key, sizeof(key), "%016d", k); + + // Bind KV values into replace_stmt + status = sqlite3_bind_blob(replace_stmt, 1, key, 16, SQLITE_STATIC); + ErrorCheck(status); + status = sqlite3_bind_blob(replace_stmt, 2, value, value_size, + SQLITE_STATIC); + ErrorCheck(status); + + // Execute replace_stmt + bytes_ += value_size + strlen(key); + status = sqlite3_step(replace_stmt); + StepErrorCheck(status); + + // Reset SQLite statement for another use + status = sqlite3_clear_bindings(replace_stmt); + ErrorCheck(status); + status = sqlite3_reset(replace_stmt); + ErrorCheck(status); + + FinishedSingleOp(); + } + + // End write transaction + if (FLAGS_transaction && transaction) { + status = sqlite3_step(end_trans_stmt); + StepErrorCheck(status); + status = sqlite3_reset(end_trans_stmt); + ErrorCheck(status); + } + } + + status = sqlite3_finalize(replace_stmt); + ErrorCheck(status); + status = sqlite3_finalize(begin_trans_stmt); + ErrorCheck(status); + status = sqlite3_finalize(end_trans_stmt); + ErrorCheck(status); + } + + void Read(Order order, int entries_per_batch) { + int status; + sqlite3_stmt *read_stmt, *begin_trans_stmt, *end_trans_stmt; + + std::string read_str = "SELECT * FROM test WHERE key = ?"; + std::string begin_trans_str = "BEGIN TRANSACTION;"; + std::string end_trans_str = "END TRANSACTION;"; + + // Preparing sqlite3 statements + status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1, + &begin_trans_stmt, nullptr); + ErrorCheck(status); + status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, &end_trans_stmt, + nullptr); + ErrorCheck(status); + status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &read_stmt, nullptr); + ErrorCheck(status); + + bool transaction = (entries_per_batch > 1); + for (int i = 0; i < reads_; i += entries_per_batch) { + // Begin read transaction + if (FLAGS_transaction && transaction) { + status = sqlite3_step(begin_trans_stmt); + StepErrorCheck(status); + status = sqlite3_reset(begin_trans_stmt); + ErrorCheck(status); + } + + // Create and execute SQL statements + for (int j = 0; j < entries_per_batch; j++) { + // Create key value + char key[100]; + int k = (order == SEQUENTIAL) ? i + j : (rand_.Next() % reads_); + std::snprintf(key, sizeof(key), "%016d", k); + + // Bind key value into read_stmt + status = sqlite3_bind_blob(read_stmt, 1, key, 16, SQLITE_STATIC); + ErrorCheck(status); + + // Execute read statement + while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) { + } + StepErrorCheck(status); + + // Reset SQLite statement for another use + status = sqlite3_clear_bindings(read_stmt); + ErrorCheck(status); + status = sqlite3_reset(read_stmt); + ErrorCheck(status); + FinishedSingleOp(); + } + + // End read transaction + if (FLAGS_transaction && transaction) { + status = sqlite3_step(end_trans_stmt); + StepErrorCheck(status); + status = sqlite3_reset(end_trans_stmt); + ErrorCheck(status); + } + } + + status = sqlite3_finalize(read_stmt); + ErrorCheck(status); + status = sqlite3_finalize(begin_trans_stmt); + ErrorCheck(status); + status = sqlite3_finalize(end_trans_stmt); + ErrorCheck(status); + } + + void ReadSequential() { + int status; + sqlite3_stmt* pStmt; + std::string read_str = "SELECT * FROM test ORDER BY key"; + + status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &pStmt, nullptr); + ErrorCheck(status); + for (int i = 0; i < reads_ && SQLITE_ROW == sqlite3_step(pStmt); i++) { + bytes_ += sqlite3_column_bytes(pStmt, 1) + sqlite3_column_bytes(pStmt, 2); + FinishedSingleOp(); + } + + status = sqlite3_finalize(pStmt); + ErrorCheck(status); + } +}; + +} // namespace leveldb + +int main(int argc, char** argv) { + std::string default_db_path; + for (int i = 1; i < argc; i++) { + double d; + int n; + char junk; + if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { + FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); + } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_histogram = n; + } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { + FLAGS_compression_ratio = d; + } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_use_existing_db = n; + } else if (sscanf(argv[i], "--use_rowids=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_use_rowids = n; + } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { + FLAGS_num = n; + } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { + FLAGS_reads = n; + } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { + FLAGS_value_size = n; + } else if (leveldb::Slice(argv[i]) == leveldb::Slice("--no_transaction")) { + FLAGS_transaction = false; + } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) { + FLAGS_page_size = n; + } else if (sscanf(argv[i], "--num_pages=%d%c", &n, &junk) == 1) { + FLAGS_num_pages = n; + } else if (sscanf(argv[i], "--WAL_enabled=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_WAL_enabled = n; + } else if (strncmp(argv[i], "--db=", 5) == 0) { + FLAGS_db = argv[i] + 5; + } else { + std::fprintf(stderr, "Invalid flag '%s'\n", argv[i]); + std::exit(1); + } + } + + // Choose a location for the test database if none given with --db= + if (FLAGS_db == nullptr) { + leveldb::Env::Default()->GetTestDirectory(&default_db_path); + default_db_path += "/dbbench"; + FLAGS_db = default_db_path.c_str(); + } + + leveldb::Benchmark benchmark; + benchmark.Run(); + return 0; +} diff --git a/leveldb/benchmarks/db_bench_tree_db.cc b/leveldb/benchmarks/db_bench_tree_db.cc new file mode 100644 index 000000000..533600b1b --- /dev/null +++ b/leveldb/benchmarks/db_bench_tree_db.cc @@ -0,0 +1,531 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include +#include + +#include "util/histogram.h" +#include "util/random.h" +#include "util/testutil.h" + +// Comma-separated list of operations to run in the specified order +// Actual benchmarks: +// +// fillseq -- write N values in sequential key order in async mode +// fillrandom -- write N values in random key order in async mode +// overwrite -- overwrite N values in random key order in async mode +// fillseqsync -- write N/100 values in sequential key order in sync mode +// fillrandsync -- write N/100 values in random key order in sync mode +// fillrand100K -- write N/1000 100K values in random order in async mode +// fillseq100K -- write N/1000 100K values in seq order in async mode +// readseq -- read N times sequentially +// readseq100K -- read N/1000 100K values in sequential order in async mode +// readrand100K -- read N/1000 100K values in sequential order in async mode +// readrandom -- read N times in random order +static const char* FLAGS_benchmarks = + "fillseq," + "fillseqsync," + "fillrandsync," + "fillrandom," + "overwrite," + "readrandom," + "readseq," + "fillrand100K," + "fillseq100K," + "readseq100K," + "readrand100K,"; + +// Number of key/values to place in database +static int FLAGS_num = 1000000; + +// Number of read operations to do. If negative, do FLAGS_num reads. +static int FLAGS_reads = -1; + +// Size of each value +static int FLAGS_value_size = 100; + +// Arrange to generate values that shrink to this fraction of +// their original size after compression +static double FLAGS_compression_ratio = 0.5; + +// Print histogram of operation timings +static bool FLAGS_histogram = false; + +// Cache size. Default 4 MB +static int FLAGS_cache_size = 4194304; + +// Page size. Default 1 KB +static int FLAGS_page_size = 1024; + +// If true, do not destroy the existing database. If you set this +// flag and also specify a benchmark that wants a fresh database, that +// benchmark will fail. +static bool FLAGS_use_existing_db = false; + +// Compression flag. If true, compression is on. If false, compression +// is off. +static bool FLAGS_compression = true; + +// Use the db with the following name. +static const char* FLAGS_db = nullptr; + +inline static void DBSynchronize(kyotocabinet::TreeDB* db_) { + // Synchronize will flush writes to disk + if (!db_->synchronize()) { + std::fprintf(stderr, "synchronize error: %s\n", db_->error().name()); + } +} + +namespace leveldb { + +// Helper for quickly generating random data. +namespace { +class RandomGenerator { + private: + std::string data_; + int pos_; + + public: + RandomGenerator() { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + // large enough to serve all typical value sizes we want to write. + Random rnd(301); + std::string piece; + while (data_.size() < 1048576) { + // Add a short fragment that is as compressible as specified + // by FLAGS_compression_ratio. + test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); + data_.append(piece); + } + pos_ = 0; + } + + Slice Generate(int len) { + if (pos_ + len > data_.size()) { + pos_ = 0; + assert(len < data_.size()); + } + pos_ += len; + return Slice(data_.data() + pos_ - len, len); + } +}; + +static Slice TrimSpace(Slice s) { + int start = 0; + while (start < s.size() && isspace(s[start])) { + start++; + } + int limit = s.size(); + while (limit > start && isspace(s[limit - 1])) { + limit--; + } + return Slice(s.data() + start, limit - start); +} + +} // namespace + +class Benchmark { + private: + kyotocabinet::TreeDB* db_; + int db_num_; + int num_; + int reads_; + double start_; + double last_op_finish_; + int64_t bytes_; + std::string message_; + Histogram hist_; + RandomGenerator gen_; + Random rand_; + kyotocabinet::LZOCompressor comp_; + + // State kept for progress messages + int done_; + int next_report_; // When to report next + + void PrintHeader() { + const int kKeySize = 16; + PrintEnvironment(); + std::fprintf(stdout, "Keys: %d bytes each\n", kKeySize); + std::fprintf( + stdout, "Values: %d bytes each (%d bytes after compression)\n", + FLAGS_value_size, + static_cast(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); + std::fprintf(stdout, "Entries: %d\n", num_); + std::fprintf(stdout, "RawSize: %.1f MB (estimated)\n", + ((static_cast(kKeySize + FLAGS_value_size) * num_) / + 1048576.0)); + std::fprintf( + stdout, "FileSize: %.1f MB (estimated)\n", + (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) / + 1048576.0)); + PrintWarnings(); + std::fprintf(stdout, "------------------------------------------------\n"); + } + + void PrintWarnings() { +#if defined(__GNUC__) && !defined(__OPTIMIZE__) + std::fprintf( + stdout, + "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"); +#endif +#ifndef NDEBUG + std::fprintf( + stdout, + "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); +#endif + } + + void PrintEnvironment() { + std::fprintf( + stderr, "Kyoto Cabinet: version %s, lib ver %d, lib rev %d\n", + kyotocabinet::VERSION, kyotocabinet::LIBVER, kyotocabinet::LIBREV); + +#if defined(__linux) + time_t now = time(nullptr); + std::fprintf(stderr, "Date: %s", + ctime(&now)); // ctime() adds newline + + FILE* cpuinfo = std::fopen("/proc/cpuinfo", "r"); + if (cpuinfo != nullptr) { + char line[1000]; + int num_cpus = 0; + std::string cpu_type; + std::string cache_size; + while (fgets(line, sizeof(line), cpuinfo) != nullptr) { + const char* sep = strchr(line, ':'); + if (sep == nullptr) { + continue; + } + Slice key = TrimSpace(Slice(line, sep - 1 - line)); + Slice val = TrimSpace(Slice(sep + 1)); + if (key == "model name") { + ++num_cpus; + cpu_type = val.ToString(); + } else if (key == "cache size") { + cache_size = val.ToString(); + } + } + std::fclose(cpuinfo); + std::fprintf(stderr, "CPU: %d * %s\n", num_cpus, + cpu_type.c_str()); + std::fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); + } +#endif + } + + void Start() { + start_ = Env::Default()->NowMicros() * 1e-6; + bytes_ = 0; + message_.clear(); + last_op_finish_ = start_; + hist_.Clear(); + done_ = 0; + next_report_ = 100; + } + + void FinishedSingleOp() { + if (FLAGS_histogram) { + double now = Env::Default()->NowMicros() * 1e-6; + double micros = (now - last_op_finish_) * 1e6; + hist_.Add(micros); + if (micros > 20000) { + std::fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); + std::fflush(stderr); + } + last_op_finish_ = now; + } + + done_++; + if (done_ >= next_report_) { + if (next_report_ < 1000) + next_report_ += 100; + else if (next_report_ < 5000) + next_report_ += 500; + else if (next_report_ < 10000) + next_report_ += 1000; + else if (next_report_ < 50000) + next_report_ += 5000; + else if (next_report_ < 100000) + next_report_ += 10000; + else if (next_report_ < 500000) + next_report_ += 50000; + else + next_report_ += 100000; + std::fprintf(stderr, "... finished %d ops%30s\r", done_, ""); + std::fflush(stderr); + } + } + + void Stop(const Slice& name) { + double finish = Env::Default()->NowMicros() * 1e-6; + + // Pretend at least one op was done in case we are running a benchmark + // that does not call FinishedSingleOp(). + if (done_ < 1) done_ = 1; + + if (bytes_ > 0) { + char rate[100]; + std::snprintf(rate, sizeof(rate), "%6.1f MB/s", + (bytes_ / 1048576.0) / (finish - start_)); + if (!message_.empty()) { + message_ = std::string(rate) + " " + message_; + } else { + message_ = rate; + } + } + + std::fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", + name.ToString().c_str(), (finish - start_) * 1e6 / done_, + (message_.empty() ? "" : " "), message_.c_str()); + if (FLAGS_histogram) { + std::fprintf(stdout, "Microseconds per op:\n%s\n", + hist_.ToString().c_str()); + } + std::fflush(stdout); + } + + public: + enum Order { SEQUENTIAL, RANDOM }; + enum DBState { FRESH, EXISTING }; + + Benchmark() + : db_(nullptr), + num_(FLAGS_num), + reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), + bytes_(0), + rand_(301) { + std::vector files; + std::string test_dir; + Env::Default()->GetTestDirectory(&test_dir); + Env::Default()->GetChildren(test_dir.c_str(), &files); + if (!FLAGS_use_existing_db) { + for (int i = 0; i < files.size(); i++) { + if (Slice(files[i]).starts_with("dbbench_polyDB")) { + std::string file_name(test_dir); + file_name += "/"; + file_name += files[i]; + Env::Default()->RemoveFile(file_name.c_str()); + } + } + } + } + + ~Benchmark() { + if (!db_->close()) { + std::fprintf(stderr, "close error: %s\n", db_->error().name()); + } + } + + void Run() { + PrintHeader(); + Open(false); + + const char* benchmarks = FLAGS_benchmarks; + while (benchmarks != nullptr) { + const char* sep = strchr(benchmarks, ','); + Slice name; + if (sep == nullptr) { + name = benchmarks; + benchmarks = nullptr; + } else { + name = Slice(benchmarks, sep - benchmarks); + benchmarks = sep + 1; + } + + Start(); + + bool known = true; + bool write_sync = false; + if (name == Slice("fillseq")) { + Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("fillrandom")) { + Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("overwrite")) { + Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("fillrandsync")) { + write_sync = true; + Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("fillseqsync")) { + write_sync = true; + Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1); + DBSynchronize(db_); + } else if (name == Slice("fillrand100K")) { + Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1); + DBSynchronize(db_); + } else if (name == Slice("fillseq100K")) { + Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1); + DBSynchronize(db_); + } else if (name == Slice("readseq")) { + ReadSequential(); + } else if (name == Slice("readrandom")) { + ReadRandom(); + } else if (name == Slice("readrand100K")) { + int n = reads_; + reads_ /= 1000; + ReadRandom(); + reads_ = n; + } else if (name == Slice("readseq100K")) { + int n = reads_; + reads_ /= 1000; + ReadSequential(); + reads_ = n; + } else { + known = false; + if (name != Slice()) { // No error message for empty name + std::fprintf(stderr, "unknown benchmark '%s'\n", + name.ToString().c_str()); + } + } + if (known) { + Stop(name); + } + } + } + + private: + void Open(bool sync) { + assert(db_ == nullptr); + + // Initialize db_ + db_ = new kyotocabinet::TreeDB(); + char file_name[100]; + db_num_++; + std::string test_dir; + Env::Default()->GetTestDirectory(&test_dir); + std::snprintf(file_name, sizeof(file_name), "%s/dbbench_polyDB-%d.kct", + test_dir.c_str(), db_num_); + + // Create tuning options and open the database + int open_options = + kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE; + int tune_options = + kyotocabinet::TreeDB::TSMALL | kyotocabinet::TreeDB::TLINEAR; + if (FLAGS_compression) { + tune_options |= kyotocabinet::TreeDB::TCOMPRESS; + db_->tune_compressor(&comp_); + } + db_->tune_options(tune_options); + db_->tune_page_cache(FLAGS_cache_size); + db_->tune_page(FLAGS_page_size); + db_->tune_map(256LL << 20); + if (sync) { + open_options |= kyotocabinet::PolyDB::OAUTOSYNC; + } + if (!db_->open(file_name, open_options)) { + std::fprintf(stderr, "open error: %s\n", db_->error().name()); + } + } + + void Write(bool sync, Order order, DBState state, int num_entries, + int value_size, int entries_per_batch) { + // Create new database if state == FRESH + if (state == FRESH) { + if (FLAGS_use_existing_db) { + message_ = "skipping (--use_existing_db is true)"; + return; + } + delete db_; + db_ = nullptr; + Open(sync); + Start(); // Do not count time taken to destroy/open + } + + if (num_entries != num_) { + char msg[100]; + std::snprintf(msg, sizeof(msg), "(%d ops)", num_entries); + message_ = msg; + } + + // Write to database + for (int i = 0; i < num_entries; i++) { + const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % num_entries); + char key[100]; + std::snprintf(key, sizeof(key), "%016d", k); + bytes_ += value_size + strlen(key); + std::string cpp_key = key; + if (!db_->set(cpp_key, gen_.Generate(value_size).ToString())) { + std::fprintf(stderr, "set error: %s\n", db_->error().name()); + } + FinishedSingleOp(); + } + } + + void ReadSequential() { + kyotocabinet::DB::Cursor* cur = db_->cursor(); + cur->jump(); + std::string ckey, cvalue; + while (cur->get(&ckey, &cvalue, true)) { + bytes_ += ckey.size() + cvalue.size(); + FinishedSingleOp(); + } + delete cur; + } + + void ReadRandom() { + std::string value; + for (int i = 0; i < reads_; i++) { + char key[100]; + const int k = rand_.Next() % reads_; + std::snprintf(key, sizeof(key), "%016d", k); + db_->get(key, &value); + FinishedSingleOp(); + } + } +}; + +} // namespace leveldb + +int main(int argc, char** argv) { + std::string default_db_path; + for (int i = 1; i < argc; i++) { + double d; + int n; + char junk; + if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { + FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); + } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { + FLAGS_compression_ratio = d; + } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_histogram = n; + } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { + FLAGS_num = n; + } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { + FLAGS_reads = n; + } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { + FLAGS_value_size = n; + } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) { + FLAGS_cache_size = n; + } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) { + FLAGS_page_size = n; + } else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_compression = (n == 1) ? true : false; + } else if (strncmp(argv[i], "--db=", 5) == 0) { + FLAGS_db = argv[i] + 5; + } else { + std::fprintf(stderr, "Invalid flag '%s'\n", argv[i]); + std::exit(1); + } + } + + // Choose a location for the test database if none given with --db= + if (FLAGS_db == nullptr) { + leveldb::Env::Default()->GetTestDirectory(&default_db_path); + default_db_path += "/dbbench"; + FLAGS_db = default_db_path.c_str(); + } + + leveldb::Benchmark benchmark; + benchmark.Run(); + return 0; +} diff --git a/leveldb/cmake/leveldbConfig.cmake.in b/leveldb/cmake/leveldbConfig.cmake.in new file mode 100644 index 000000000..2572728f6 --- /dev/null +++ b/leveldb/cmake/leveldbConfig.cmake.in @@ -0,0 +1,9 @@ +# Copyright 2019 The LevelDB Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. See the AUTHORS file for names of contributors. + +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/leveldbTargets.cmake") + +check_required_components(leveldb) \ No newline at end of file diff --git a/leveldb/db/autocompact_test.cc b/leveldb/db/autocompact_test.cc new file mode 100644 index 000000000..69341e3c9 --- /dev/null +++ b/leveldb/db/autocompact_test.cc @@ -0,0 +1,110 @@ +// Copyright (c) 2013 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "gtest/gtest.h" +#include "db/db_impl.h" +#include "leveldb/cache.h" +#include "leveldb/db.h" +#include "util/testutil.h" + +namespace leveldb { + +class AutoCompactTest : public testing::Test { + public: + AutoCompactTest() { + dbname_ = testing::TempDir() + "autocompact_test"; + tiny_cache_ = NewLRUCache(100); + options_.block_cache = tiny_cache_; + DestroyDB(dbname_, options_); + options_.create_if_missing = true; + options_.compression = kNoCompression; + EXPECT_LEVELDB_OK(DB::Open(options_, dbname_, &db_)); + } + + ~AutoCompactTest() { + delete db_; + DestroyDB(dbname_, Options()); + delete tiny_cache_; + } + + std::string Key(int i) { + char buf[100]; + std::snprintf(buf, sizeof(buf), "key%06d", i); + return std::string(buf); + } + + uint64_t Size(const Slice& start, const Slice& limit) { + Range r(start, limit); + uint64_t size; + db_->GetApproximateSizes(&r, 1, &size); + return size; + } + + void DoReads(int n); + + private: + std::string dbname_; + Cache* tiny_cache_; + Options options_; + DB* db_; +}; + +static const int kValueSize = 200 * 1024; +static const int kTotalSize = 100 * 1024 * 1024; +static const int kCount = kTotalSize / kValueSize; + +// Read through the first n keys repeatedly and check that they get +// compacted (verified by checking the size of the key space). +void AutoCompactTest::DoReads(int n) { + std::string value(kValueSize, 'x'); + DBImpl* dbi = reinterpret_cast(db_); + + // Fill database + for (int i = 0; i < kCount; i++) { + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), Key(i), value)); + } + ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable()); + + // Delete everything + for (int i = 0; i < kCount; i++) { + ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), Key(i))); + } + ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable()); + + // Get initial measurement of the space we will be reading. + const int64_t initial_size = Size(Key(0), Key(n)); + const int64_t initial_other_size = Size(Key(n), Key(kCount)); + + // Read until size drops significantly. + std::string limit_key = Key(n); + for (int read = 0; true; read++) { + ASSERT_LT(read, 100) << "Taking too long to compact"; + Iterator* iter = db_->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); + iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) { + // Drop data + } + delete iter; + // Wait a little bit to allow any triggered compactions to complete. + Env::Default()->SleepForMicroseconds(1000000); + uint64_t size = Size(Key(0), Key(n)); + std::fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1, + size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0); + if (size <= initial_size / 10) { + break; + } + } + + // Verify that the size of the key space not touched by the reads + // is pretty much unchanged. + const int64_t final_other_size = Size(Key(n), Key(kCount)); + ASSERT_LE(final_other_size, initial_other_size + 1048576); + ASSERT_GE(final_other_size, initial_other_size / 5 - 1048576); +} + +TEST_F(AutoCompactTest, ReadAll) { DoReads(kCount); } + +TEST_F(AutoCompactTest, ReadHalf) { DoReads(kCount / 2); } + +} // namespace leveldb diff --git a/leveldb/db/builder.cc b/leveldb/db/builder.cc new file mode 100644 index 000000000..e6329e05e --- /dev/null +++ b/leveldb/db/builder.cc @@ -0,0 +1,82 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/builder.h" + +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/table_cache.h" +#include "db/version_edit.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" + +namespace leveldb { + +Status BuildTable(const std::string& dbname, Env* env, const Options& options, + TableCache* table_cache, Iterator* iter, FileMetaData* meta) { + Status s; + meta->file_size = 0; + iter->SeekToFirst(); + + std::string fname = TableFileName(dbname, meta->number); + if (iter->Valid()) { + WritableFile* file; + s = env->NewWritableFile(fname, &file); + if (!s.ok()) { + return s; + } + + TableBuilder* builder = new TableBuilder(options, file); + meta->smallest.DecodeFrom(iter->key()); + Slice key; + for (; iter->Valid(); iter->Next()) { + key = iter->key(); + builder->Add(key, iter->value()); + } + if (!key.empty()) { + meta->largest.DecodeFrom(key); + } + + // Finish and check for builder errors + s = builder->Finish(); + if (s.ok()) { + meta->file_size = builder->FileSize(); + assert(meta->file_size > 0); + } + delete builder; + + // Finish and check for file errors + if (s.ok()) { + s = file->Sync(); + } + if (s.ok()) { + s = file->Close(); + } + delete file; + file = nullptr; + + if (s.ok()) { + // Verify that the table is usable + Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number, + meta->file_size); + s = it->status(); + delete it; + } + } + + // Check for input iterator errors + if (!iter->status().ok()) { + s = iter->status(); + } + + if (s.ok() && meta->file_size > 0) { + // Keep it + } else { + env->RemoveFile(fname); + } + return s; +} + +} // namespace leveldb diff --git a/leveldb/db/builder.h b/leveldb/db/builder.h new file mode 100644 index 000000000..7bd0b8049 --- /dev/null +++ b/leveldb/db/builder.h @@ -0,0 +1,30 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_BUILDER_H_ +#define STORAGE_LEVELDB_DB_BUILDER_H_ + +#include "leveldb/status.h" + +namespace leveldb { + +struct Options; +struct FileMetaData; + +class Env; +class Iterator; +class TableCache; +class VersionEdit; + +// Build a Table file from the contents of *iter. The generated file +// will be named according to meta->number. On success, the rest of +// *meta will be filled with metadata about the generated table. +// If no data is present in *iter, meta->file_size will be set to +// zero, and no Table file will be produced. +Status BuildTable(const std::string& dbname, Env* env, const Options& options, + TableCache* table_cache, Iterator* iter, FileMetaData* meta); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_BUILDER_H_ diff --git a/leveldb/db/c.cc b/leveldb/db/c.cc new file mode 100644 index 000000000..8bdde3834 --- /dev/null +++ b/leveldb/db/c.cc @@ -0,0 +1,565 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/c.h" + +#include + +#include +#include + +#include "leveldb/cache.h" +#include "leveldb/comparator.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/filter_policy.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" +#include "leveldb/status.h" +#include "leveldb/write_batch.h" + +using leveldb::Cache; +using leveldb::Comparator; +using leveldb::CompressionType; +using leveldb::DB; +using leveldb::Env; +using leveldb::FileLock; +using leveldb::FilterPolicy; +using leveldb::Iterator; +using leveldb::kMajorVersion; +using leveldb::kMinorVersion; +using leveldb::Logger; +using leveldb::NewBloomFilterPolicy; +using leveldb::NewLRUCache; +using leveldb::Options; +using leveldb::RandomAccessFile; +using leveldb::Range; +using leveldb::ReadOptions; +using leveldb::SequentialFile; +using leveldb::Slice; +using leveldb::Snapshot; +using leveldb::Status; +using leveldb::WritableFile; +using leveldb::WriteBatch; +using leveldb::WriteOptions; + +extern "C" { + +struct leveldb_t { + DB* rep; +}; +struct leveldb_iterator_t { + Iterator* rep; +}; +struct leveldb_writebatch_t { + WriteBatch rep; +}; +struct leveldb_snapshot_t { + const Snapshot* rep; +}; +struct leveldb_readoptions_t { + ReadOptions rep; +}; +struct leveldb_writeoptions_t { + WriteOptions rep; +}; +struct leveldb_options_t { + Options rep; +}; +struct leveldb_cache_t { + Cache* rep; +}; +struct leveldb_seqfile_t { + SequentialFile* rep; +}; +struct leveldb_randomfile_t { + RandomAccessFile* rep; +}; +struct leveldb_writablefile_t { + WritableFile* rep; +}; +struct leveldb_logger_t { + Logger* rep; +}; +struct leveldb_filelock_t { + FileLock* rep; +}; + +struct leveldb_comparator_t : public Comparator { + ~leveldb_comparator_t() override { (*destructor_)(state_); } + + int Compare(const Slice& a, const Slice& b) const override { + return (*compare_)(state_, a.data(), a.size(), b.data(), b.size()); + } + + const char* Name() const override { return (*name_)(state_); } + + // No-ops since the C binding does not support key shortening methods. + void FindShortestSeparator(std::string*, const Slice&) const override {} + void FindShortSuccessor(std::string* key) const override {} + + void* state_; + void (*destructor_)(void*); + int (*compare_)(void*, const char* a, size_t alen, const char* b, + size_t blen); + const char* (*name_)(void*); +}; + +struct leveldb_filterpolicy_t : public FilterPolicy { + ~leveldb_filterpolicy_t() override { (*destructor_)(state_); } + + const char* Name() const override { return (*name_)(state_); } + + void CreateFilter(const Slice* keys, int n, std::string* dst) const override { + std::vector key_pointers(n); + std::vector key_sizes(n); + for (int i = 0; i < n; i++) { + key_pointers[i] = keys[i].data(); + key_sizes[i] = keys[i].size(); + } + size_t len; + char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len); + dst->append(filter, len); + std::free(filter); + } + + bool KeyMayMatch(const Slice& key, const Slice& filter) const override { + return (*key_match_)(state_, key.data(), key.size(), filter.data(), + filter.size()); + } + + void* state_; + void (*destructor_)(void*); + const char* (*name_)(void*); + char* (*create_)(void*, const char* const* key_array, + const size_t* key_length_array, int num_keys, + size_t* filter_length); + uint8_t (*key_match_)(void*, const char* key, size_t length, + const char* filter, size_t filter_length); +}; + +struct leveldb_env_t { + Env* rep; + bool is_default; +}; + +static bool SaveError(char** errptr, const Status& s) { + assert(errptr != nullptr); + if (s.ok()) { + return false; + } else if (*errptr == nullptr) { + *errptr = strdup(s.ToString().c_str()); + } else { + // TODO(sanjay): Merge with existing error? + std::free(*errptr); + *errptr = strdup(s.ToString().c_str()); + } + return true; +} + +static char* CopyString(const std::string& str) { + char* result = + reinterpret_cast(std::malloc(sizeof(char) * str.size())); + std::memcpy(result, str.data(), sizeof(char) * str.size()); + return result; +} + +leveldb_t* leveldb_open(const leveldb_options_t* options, const char* name, + char** errptr) { + DB* db; + if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) { + return nullptr; + } + leveldb_t* result = new leveldb_t; + result->rep = db; + return result; +} + +void leveldb_close(leveldb_t* db) { + delete db->rep; + delete db; +} + +void leveldb_put(leveldb_t* db, const leveldb_writeoptions_t* options, + const char* key, size_t keylen, const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, + db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen))); +} + +void leveldb_delete(leveldb_t* db, const leveldb_writeoptions_t* options, + const char* key, size_t keylen, char** errptr) { + SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen))); +} + +void leveldb_write(leveldb_t* db, const leveldb_writeoptions_t* options, + leveldb_writebatch_t* batch, char** errptr) { + SaveError(errptr, db->rep->Write(options->rep, &batch->rep)); +} + +char* leveldb_get(leveldb_t* db, const leveldb_readoptions_t* options, + const char* key, size_t keylen, size_t* vallen, + char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +leveldb_iterator_t* leveldb_create_iterator( + leveldb_t* db, const leveldb_readoptions_t* options) { + leveldb_iterator_t* result = new leveldb_iterator_t; + result->rep = db->rep->NewIterator(options->rep); + return result; +} + +const leveldb_snapshot_t* leveldb_create_snapshot(leveldb_t* db) { + leveldb_snapshot_t* result = new leveldb_snapshot_t; + result->rep = db->rep->GetSnapshot(); + return result; +} + +void leveldb_release_snapshot(leveldb_t* db, + const leveldb_snapshot_t* snapshot) { + db->rep->ReleaseSnapshot(snapshot->rep); + delete snapshot; +} + +char* leveldb_property_value(leveldb_t* db, const char* propname) { + std::string tmp; + if (db->rep->GetProperty(Slice(propname), &tmp)) { + // We use strdup() since we expect human readable output. + return strdup(tmp.c_str()); + } else { + return nullptr; + } +} + +void leveldb_approximate_sizes(leveldb_t* db, int num_ranges, + const char* const* range_start_key, + const size_t* range_start_key_len, + const char* const* range_limit_key, + const size_t* range_limit_key_len, + uint64_t* sizes) { + Range* ranges = new Range[num_ranges]; + for (int i = 0; i < num_ranges; i++) { + ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]); + ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]); + } + db->rep->GetApproximateSizes(ranges, num_ranges, sizes); + delete[] ranges; +} + +void leveldb_compact_range(leveldb_t* db, const char* start_key, + size_t start_key_len, const char* limit_key, + size_t limit_key_len) { + Slice a, b; + db->rep->CompactRange( + // Pass null Slice if corresponding "const char*" is null + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); +} + +void leveldb_destroy_db(const leveldb_options_t* options, const char* name, + char** errptr) { + SaveError(errptr, DestroyDB(name, options->rep)); +} + +void leveldb_repair_db(const leveldb_options_t* options, const char* name, + char** errptr) { + SaveError(errptr, RepairDB(name, options->rep)); +} + +void leveldb_iter_destroy(leveldb_iterator_t* iter) { + delete iter->rep; + delete iter; +} + +uint8_t leveldb_iter_valid(const leveldb_iterator_t* iter) { + return iter->rep->Valid(); +} + +void leveldb_iter_seek_to_first(leveldb_iterator_t* iter) { + iter->rep->SeekToFirst(); +} + +void leveldb_iter_seek_to_last(leveldb_iterator_t* iter) { + iter->rep->SeekToLast(); +} + +void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) { + iter->rep->Seek(Slice(k, klen)); +} + +void leveldb_iter_next(leveldb_iterator_t* iter) { iter->rep->Next(); } + +void leveldb_iter_prev(leveldb_iterator_t* iter) { iter->rep->Prev(); } + +const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) { + Slice s = iter->rep->key(); + *klen = s.size(); + return s.data(); +} + +const char* leveldb_iter_value(const leveldb_iterator_t* iter, size_t* vlen) { + Slice s = iter->rep->value(); + *vlen = s.size(); + return s.data(); +} + +void leveldb_iter_get_error(const leveldb_iterator_t* iter, char** errptr) { + SaveError(errptr, iter->rep->status()); +} + +leveldb_writebatch_t* leveldb_writebatch_create() { + return new leveldb_writebatch_t; +} + +void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { delete b; } + +void leveldb_writebatch_clear(leveldb_writebatch_t* b) { b->rep.Clear(); } + +void leveldb_writebatch_put(leveldb_writebatch_t* b, const char* key, + size_t klen, const char* val, size_t vlen) { + b->rep.Put(Slice(key, klen), Slice(val, vlen)); +} + +void leveldb_writebatch_delete(leveldb_writebatch_t* b, const char* key, + size_t klen) { + b->rep.Delete(Slice(key, klen)); +} + +void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, + const char* v, size_t vlen), + void (*deleted)(void*, const char* k, + size_t klen)) { + class H : public WriteBatch::Handler { + public: + void* state_; + void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen); + void (*deleted_)(void*, const char* k, size_t klen); + void Put(const Slice& key, const Slice& value) override { + (*put_)(state_, key.data(), key.size(), value.data(), value.size()); + } + void Delete(const Slice& key) override { + (*deleted_)(state_, key.data(), key.size()); + } + }; + H handler; + handler.state_ = state; + handler.put_ = put; + handler.deleted_ = deleted; + b->rep.Iterate(&handler); +} + +void leveldb_writebatch_append(leveldb_writebatch_t* destination, + const leveldb_writebatch_t* source) { + destination->rep.Append(source->rep); +} + +leveldb_options_t* leveldb_options_create() { return new leveldb_options_t; } + +void leveldb_options_destroy(leveldb_options_t* options) { delete options; } + +void leveldb_options_set_comparator(leveldb_options_t* opt, + leveldb_comparator_t* cmp) { + opt->rep.comparator = cmp; +} + +void leveldb_options_set_filter_policy(leveldb_options_t* opt, + leveldb_filterpolicy_t* policy) { + opt->rep.filter_policy = policy; +} + +void leveldb_options_set_create_if_missing(leveldb_options_t* opt, uint8_t v) { + opt->rep.create_if_missing = v; +} + +void leveldb_options_set_error_if_exists(leveldb_options_t* opt, uint8_t v) { + opt->rep.error_if_exists = v; +} + +void leveldb_options_set_paranoid_checks(leveldb_options_t* opt, uint8_t v) { + opt->rep.paranoid_checks = v; +} + +void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) { + opt->rep.env = (env ? env->rep : nullptr); +} + +void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) { + opt->rep.info_log = (l ? l->rep : nullptr); +} + +void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) { + opt->rep.write_buffer_size = s; +} + +void leveldb_options_set_max_open_files(leveldb_options_t* opt, int n) { + opt->rep.max_open_files = n; +} + +void leveldb_options_set_cache(leveldb_options_t* opt, leveldb_cache_t* c) { + opt->rep.block_cache = c->rep; +} + +void leveldb_options_set_block_size(leveldb_options_t* opt, size_t s) { + opt->rep.block_size = s; +} + +void leveldb_options_set_block_restart_interval(leveldb_options_t* opt, int n) { + opt->rep.block_restart_interval = n; +} + +void leveldb_options_set_max_file_size(leveldb_options_t* opt, size_t s) { + opt->rep.max_file_size = s; +} + +void leveldb_options_set_compression(leveldb_options_t* opt, int t) { + opt->rep.compression = static_cast(t); +} + +leveldb_comparator_t* leveldb_comparator_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + const char* (*name)(void*)) { + leveldb_comparator_t* result = new leveldb_comparator_t; + result->state_ = state; + result->destructor_ = destructor; + result->compare_ = compare; + result->name_ = name; + return result; +} + +void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { delete cmp; } + +leveldb_filterpolicy_t* leveldb_filterpolicy_create( + void* state, void (*destructor)(void*), + char* (*create_filter)(void*, const char* const* key_array, + const size_t* key_length_array, int num_keys, + size_t* filter_length), + uint8_t (*key_may_match)(void*, const char* key, size_t length, + const char* filter, size_t filter_length), + const char* (*name)(void*)) { + leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t; + result->state_ = state; + result->destructor_ = destructor; + result->create_ = create_filter; + result->key_match_ = key_may_match; + result->name_ = name; + return result; +} + +void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t* filter) { + delete filter; +} + +leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) { + // Make a leveldb_filterpolicy_t, but override all of its methods so + // they delegate to a NewBloomFilterPolicy() instead of user + // supplied C functions. + struct Wrapper : public leveldb_filterpolicy_t { + static void DoNothing(void*) {} + + ~Wrapper() { delete rep_; } + const char* Name() const { return rep_->Name(); } + void CreateFilter(const Slice* keys, int n, std::string* dst) const { + return rep_->CreateFilter(keys, n, dst); + } + bool KeyMayMatch(const Slice& key, const Slice& filter) const { + return rep_->KeyMayMatch(key, filter); + } + + const FilterPolicy* rep_; + }; + Wrapper* wrapper = new Wrapper; + wrapper->rep_ = NewBloomFilterPolicy(bits_per_key); + wrapper->state_ = nullptr; + wrapper->destructor_ = &Wrapper::DoNothing; + return wrapper; +} + +leveldb_readoptions_t* leveldb_readoptions_create() { + return new leveldb_readoptions_t; +} + +void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { delete opt; } + +void leveldb_readoptions_set_verify_checksums(leveldb_readoptions_t* opt, + uint8_t v) { + opt->rep.verify_checksums = v; +} + +void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t* opt, uint8_t v) { + opt->rep.fill_cache = v; +} + +void leveldb_readoptions_set_snapshot(leveldb_readoptions_t* opt, + const leveldb_snapshot_t* snap) { + opt->rep.snapshot = (snap ? snap->rep : nullptr); +} + +leveldb_writeoptions_t* leveldb_writeoptions_create() { + return new leveldb_writeoptions_t; +} + +void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { delete opt; } + +void leveldb_writeoptions_set_sync(leveldb_writeoptions_t* opt, uint8_t v) { + opt->rep.sync = v; +} + +leveldb_cache_t* leveldb_cache_create_lru(size_t capacity) { + leveldb_cache_t* c = new leveldb_cache_t; + c->rep = NewLRUCache(capacity); + return c; +} + +void leveldb_cache_destroy(leveldb_cache_t* cache) { + delete cache->rep; + delete cache; +} + +leveldb_env_t* leveldb_create_default_env() { + leveldb_env_t* result = new leveldb_env_t; + result->rep = Env::Default(); + result->is_default = true; + return result; +} + +void leveldb_env_destroy(leveldb_env_t* env) { + if (!env->is_default) delete env->rep; + delete env; +} + +char* leveldb_env_get_test_directory(leveldb_env_t* env) { + std::string result; + if (!env->rep->GetTestDirectory(&result).ok()) { + return nullptr; + } + + char* buffer = static_cast(std::malloc(result.size() + 1)); + std::memcpy(buffer, result.data(), result.size()); + buffer[result.size()] = '\0'; + return buffer; +} + +void leveldb_free(void* ptr) { std::free(ptr); } + +int leveldb_major_version() { return kMajorVersion; } + +int leveldb_minor_version() { return kMinorVersion; } + +} // end extern "C" diff --git a/leveldb/db/c_test.c b/leveldb/db/c_test.c new file mode 100644 index 000000000..16c77eed6 --- /dev/null +++ b/leveldb/db/c_test.c @@ -0,0 +1,384 @@ +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. */ + +#include "leveldb/c.h" + +#include +#include +#include +#include + +const char* phase = ""; + +static void StartPhase(const char* name) { + fprintf(stderr, "=== Test %s\n", name); + phase = name; +} + +#define CheckNoError(err) \ + if ((err) != NULL) { \ + fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \ + abort(); \ + } + +#define CheckCondition(cond) \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \ + abort(); \ + } + +static void CheckEqual(const char* expected, const char* v, size_t n) { + if (expected == NULL && v == NULL) { + // ok + } else if (expected != NULL && v != NULL && n == strlen(expected) && + memcmp(expected, v, n) == 0) { + // ok + return; + } else { + fprintf(stderr, "%s: expected '%s', got '%s'\n", + phase, + (expected ? expected : "(null)"), + (v ? v : "(null")); + abort(); + } +} + +static void Free(char** ptr) { + if (*ptr) { + free(*ptr); + *ptr = NULL; + } +} + +static void CheckGet( + leveldb_t* db, + const leveldb_readoptions_t* options, + const char* key, + const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = leveldb_get(db, options, key, strlen(key), &val_len, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckIter(leveldb_iterator_t* iter, + const char* key, const char* val) { + size_t len; + const char* str; + str = leveldb_iter_key(iter, &len); + CheckEqual(key, str, len); + str = leveldb_iter_value(iter, &len); + CheckEqual(val, str, len); +} + +// Callback from leveldb_writebatch_iterate() +static void CheckPut(void* ptr, + const char* k, size_t klen, + const char* v, size_t vlen) { + int* state = (int*) ptr; + CheckCondition(*state < 2); + switch (*state) { + case 0: + CheckEqual("bar", k, klen); + CheckEqual("b", v, vlen); + break; + case 1: + CheckEqual("box", k, klen); + CheckEqual("c", v, vlen); + break; + } + (*state)++; +} + +// Callback from leveldb_writebatch_iterate() +static void CheckDel(void* ptr, const char* k, size_t klen) { + int* state = (int*) ptr; + CheckCondition(*state == 2); + CheckEqual("bar", k, klen); + (*state)++; +} + +static void CmpDestroy(void* arg) { } + +static int CmpCompare(void* arg, const char* a, size_t alen, + const char* b, size_t blen) { + int n = (alen < blen) ? alen : blen; + int r = memcmp(a, b, n); + if (r == 0) { + if (alen < blen) r = -1; + else if (alen > blen) r = +1; + } + return r; +} + +static const char* CmpName(void* arg) { + return "foo"; +} + +// Custom filter policy +static uint8_t fake_filter_result = 1; +static void FilterDestroy(void* arg) { } +static const char* FilterName(void* arg) { + return "TestFilter"; +} +static char* FilterCreate( + void* arg, + const char* const* key_array, const size_t* key_length_array, + int num_keys, + size_t* filter_length) { + *filter_length = 4; + char* result = malloc(4); + memcpy(result, "fake", 4); + return result; +} +uint8_t FilterKeyMatch(void* arg, const char* key, size_t length, + const char* filter, size_t filter_length) { + CheckCondition(filter_length == 4); + CheckCondition(memcmp(filter, "fake", 4) == 0); + return fake_filter_result; +} + +int main(int argc, char** argv) { + leveldb_t* db; + leveldb_comparator_t* cmp; + leveldb_cache_t* cache; + leveldb_env_t* env; + leveldb_options_t* options; + leveldb_readoptions_t* roptions; + leveldb_writeoptions_t* woptions; + char* dbname; + char* err = NULL; + int run = -1; + + CheckCondition(leveldb_major_version() >= 1); + CheckCondition(leveldb_minor_version() >= 1); + + StartPhase("create_objects"); + cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); + env = leveldb_create_default_env(); + cache = leveldb_cache_create_lru(100000); + dbname = leveldb_env_get_test_directory(env); + CheckCondition(dbname != NULL); + + options = leveldb_options_create(); + leveldb_options_set_comparator(options, cmp); + leveldb_options_set_error_if_exists(options, 1); + leveldb_options_set_cache(options, cache); + leveldb_options_set_env(options, env); + leveldb_options_set_info_log(options, NULL); + leveldb_options_set_write_buffer_size(options, 100000); + leveldb_options_set_paranoid_checks(options, 1); + leveldb_options_set_max_open_files(options, 10); + leveldb_options_set_block_size(options, 1024); + leveldb_options_set_block_restart_interval(options, 8); + leveldb_options_set_max_file_size(options, 3 << 20); + leveldb_options_set_compression(options, leveldb_no_compression); + + roptions = leveldb_readoptions_create(); + leveldb_readoptions_set_verify_checksums(roptions, 1); + leveldb_readoptions_set_fill_cache(roptions, 0); + + woptions = leveldb_writeoptions_create(); + leveldb_writeoptions_set_sync(woptions, 1); + + StartPhase("destroy"); + leveldb_destroy_db(options, dbname, &err); + Free(&err); + + StartPhase("open_error"); + db = leveldb_open(options, dbname, &err); + CheckCondition(err != NULL); + Free(&err); + + StartPhase("leveldb_free"); + db = leveldb_open(options, dbname, &err); + CheckCondition(err != NULL); + leveldb_free(err); + err = NULL; + + StartPhase("open"); + leveldb_options_set_create_if_missing(options, 1); + db = leveldb_open(options, dbname, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", NULL); + + StartPhase("put"); + leveldb_put(db, woptions, "foo", 3, "hello", 5, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("compactall"); + leveldb_compact_range(db, NULL, 0, NULL, 0); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("compactrange"); + leveldb_compact_range(db, "a", 1, "z", 1); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("writebatch"); + { + leveldb_writebatch_t* wb = leveldb_writebatch_create(); + leveldb_writebatch_put(wb, "foo", 3, "a", 1); + leveldb_writebatch_clear(wb); + leveldb_writebatch_put(wb, "bar", 3, "b", 1); + leveldb_writebatch_put(wb, "box", 3, "c", 1); + + leveldb_writebatch_t* wb2 = leveldb_writebatch_create(); + leveldb_writebatch_delete(wb2, "bar", 3); + leveldb_writebatch_append(wb, wb2); + leveldb_writebatch_destroy(wb2); + + leveldb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "hello"); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "box", "c"); + + int pos = 0; + leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); + CheckCondition(pos == 3); + leveldb_writebatch_destroy(wb); + } + + StartPhase("iter"); + { + leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions); + CheckCondition(!leveldb_iter_valid(iter)); + leveldb_iter_seek_to_first(iter); + CheckCondition(leveldb_iter_valid(iter)); + CheckIter(iter, "box", "c"); + leveldb_iter_next(iter); + CheckIter(iter, "foo", "hello"); + leveldb_iter_prev(iter); + CheckIter(iter, "box", "c"); + leveldb_iter_prev(iter); + CheckCondition(!leveldb_iter_valid(iter)); + leveldb_iter_seek_to_last(iter); + CheckIter(iter, "foo", "hello"); + leveldb_iter_seek(iter, "b", 1); + CheckIter(iter, "box", "c"); + leveldb_iter_get_error(iter, &err); + CheckNoError(err); + leveldb_iter_destroy(iter); + } + + StartPhase("approximate_sizes"); + { + int i; + int n = 20000; + char keybuf[100]; + char valbuf[100]; + uint64_t sizes[2]; + const char* start[2] = { "a", "k00000000000000010000" }; + size_t start_len[2] = { 1, 21 }; + const char* limit[2] = { "k00000000000000010000", "z" }; + size_t limit_len[2] = { 21, 1 }; + leveldb_writeoptions_set_sync(woptions, 0); + for (i = 0; i < n; i++) { + snprintf(keybuf, sizeof(keybuf), "k%020d", i); + snprintf(valbuf, sizeof(valbuf), "v%020d", i); + leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf), + &err); + CheckNoError(err); + } + leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes); + CheckCondition(sizes[0] > 0); + CheckCondition(sizes[1] > 0); + } + + StartPhase("property"); + { + char* prop = leveldb_property_value(db, "nosuchprop"); + CheckCondition(prop == NULL); + prop = leveldb_property_value(db, "leveldb.stats"); + CheckCondition(prop != NULL); + Free(&prop); + } + + StartPhase("snapshot"); + { + const leveldb_snapshot_t* snap; + snap = leveldb_create_snapshot(db); + leveldb_delete(db, woptions, "foo", 3, &err); + CheckNoError(err); + leveldb_readoptions_set_snapshot(roptions, snap); + CheckGet(db, roptions, "foo", "hello"); + leveldb_readoptions_set_snapshot(roptions, NULL); + CheckGet(db, roptions, "foo", NULL); + leveldb_release_snapshot(db, snap); + } + + StartPhase("repair"); + { + leveldb_close(db); + leveldb_options_set_create_if_missing(options, 0); + leveldb_options_set_error_if_exists(options, 0); + leveldb_repair_db(options, dbname, &err); + CheckNoError(err); + db = leveldb_open(options, dbname, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", NULL); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "box", "c"); + leveldb_options_set_create_if_missing(options, 1); + leveldb_options_set_error_if_exists(options, 1); + } + + StartPhase("filter"); + for (run = 0; run < 2; run++) { + // First run uses custom filter, second run uses bloom filter + CheckNoError(err); + leveldb_filterpolicy_t* policy; + if (run == 0) { + policy = leveldb_filterpolicy_create( + NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName); + } else { + policy = leveldb_filterpolicy_create_bloom(10); + } + + // Create new database + leveldb_close(db); + leveldb_destroy_db(options, dbname, &err); + leveldb_options_set_filter_policy(options, policy); + db = leveldb_open(options, dbname, &err); + CheckNoError(err); + leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err); + CheckNoError(err); + leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err); + CheckNoError(err); + leveldb_compact_range(db, NULL, 0, NULL, 0); + + fake_filter_result = 1; + CheckGet(db, roptions, "foo", "foovalue"); + CheckGet(db, roptions, "bar", "barvalue"); + if (phase == 0) { + // Must not find value when custom filter returns false + fake_filter_result = 0; + CheckGet(db, roptions, "foo", NULL); + CheckGet(db, roptions, "bar", NULL); + fake_filter_result = 1; + + CheckGet(db, roptions, "foo", "foovalue"); + CheckGet(db, roptions, "bar", "barvalue"); + } + leveldb_options_set_filter_policy(options, NULL); + leveldb_filterpolicy_destroy(policy); + } + + StartPhase("cleanup"); + leveldb_close(db); + leveldb_options_destroy(options); + leveldb_readoptions_destroy(roptions); + leveldb_writeoptions_destroy(woptions); + leveldb_free(dbname); + leveldb_cache_destroy(cache); + leveldb_comparator_destroy(cmp); + leveldb_env_destroy(env); + + fprintf(stderr, "PASS\n"); + return 0; +} diff --git a/leveldb/db/corruption_test.cc b/leveldb/db/corruption_test.cc new file mode 100644 index 000000000..dc7da763f --- /dev/null +++ b/leveldb/db/corruption_test.cc @@ -0,0 +1,362 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "gtest/gtest.h" +#include "db/db_impl.h" +#include "db/filename.h" +#include "db/log_format.h" +#include "db/version_set.h" +#include "leveldb/cache.h" +#include "leveldb/db.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" +#include "util/testutil.h" + +namespace leveldb { + +static const int kValueSize = 1000; + +class CorruptionTest : public testing::Test { + public: + CorruptionTest() + : db_(nullptr), + dbname_("/memenv/corruption_test"), + tiny_cache_(NewLRUCache(100)) { + options_.env = &env_; + options_.block_cache = tiny_cache_; + DestroyDB(dbname_, options_); + + options_.create_if_missing = true; + Reopen(); + options_.create_if_missing = false; + } + + ~CorruptionTest() { + delete db_; + delete tiny_cache_; + } + + Status TryReopen() { + delete db_; + db_ = nullptr; + return DB::Open(options_, dbname_, &db_); + } + + void Reopen() { ASSERT_LEVELDB_OK(TryReopen()); } + + void RepairDB() { + delete db_; + db_ = nullptr; + ASSERT_LEVELDB_OK(::leveldb::RepairDB(dbname_, options_)); + } + + void Build(int n) { + std::string key_space, value_space; + WriteBatch batch; + for (int i = 0; i < n; i++) { + // if ((i % 100) == 0) std::fprintf(stderr, "@ %d of %d\n", i, n); + Slice key = Key(i, &key_space); + batch.Clear(); + batch.Put(key, Value(i, &value_space)); + WriteOptions options; + // Corrupt() doesn't work without this sync on windows; stat reports 0 for + // the file size. + if (i == n - 1) { + options.sync = true; + } + ASSERT_LEVELDB_OK(db_->Write(options, &batch)); + } + } + + void Check(int min_expected, int max_expected) { + int next_expected = 0; + int missed = 0; + int bad_keys = 0; + int bad_values = 0; + int correct = 0; + std::string value_space; + Iterator* iter = db_->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + uint64_t key; + Slice in(iter->key()); + if (in == "" || in == "~") { + // Ignore boundary keys. + continue; + } + if (!ConsumeDecimalNumber(&in, &key) || !in.empty() || + key < next_expected) { + bad_keys++; + continue; + } + missed += (key - next_expected); + next_expected = key + 1; + if (iter->value() != Value(key, &value_space)) { + bad_values++; + } else { + correct++; + } + } + delete iter; + + std::fprintf( + stderr, + "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n", + min_expected, max_expected, correct, bad_keys, bad_values, missed); + ASSERT_LE(min_expected, correct); + ASSERT_GE(max_expected, correct); + } + + void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { + // Pick file to corrupt + std::vector filenames; + ASSERT_LEVELDB_OK(env_.target()->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + std::string fname; + int picked_number = -1; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == filetype && + int(number) > picked_number) { // Pick latest file + fname = dbname_ + "/" + filenames[i]; + picked_number = number; + } + } + ASSERT_TRUE(!fname.empty()) << filetype; + + uint64_t file_size; + ASSERT_LEVELDB_OK(env_.target()->GetFileSize(fname, &file_size)); + + if (offset < 0) { + // Relative to end of file; make it absolute + if (-offset > file_size) { + offset = 0; + } else { + offset = file_size + offset; + } + } + if (offset > file_size) { + offset = file_size; + } + if (offset + bytes_to_corrupt > file_size) { + bytes_to_corrupt = file_size - offset; + } + + // Do it + std::string contents; + Status s = ReadFileToString(env_.target(), fname, &contents); + ASSERT_TRUE(s.ok()) << s.ToString(); + for (int i = 0; i < bytes_to_corrupt; i++) { + contents[i + offset] ^= 0x80; + } + s = WriteStringToFile(env_.target(), contents, fname); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + int Property(const std::string& name) { + std::string property; + int result; + if (db_->GetProperty(name, &property) && + sscanf(property.c_str(), "%d", &result) == 1) { + return result; + } else { + return -1; + } + } + + // Return the ith key + Slice Key(int i, std::string* storage) { + char buf[100]; + std::snprintf(buf, sizeof(buf), "%016d", i); + storage->assign(buf, strlen(buf)); + return Slice(*storage); + } + + // Return the value to associate with the specified key + Slice Value(int k, std::string* storage) { + Random r(k); + return test::RandomString(&r, kValueSize, storage); + } + + test::ErrorEnv env_; + Options options_; + DB* db_; + + private: + std::string dbname_; + Cache* tiny_cache_; +}; + +TEST_F(CorruptionTest, Recovery) { + Build(100); + Check(100, 100); + Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record + Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block + Reopen(); + + // The 64 records in the first two log blocks are completely lost. + Check(36, 36); +} + +TEST_F(CorruptionTest, RecoverWriteError) { + env_.writable_file_error_ = true; + Status s = TryReopen(); + ASSERT_TRUE(!s.ok()); +} + +TEST_F(CorruptionTest, NewFileErrorDuringWrite) { + // Do enough writing to force minor compaction + env_.writable_file_error_ = true; + const int num = 3 + (Options().write_buffer_size / kValueSize); + std::string value_storage; + Status s; + for (int i = 0; s.ok() && i < num; i++) { + WriteBatch batch; + batch.Put("a", Value(100, &value_storage)); + s = db_->Write(WriteOptions(), &batch); + } + ASSERT_TRUE(!s.ok()); + ASSERT_GE(env_.num_writable_file_errors_, 1); + env_.writable_file_error_ = false; + Reopen(); +} + +TEST_F(CorruptionTest, TableFile) { + Build(100); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + dbi->TEST_CompactRange(0, nullptr, nullptr); + dbi->TEST_CompactRange(1, nullptr, nullptr); + + Corrupt(kTableFile, 100, 1); + Check(90, 99); +} + +TEST_F(CorruptionTest, TableFileRepair) { + options_.block_size = 2 * kValueSize; // Limit scope of corruption + options_.paranoid_checks = true; + Reopen(); + Build(100); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + dbi->TEST_CompactRange(0, nullptr, nullptr); + dbi->TEST_CompactRange(1, nullptr, nullptr); + + Corrupt(kTableFile, 100, 1); + RepairDB(); + Reopen(); + Check(95, 99); +} + +TEST_F(CorruptionTest, TableFileIndexData) { + Build(10000); // Enough to build multiple Tables + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + + Corrupt(kTableFile, -2000, 500); + Reopen(); + Check(5000, 9999); +} + +TEST_F(CorruptionTest, MissingDescriptor) { + Build(1000); + RepairDB(); + Reopen(); + Check(1000, 1000); +} + +TEST_F(CorruptionTest, SequenceNumberRecovery) { + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v1")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v2")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v3")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v4")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v5")); + RepairDB(); + Reopen(); + std::string v; + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v5", v); + // Write something. If sequence number was not recovered properly, + // it will be hidden by an earlier write. + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v6")); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v6", v); + Reopen(); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v6", v); +} + +TEST_F(CorruptionTest, CorruptedDescriptor) { + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "hello")); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + dbi->TEST_CompactRange(0, nullptr, nullptr); + + Corrupt(kDescriptorFile, 0, 1000); + Status s = TryReopen(); + ASSERT_TRUE(!s.ok()); + + RepairDB(); + Reopen(); + std::string v; + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("hello", v); +} + +TEST_F(CorruptionTest, CompactionInputError) { + Build(10); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last))); + + Corrupt(kTableFile, 100, 1); + Check(5, 9); + + // Force compactions by writing lots of values + Build(10000); + Check(10000, 10000); +} + +TEST_F(CorruptionTest, CompactionInputErrorParanoid) { + options_.paranoid_checks = true; + options_.write_buffer_size = 512 << 10; + Reopen(); + DBImpl* dbi = reinterpret_cast(db_); + + // Make multiple inputs so we need to compact. + for (int i = 0; i < 2; i++) { + Build(10); + dbi->TEST_CompactMemTable(); + Corrupt(kTableFile, 100, 1); + env_.SleepForMicroseconds(100000); + } + dbi->CompactRange(nullptr, nullptr); + + // Write must fail because of corrupted table + std::string tmp1, tmp2; + Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2)); + ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db"; +} + +TEST_F(CorruptionTest, UnrelatedKeys) { + Build(10); + DBImpl* dbi = reinterpret_cast(db_); + dbi->TEST_CompactMemTable(); + Corrupt(kTableFile, 100, 1); + + std::string tmp1, tmp2; + ASSERT_LEVELDB_OK( + db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2))); + std::string v; + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); + ASSERT_EQ(Value(1000, &tmp2).ToString(), v); + dbi->TEST_CompactMemTable(); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); + ASSERT_EQ(Value(1000, &tmp2).ToString(), v); +} + +} // namespace leveldb diff --git a/leveldb/db/db_impl.cc b/leveldb/db/db_impl.cc new file mode 100644 index 000000000..a0afbc8fa --- /dev/null +++ b/leveldb/db/db_impl.cc @@ -0,0 +1,1579 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_impl.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "db/builder.h" +#include "db/db_iter.h" +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/memtable.h" +#include "db/table_cache.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/status.h" +#include "leveldb/table.h" +#include "leveldb/table_builder.h" +#include "port/port.h" +#include "table/block.h" +#include "table/merger.h" +#include "table/two_level_iterator.h" +#include "util/coding.h" +#include "util/logging.h" +#include "util/mutexlock.h" + +namespace leveldb { + +const int kNumNonTableCacheFiles = 10; + +// Information kept for every waiting writer +struct DBImpl::Writer { + explicit Writer(port::Mutex* mu) + : batch(nullptr), sync(false), done(false), cv(mu) {} + + Status status; + WriteBatch* batch; + bool sync; + bool done; + port::CondVar cv; +}; + +struct DBImpl::CompactionState { + // Files produced by compaction + struct Output { + uint64_t number; + uint64_t file_size; + InternalKey smallest, largest; + }; + + Output* current_output() { return &outputs[outputs.size() - 1]; } + + explicit CompactionState(Compaction* c) + : compaction(c), + smallest_snapshot(0), + outfile(nullptr), + builder(nullptr), + total_bytes(0) {} + + Compaction* const compaction; + + // Sequence numbers < smallest_snapshot are not significant since we + // will never have to service a snapshot below smallest_snapshot. + // Therefore if we have seen a sequence number S <= smallest_snapshot, + // we can drop all entries for the same key with sequence numbers < S. + SequenceNumber smallest_snapshot; + + std::vector outputs; + + // State kept for output being generated + WritableFile* outfile; + TableBuilder* builder; + + uint64_t total_bytes; +}; + +// Fix user-supplied options to be reasonable +template +static void ClipToRange(T* ptr, V minvalue, V maxvalue) { + if (static_cast(*ptr) > maxvalue) *ptr = maxvalue; + if (static_cast(*ptr) < minvalue) *ptr = minvalue; +} +Options SanitizeOptions(const std::string& dbname, + const InternalKeyComparator* icmp, + const InternalFilterPolicy* ipolicy, + const Options& src) { + Options result = src; + result.comparator = icmp; + result.filter_policy = (src.filter_policy != nullptr) ? ipolicy : nullptr; + ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000); + ClipToRange(&result.write_buffer_size, 64 << 10, 1 << 30); + ClipToRange(&result.max_file_size, 1 << 20, 1 << 30); + ClipToRange(&result.block_size, 1 << 10, 4 << 20); + if (result.info_log == nullptr) { + // Open a log file in the same directory as the db + src.env->CreateDir(dbname); // In case it does not exist + src.env->RenameFile(InfoLogFileName(dbname), OldInfoLogFileName(dbname)); + Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log); + if (!s.ok()) { + // No place suitable for logging + result.info_log = nullptr; + } + } + if (result.block_cache == nullptr) { + result.block_cache = NewLRUCache(8 << 20); + } + return result; +} + +static int TableCacheSize(const Options& sanitized_options) { + // Reserve ten files or so for other uses and give the rest to TableCache. + return sanitized_options.max_open_files - kNumNonTableCacheFiles; +} + +DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) + : env_(raw_options.env), + internal_comparator_(raw_options.comparator), + internal_filter_policy_(raw_options.filter_policy), + options_(SanitizeOptions(dbname, &internal_comparator_, + &internal_filter_policy_, raw_options)), + owns_info_log_(options_.info_log != raw_options.info_log), + owns_cache_(options_.block_cache != raw_options.block_cache), + dbname_(dbname), + table_cache_(new TableCache(dbname_, options_, TableCacheSize(options_))), + db_lock_(nullptr), + shutting_down_(false), + background_work_finished_signal_(&mutex_), + mem_(nullptr), + imm_(nullptr), + has_imm_(false), + logfile_(nullptr), + logfile_number_(0), + log_(nullptr), + seed_(0), + tmp_batch_(new WriteBatch), + background_compaction_scheduled_(false), + manual_compaction_(nullptr), + versions_(new VersionSet(dbname_, &options_, table_cache_, + &internal_comparator_)) {} + +DBImpl::~DBImpl() { + // Wait for background work to finish. + mutex_.Lock(); + shutting_down_.store(true, std::memory_order_release); + while (background_compaction_scheduled_) { + background_work_finished_signal_.Wait(); + } + mutex_.Unlock(); + + if (db_lock_ != nullptr) { + env_->UnlockFile(db_lock_); + } + + delete versions_; + if (mem_ != nullptr) mem_->Unref(); + if (imm_ != nullptr) imm_->Unref(); + delete tmp_batch_; + delete log_; + delete logfile_; + delete table_cache_; + + if (owns_info_log_) { + delete options_.info_log; + } + if (owns_cache_) { + delete options_.block_cache; + } +} + +Status DBImpl::NewDB() { + VersionEdit new_db; + new_db.SetComparatorName(user_comparator()->Name()); + new_db.SetLogNumber(0); + new_db.SetNextFile(2); + new_db.SetLastSequence(0); + + const std::string manifest = DescriptorFileName(dbname_, 1); + WritableFile* file; + Status s = env_->NewWritableFile(manifest, &file); + if (!s.ok()) { + return s; + } + { + log::Writer log(file); + std::string record; + new_db.EncodeTo(&record); + s = log.AddRecord(record); + if (s.ok()) { + s = file->Sync(); + } + if (s.ok()) { + s = file->Close(); + } + } + delete file; + if (s.ok()) { + // Make "CURRENT" file that points to the new manifest file. + s = SetCurrentFile(env_, dbname_, 1); + } else { + env_->RemoveFile(manifest); + } + return s; +} + +void DBImpl::MaybeIgnoreError(Status* s) const { + if (s->ok() || options_.paranoid_checks) { + // No change needed + } else { + Log(options_.info_log, "Ignoring error %s", s->ToString().c_str()); + *s = Status::OK(); + } +} + +void DBImpl::RemoveObsoleteFiles() { + mutex_.AssertHeld(); + + if (!bg_error_.ok()) { + // After a background error, we don't know whether a new version may + // or may not have been committed, so we cannot safely garbage collect. + return; + } + + // Make a set of all of the live files + std::set live = pending_outputs_; + versions_->AddLiveFiles(&live); + + std::vector filenames; + env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose + uint64_t number; + FileType type; + std::vector files_to_delete; + for (std::string& filename : filenames) { + if (ParseFileName(filename, &number, &type)) { + bool keep = true; + switch (type) { + case kLogFile: + keep = ((number >= versions_->LogNumber()) || + (number == versions_->PrevLogNumber())); + break; + case kDescriptorFile: + // Keep my manifest file, and any newer incarnations' + // (in case there is a race that allows other incarnations) + keep = (number >= versions_->ManifestFileNumber()); + break; + case kTableFile: + keep = (live.find(number) != live.end()); + break; + case kTempFile: + // Any temp files that are currently being written to must + // be recorded in pending_outputs_, which is inserted into "live" + keep = (live.find(number) != live.end()); + break; + case kCurrentFile: + case kDBLockFile: + case kInfoLogFile: + keep = true; + break; + } + + if (!keep) { + files_to_delete.push_back(std::move(filename)); + if (type == kTableFile) { + table_cache_->Evict(number); + } + Log(options_.info_log, "Delete type=%d #%lld\n", static_cast(type), + static_cast(number)); + } + } + } + + // While deleting all files unblock other threads. All files being deleted + // have unique names which will not collide with newly created files and + // are therefore safe to delete while allowing other threads to proceed. + mutex_.Unlock(); + for (const std::string& filename : files_to_delete) { + env_->RemoveFile(dbname_ + "/" + filename); + } + mutex_.Lock(); +} + +Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) { + mutex_.AssertHeld(); + + // Ignore error from CreateDir since the creation of the DB is + // committed only when the descriptor is created, and this directory + // may already exist from a previous failed creation attempt. + env_->CreateDir(dbname_); + assert(db_lock_ == nullptr); + Status s = env_->LockFile(LockFileName(dbname_), &db_lock_); + if (!s.ok()) { + return s; + } + + if (!env_->FileExists(CurrentFileName(dbname_))) { + if (options_.create_if_missing) { + Log(options_.info_log, "Creating DB %s since it was missing.", + dbname_.c_str()); + s = NewDB(); + if (!s.ok()) { + return s; + } + } else { + return Status::InvalidArgument( + dbname_, "does not exist (create_if_missing is false)"); + } + } else { + if (options_.error_if_exists) { + return Status::InvalidArgument(dbname_, + "exists (error_if_exists is true)"); + } + } + + s = versions_->Recover(save_manifest); + if (!s.ok()) { + return s; + } + SequenceNumber max_sequence(0); + + // Recover from all newer log files than the ones named in the + // descriptor (new log files may have been added by the previous + // incarnation without registering them in the descriptor). + // + // Note that PrevLogNumber() is no longer used, but we pay + // attention to it in case we are recovering a database + // produced by an older version of leveldb. + const uint64_t min_log = versions_->LogNumber(); + const uint64_t prev_log = versions_->PrevLogNumber(); + std::vector filenames; + s = env_->GetChildren(dbname_, &filenames); + if (!s.ok()) { + return s; + } + std::set expected; + versions_->AddLiveFiles(&expected); + uint64_t number; + FileType type; + std::vector logs; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type)) { + expected.erase(number); + if (type == kLogFile && ((number >= min_log) || (number == prev_log))) + logs.push_back(number); + } + } + if (!expected.empty()) { + char buf[50]; + std::snprintf(buf, sizeof(buf), "%d missing files; e.g.", + static_cast(expected.size())); + return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); + } + + // Recover in the order in which the logs were generated + std::sort(logs.begin(), logs.end()); + for (size_t i = 0; i < logs.size(); i++) { + s = RecoverLogFile(logs[i], (i == logs.size() - 1), save_manifest, edit, + &max_sequence); + if (!s.ok()) { + return s; + } + + // The previous incarnation may not have written any MANIFEST + // records after allocating this log number. So we manually + // update the file number allocation counter in VersionSet. + versions_->MarkFileNumberUsed(logs[i]); + } + + if (versions_->LastSequence() < max_sequence) { + versions_->SetLastSequence(max_sequence); + } + + return Status::OK(); +} + +Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, + bool* save_manifest, VersionEdit* edit, + SequenceNumber* max_sequence) { + struct LogReporter : public log::Reader::Reporter { + Env* env; + Logger* info_log; + const char* fname; + Status* status; // null if options_.paranoid_checks==false + void Corruption(size_t bytes, const Status& s) override { + Log(info_log, "%s%s: dropping %d bytes; %s", + (this->status == nullptr ? "(ignoring error) " : ""), fname, + static_cast(bytes), s.ToString().c_str()); + if (this->status != nullptr && this->status->ok()) *this->status = s; + } + }; + + mutex_.AssertHeld(); + + // Open the log file + std::string fname = LogFileName(dbname_, log_number); + SequentialFile* file; + Status status = env_->NewSequentialFile(fname, &file); + if (!status.ok()) { + MaybeIgnoreError(&status); + return status; + } + + // Create the log reader. + LogReporter reporter; + reporter.env = env_; + reporter.info_log = options_.info_log; + reporter.fname = fname.c_str(); + reporter.status = (options_.paranoid_checks ? &status : nullptr); + // We intentionally make log::Reader do checksumming even if + // paranoid_checks==false so that corruptions cause entire commits + // to be skipped instead of propagating bad information (like overly + // large sequence numbers). + log::Reader reader(file, &reporter, true /*checksum*/, 0 /*initial_offset*/); + Log(options_.info_log, "Recovering log #%llu", + (unsigned long long)log_number); + + // Read all the records and add to a memtable + std::string scratch; + Slice record; + WriteBatch batch; + int compactions = 0; + MemTable* mem = nullptr; + while (reader.ReadRecord(&record, &scratch) && status.ok()) { + if (record.size() < 12) { + reporter.Corruption(record.size(), + Status::Corruption("log record too small")); + continue; + } + WriteBatchInternal::SetContents(&batch, record); + + if (mem == nullptr) { + mem = new MemTable(internal_comparator_); + mem->Ref(); + } + status = WriteBatchInternal::InsertInto(&batch, mem); + MaybeIgnoreError(&status); + if (!status.ok()) { + break; + } + const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) + + WriteBatchInternal::Count(&batch) - 1; + if (last_seq > *max_sequence) { + *max_sequence = last_seq; + } + + if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) { + compactions++; + *save_manifest = true; + status = WriteLevel0Table(mem, edit, nullptr); + mem->Unref(); + mem = nullptr; + if (!status.ok()) { + // Reflect errors immediately so that conditions like full + // file-systems cause the DB::Open() to fail. + break; + } + } + } + + delete file; + + // See if we should keep reusing the last log file. + if (status.ok() && options_.reuse_logs && last_log && compactions == 0) { + assert(logfile_ == nullptr); + assert(log_ == nullptr); + assert(mem_ == nullptr); + uint64_t lfile_size; + if (env_->GetFileSize(fname, &lfile_size).ok() && + env_->NewAppendableFile(fname, &logfile_).ok()) { + Log(options_.info_log, "Reusing old log %s \n", fname.c_str()); + log_ = new log::Writer(logfile_, lfile_size); + logfile_number_ = log_number; + if (mem != nullptr) { + mem_ = mem; + mem = nullptr; + } else { + // mem can be nullptr if lognum exists but was empty. + mem_ = new MemTable(internal_comparator_); + mem_->Ref(); + } + } + } + + if (mem != nullptr) { + // mem did not get reused; compact it. + if (status.ok()) { + *save_manifest = true; + status = WriteLevel0Table(mem, edit, nullptr); + } + mem->Unref(); + } + + return status; +} + +Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, + Version* base) { + mutex_.AssertHeld(); + const uint64_t start_micros = env_->NowMicros(); + FileMetaData meta; + meta.number = versions_->NewFileNumber(); + pending_outputs_.insert(meta.number); + Iterator* iter = mem->NewIterator(); + Log(options_.info_log, "Level-0 table #%llu: started", + (unsigned long long)meta.number); + + Status s; + { + mutex_.Unlock(); + s = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta); + mutex_.Lock(); + } + + Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s", + (unsigned long long)meta.number, (unsigned long long)meta.file_size, + s.ToString().c_str()); + delete iter; + pending_outputs_.erase(meta.number); + + // Note that if file_size is zero, the file has been deleted and + // should not be added to the manifest. + int level = 0; + if (s.ok() && meta.file_size > 0) { + const Slice min_user_key = meta.smallest.user_key(); + const Slice max_user_key = meta.largest.user_key(); + if (base != nullptr) { + level = base->PickLevelForMemTableOutput(min_user_key, max_user_key); + } + edit->AddFile(level, meta.number, meta.file_size, meta.smallest, + meta.largest); + } + + CompactionStats stats; + stats.micros = env_->NowMicros() - start_micros; + stats.bytes_written = meta.file_size; + stats_[level].Add(stats); + return s; +} + +void DBImpl::CompactMemTable() { + mutex_.AssertHeld(); + assert(imm_ != nullptr); + + // Save the contents of the memtable as a new Table + VersionEdit edit; + Version* base = versions_->current(); + base->Ref(); + Status s = WriteLevel0Table(imm_, &edit, base); + base->Unref(); + + if (s.ok() && shutting_down_.load(std::memory_order_acquire)) { + s = Status::IOError("Deleting DB during memtable compaction"); + } + + // Replace immutable memtable with the generated Table + if (s.ok()) { + edit.SetPrevLogNumber(0); + edit.SetLogNumber(logfile_number_); // Earlier logs no longer needed + s = versions_->LogAndApply(&edit, &mutex_); + } + + if (s.ok()) { + // Commit to the new state + imm_->Unref(); + imm_ = nullptr; + has_imm_.store(false, std::memory_order_release); + RemoveObsoleteFiles(); + } else { + RecordBackgroundError(s); + } +} + +void DBImpl::CompactRange(const Slice* begin, const Slice* end) { + int max_level_with_files = 1; + { + MutexLock l(&mutex_); + Version* base = versions_->current(); + for (int level = 1; level < config::kNumLevels; level++) { + if (base->OverlapInLevel(level, begin, end)) { + max_level_with_files = level; + } + } + } + TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap + for (int level = 0; level < max_level_with_files; level++) { + TEST_CompactRange(level, begin, end); + } +} + +void DBImpl::TEST_CompactRange(int level, const Slice* begin, + const Slice* end) { + assert(level >= 0); + assert(level + 1 < config::kNumLevels); + + InternalKey begin_storage, end_storage; + + ManualCompaction manual; + manual.level = level; + manual.done = false; + if (begin == nullptr) { + manual.begin = nullptr; + } else { + begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek); + manual.begin = &begin_storage; + } + if (end == nullptr) { + manual.end = nullptr; + } else { + end_storage = InternalKey(*end, 0, static_cast(0)); + manual.end = &end_storage; + } + + MutexLock l(&mutex_); + while (!manual.done && !shutting_down_.load(std::memory_order_acquire) && + bg_error_.ok()) { + if (manual_compaction_ == nullptr) { // Idle + manual_compaction_ = &manual; + MaybeScheduleCompaction(); + } else { // Running either my compaction or another compaction. + background_work_finished_signal_.Wait(); + } + } + if (manual_compaction_ == &manual) { + // Cancel my manual compaction since we aborted early for some reason. + manual_compaction_ = nullptr; + } +} + +Status DBImpl::TEST_CompactMemTable() { + // nullptr batch means just wait for earlier writes to be done + Status s = Write(WriteOptions(), nullptr); + if (s.ok()) { + // Wait until the compaction completes + MutexLock l(&mutex_); + while (imm_ != nullptr && bg_error_.ok()) { + background_work_finished_signal_.Wait(); + } + if (imm_ != nullptr) { + s = bg_error_; + } + } + return s; +} + +void DBImpl::RecordBackgroundError(const Status& s) { + mutex_.AssertHeld(); + if (bg_error_.ok()) { + bg_error_ = s; + background_work_finished_signal_.SignalAll(); + } +} + +void DBImpl::MaybeScheduleCompaction() { + mutex_.AssertHeld(); + if (background_compaction_scheduled_) { + // Already scheduled + } else if (shutting_down_.load(std::memory_order_acquire)) { + // DB is being deleted; no more background compactions + } else if (!bg_error_.ok()) { + // Already got an error; no more changes + } else if (imm_ == nullptr && manual_compaction_ == nullptr && + !versions_->NeedsCompaction()) { + // No work to be done + } else { + background_compaction_scheduled_ = true; + env_->Schedule(&DBImpl::BGWork, this); + } +} + +void DBImpl::BGWork(void* db) { + reinterpret_cast(db)->BackgroundCall(); +} + +void DBImpl::BackgroundCall() { + MutexLock l(&mutex_); + assert(background_compaction_scheduled_); + if (shutting_down_.load(std::memory_order_acquire)) { + // No more background work when shutting down. + } else if (!bg_error_.ok()) { + // No more background work after a background error. + } else { + BackgroundCompaction(); + } + + background_compaction_scheduled_ = false; + + // Previous compaction may have produced too many files in a level, + // so reschedule another compaction if needed. + MaybeScheduleCompaction(); + background_work_finished_signal_.SignalAll(); +} + +void DBImpl::BackgroundCompaction() { + mutex_.AssertHeld(); + + if (imm_ != nullptr) { + CompactMemTable(); + return; + } + + Compaction* c; + bool is_manual = (manual_compaction_ != nullptr); + InternalKey manual_end; + if (is_manual) { + ManualCompaction* m = manual_compaction_; + c = versions_->CompactRange(m->level, m->begin, m->end); + m->done = (c == nullptr); + if (c != nullptr) { + manual_end = c->input(0, c->num_input_files(0) - 1)->largest; + } + Log(options_.info_log, + "Manual compaction at level-%d from %s .. %s; will stop at %s\n", + m->level, (m->begin ? m->begin->DebugString().c_str() : "(begin)"), + (m->end ? m->end->DebugString().c_str() : "(end)"), + (m->done ? "(end)" : manual_end.DebugString().c_str())); + } else { + c = versions_->PickCompaction(); + } + + Status status; + if (c == nullptr) { + // Nothing to do + } else if (!is_manual && c->IsTrivialMove()) { + // Move file to next level + assert(c->num_input_files(0) == 1); + FileMetaData* f = c->input(0, 0); + //TODO: Move file to next level + //Tips1: First you should use the function "c->edit()->RemoveFile()" to remove the file from "c->level()" + //Tips2: Then you should use the function "c->edit()->AddFile()" to add the file to "c->level()+1" + //Tips3: See the definition of these functions in db/version_edit.h + status = versions_->LogAndApply(c->edit(), &mutex_); + if (!status.ok()) { + RecordBackgroundError(status); + } + VersionSet::LevelSummaryStorage tmp; + Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n", + static_cast(f->number), c->level() + 1, + static_cast(f->file_size), + status.ToString().c_str(), versions_->LevelSummary(&tmp)); + } else { + CompactionState* compact = new CompactionState(c); + status = DoCompactionWork(compact); + if (!status.ok()) { + RecordBackgroundError(status); + } + CleanupCompaction(compact); + c->ReleaseInputs(); + RemoveObsoleteFiles(); + } + delete c; + + if (status.ok()) { + // Done + } else if (shutting_down_.load(std::memory_order_acquire)) { + // Ignore compaction errors found during shutting down + } else { + Log(options_.info_log, "Compaction error: %s", status.ToString().c_str()); + } + + if (is_manual) { + ManualCompaction* m = manual_compaction_; + if (!status.ok()) { + m->done = true; + } + if (!m->done) { + // We only compacted part of the requested range. Update *m + // to the range that is left to be compacted. + m->tmp_storage = manual_end; + m->begin = &m->tmp_storage; + } + manual_compaction_ = nullptr; + } +} + +void DBImpl::CleanupCompaction(CompactionState* compact) { + mutex_.AssertHeld(); + if (compact->builder != nullptr) { + // May happen if we get a shutdown call in the middle of compaction + compact->builder->Abandon(); + delete compact->builder; + } else { + assert(compact->outfile == nullptr); + } + delete compact->outfile; + for (size_t i = 0; i < compact->outputs.size(); i++) { + const CompactionState::Output& out = compact->outputs[i]; + pending_outputs_.erase(out.number); + } + delete compact; +} + +Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) { + assert(compact != nullptr); + assert(compact->builder == nullptr); + uint64_t file_number; + { + mutex_.Lock(); + file_number = versions_->NewFileNumber(); + pending_outputs_.insert(file_number); + CompactionState::Output out; + out.number = file_number; + out.smallest.Clear(); + out.largest.Clear(); + compact->outputs.push_back(out); + mutex_.Unlock(); + } + + // Make the output file + std::string fname = TableFileName(dbname_, file_number); + Status s = env_->NewWritableFile(fname, &compact->outfile); + if (s.ok()) { + compact->builder = new TableBuilder(options_, compact->outfile); + } + return s; +} + +Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, + Iterator* input) { + assert(compact != nullptr); + assert(compact->outfile != nullptr); + assert(compact->builder != nullptr); + + const uint64_t output_number = compact->current_output()->number; + assert(output_number != 0); + + // Check for iterator errors + Status s = input->status(); + const uint64_t current_entries = compact->builder->NumEntries(); + if (s.ok()) { + s = compact->builder->Finish(); + } else { + compact->builder->Abandon(); + } + const uint64_t current_bytes = compact->builder->FileSize(); + compact->current_output()->file_size = current_bytes; + compact->total_bytes += current_bytes; + delete compact->builder; + compact->builder = nullptr; + + // Finish and check for file errors + if (s.ok()) { + s = compact->outfile->Sync(); + } + if (s.ok()) { + s = compact->outfile->Close(); + } + delete compact->outfile; + compact->outfile = nullptr; + + if (s.ok() && current_entries > 0) { + // Verify that the table is usable + Iterator* iter = + table_cache_->NewIterator(ReadOptions(), output_number, current_bytes); + s = iter->status(); + delete iter; + if (s.ok()) { + Log(options_.info_log, "Generated table #%llu@%d: %lld keys, %lld bytes", + (unsigned long long)output_number, compact->compaction->level(), + (unsigned long long)current_entries, + (unsigned long long)current_bytes); + } + } + return s; +} + +Status DBImpl::InstallCompactionResults(CompactionState* compact) { + mutex_.AssertHeld(); + Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes", + compact->compaction->num_input_files(0), compact->compaction->level(), + compact->compaction->num_input_files(1), compact->compaction->level() + 1, + static_cast(compact->total_bytes)); + + // Add compaction outputs + compact->compaction->AddInputDeletions(compact->compaction->edit()); + const int level = compact->compaction->level(); + for (size_t i = 0; i < compact->outputs.size(); i++) { + //TODO: Add compaction outputs + //Tips1: Outputfiles should be put in level+1. + //Tips2: To add output files, you should use the function "compaction->edit()->AddFile()" + //Tips3: See the definition of "AddFile()" in db/version_edit.h. + } + return versions_->LogAndApply(compact->compaction->edit(), &mutex_); +} + +Status DBImpl::DoCompactionWork(CompactionState* compact) { + const uint64_t start_micros = env_->NowMicros(); + int64_t imm_micros = 0; // Micros spent doing imm_ compactions + + Log(options_.info_log, "Compacting %d@%d + %d@%d files", + compact->compaction->num_input_files(0), compact->compaction->level(), + compact->compaction->num_input_files(1), + compact->compaction->level() + 1); + + assert(versions_->NumLevelFiles(compact->compaction->level()) > 0); + assert(compact->builder == nullptr); + assert(compact->outfile == nullptr); + if (snapshots_.empty()) { + compact->smallest_snapshot = versions_->LastSequence(); + } else { + compact->smallest_snapshot = snapshots_.oldest()->sequence_number(); + } + + Iterator* input = versions_->MakeInputIterator(compact->compaction); + + // Release mutex while we're actually doing the compaction work + mutex_.Unlock(); + + input->SeekToFirst(); + Status status; + ParsedInternalKey ikey; + std::string current_user_key; + bool has_current_user_key = false; + SequenceNumber last_sequence_for_key = kMaxSequenceNumber; + while (input->Valid() && !shutting_down_.load(std::memory_order_acquire)) { + // Prioritize immutable compaction work + if (has_imm_.load(std::memory_order_relaxed)) { + const uint64_t imm_start = env_->NowMicros(); + mutex_.Lock(); + if (imm_ != nullptr) { + CompactMemTable(); + // Wake up MakeRoomForWrite() if necessary. + background_work_finished_signal_.SignalAll(); + } + mutex_.Unlock(); + imm_micros += (env_->NowMicros() - imm_start); + } + + Slice key = input->key(); + if (compact->compaction->ShouldStopBefore(key) && + compact->builder != nullptr) { + status = FinishCompactionOutputFile(compact, input); + if (!status.ok()) { + break; + } + } + + // Handle key/value, add to state, etc. + bool drop = false; + if (!ParseInternalKey(key, &ikey)) { + // Do not hide error keys + current_user_key.clear(); + has_current_user_key = false; + last_sequence_for_key = kMaxSequenceNumber; + } else { + if (!has_current_user_key || + user_comparator()->Compare(ikey.user_key, Slice(current_user_key)) != + 0) { + // First occurrence of this user key + current_user_key.assign(ikey.user_key.data(), ikey.user_key.size()); + has_current_user_key = true; + last_sequence_for_key = kMaxSequenceNumber; + } + + if (last_sequence_for_key <= compact->smallest_snapshot) { + // Hidden by an newer entry for same user key + drop = true; // (A) + } else if (ikey.type == kTypeDeletion && + ikey.sequence <= compact->smallest_snapshot && + compact->compaction->IsBaseLevelForKey(ikey.user_key)) { + // For this user key: + // (1) there is no data in higher levels + // (2) data in lower levels will have larger sequence numbers + // (3) data in layers that are being compacted here and have + // smaller sequence numbers will be dropped in the next + // few iterations of this loop (by rule (A) above). + // Therefore this deletion marker is obsolete and can be dropped. + drop = true; + } + + last_sequence_for_key = ikey.sequence; + } +#if 0 + Log(options_.info_log, + " Compact: %s, seq %d, type: %d %d, drop: %d, is_base: %d, " + "%d smallest_snapshot: %d", + ikey.user_key.ToString().c_str(), + (int)ikey.sequence, ikey.type, kTypeValue, drop, + compact->compaction->IsBaseLevelForKey(ikey.user_key), + (int)last_sequence_for_key, (int)compact->smallest_snapshot); +#endif + + if (!drop) { + // Open output file if necessary + if (compact->builder == nullptr) { + status = OpenCompactionOutputFile(compact); + if (!status.ok()) { + break; + } + } + if (compact->builder->NumEntries() == 0) { + compact->current_output()->smallest.DecodeFrom(key); + } + compact->current_output()->largest.DecodeFrom(key); + compact->builder->Add(key, input->value()); + + // Close output file if it is big enough + if (compact->builder->FileSize() >= + compact->compaction->MaxOutputFileSize()) { + status = FinishCompactionOutputFile(compact, input); + if (!status.ok()) { + break; + } + } + } + + input->Next(); + } + + if (status.ok() && shutting_down_.load(std::memory_order_acquire)) { + status = Status::IOError("Deleting DB during compaction"); + } + if (status.ok() && compact->builder != nullptr) { + status = FinishCompactionOutputFile(compact, input); + } + if (status.ok()) { + status = input->status(); + } + delete input; + input = nullptr; + + CompactionStats stats; + stats.micros = env_->NowMicros() - start_micros - imm_micros; + for (int which = 0; which < 2; which++) { + for (int i = 0; i < compact->compaction->num_input_files(which); i++) { + stats.bytes_read += compact->compaction->input(which, i)->file_size; + } + } + for (size_t i = 0; i < compact->outputs.size(); i++) { + stats.bytes_written += compact->outputs[i].file_size; + } + + mutex_.Lock(); + stats_[compact->compaction->level() + 1].Add(stats); + + if (status.ok()) { + status = InstallCompactionResults(compact); + } + if (!status.ok()) { + RecordBackgroundError(status); + } + VersionSet::LevelSummaryStorage tmp; + Log(options_.info_log, "compacted to: %s", versions_->LevelSummary(&tmp)); + return status; +} + +namespace { + +struct IterState { + port::Mutex* const mu; + Version* const version GUARDED_BY(mu); + MemTable* const mem GUARDED_BY(mu); + MemTable* const imm GUARDED_BY(mu); + + IterState(port::Mutex* mutex, MemTable* mem, MemTable* imm, Version* version) + : mu(mutex), version(version), mem(mem), imm(imm) {} +}; + +static void CleanupIteratorState(void* arg1, void* arg2) { + IterState* state = reinterpret_cast(arg1); + state->mu->Lock(); + state->mem->Unref(); + if (state->imm != nullptr) state->imm->Unref(); + state->version->Unref(); + state->mu->Unlock(); + delete state; +} + +} // anonymous namespace + +Iterator* DBImpl::NewInternalIterator(const ReadOptions& options, + SequenceNumber* latest_snapshot, + uint32_t* seed) { + mutex_.Lock(); + *latest_snapshot = versions_->LastSequence(); + + // Collect together all needed child iterators + std::vector list; + list.push_back(mem_->NewIterator()); + mem_->Ref(); + if (imm_ != nullptr) { + list.push_back(imm_->NewIterator()); + imm_->Ref(); + } + versions_->current()->AddIterators(options, &list); + Iterator* internal_iter = + NewMergingIterator(&internal_comparator_, &list[0], list.size()); + versions_->current()->Ref(); + + IterState* cleanup = new IterState(&mutex_, mem_, imm_, versions_->current()); + internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, nullptr); + + *seed = ++seed_; + mutex_.Unlock(); + return internal_iter; +} + +Iterator* DBImpl::TEST_NewInternalIterator() { + SequenceNumber ignored; + uint32_t ignored_seed; + return NewInternalIterator(ReadOptions(), &ignored, &ignored_seed); +} + +int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() { + MutexLock l(&mutex_); + return versions_->MaxNextLevelOverlappingBytes(); +} + +Status DBImpl::Get(const ReadOptions& options, const Slice& key, + std::string* value) { + Status s; + MutexLock l(&mutex_); + SequenceNumber snapshot; + if (options.snapshot != nullptr) { + snapshot = + static_cast(options.snapshot)->sequence_number(); + } else { + snapshot = versions_->LastSequence(); + } + + MemTable* mem = mem_; + MemTable* imm = imm_; + Version* current = versions_->current(); + mem->Ref(); + if (imm != nullptr) imm->Ref(); + current->Ref(); + + bool have_stat_update = false; + Version::GetStats stats; + + // Unlock while reading from files and memtables + { + mutex_.Unlock(); + // First look in the memtable, then in the immutable memtable (if any). + LookupKey lkey(key, snapshot); + + // blank1.3 + // todo: fill blank1.3 to implement the process of finding the key + // tips1. should consider memtable、immutable memtable and current version + // tips2. the ptr value shoud point to the value + // tips3. how to deal with have_stat_update + // tips4. The operations are all for 'lkey'; see the related functions in file db\memtable.cc & db\version_set.cc + { + // add your code here + } + // blank1.3 ends + + mutex_.Lock(); + } + + if (have_stat_update && current->UpdateStats(stats)) { + MaybeScheduleCompaction(); + } + mem->Unref(); + if (imm != nullptr) imm->Unref(); + current->Unref(); + return s; +} + +Iterator* DBImpl::NewIterator(const ReadOptions& options) { + SequenceNumber latest_snapshot; + uint32_t seed; + Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed); + return NewDBIterator(this, user_comparator(), iter, + (options.snapshot != nullptr + ? static_cast(options.snapshot) + ->sequence_number() + : latest_snapshot), + seed); +} + +void DBImpl::RecordReadSample(Slice key) { + MutexLock l(&mutex_); + if (versions_->current()->RecordReadSample(key)) { + MaybeScheduleCompaction(); + } +} + +const Snapshot* DBImpl::GetSnapshot() { + MutexLock l(&mutex_); + return snapshots_.New(versions_->LastSequence()); +} + +void DBImpl::ReleaseSnapshot(const Snapshot* snapshot) { + MutexLock l(&mutex_); + snapshots_.Delete(static_cast(snapshot)); +} + +// Convenience methods +Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val) { + return DB::Put(o, key, val); +} + +Status DBImpl::Delete(const WriteOptions& options, const Slice& key) { + return DB::Delete(options, key); +} + +Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) { + Writer w(&mutex_); + w.batch = updates; + w.sync = options.sync; + w.done = false; + + MutexLock l(&mutex_); + writers_.push_back(&w); + while (!w.done && &w != writers_.front()) { + w.cv.Wait(); + } + if (w.done) { + return w.status; + } + + // May temporarily unlock and wait. + Status status = MakeRoomForWrite(updates == nullptr); + uint64_t last_sequence = versions_->LastSequence(); + Writer* last_writer = &w; + if (status.ok() && updates != nullptr) { // nullptr batch is for compactions + WriteBatch* write_batch = BuildBatchGroup(&last_writer); + WriteBatchInternal::SetSequence(write_batch, last_sequence + 1); + last_sequence += WriteBatchInternal::Count(write_batch); + + // Add to log and apply to memtable. We can release the lock + // during this phase since &w is currently responsible for logging + // and protects against concurrent loggers and concurrent writes + // into mem_. + { + mutex_.Unlock(); + status = log_->AddRecord(WriteBatchInternal::Contents(write_batch)); + bool sync_error = false; + if (status.ok() && options.sync) { + status = logfile_->Sync(); + if (!status.ok()) { + sync_error = true; + } + } + if (status.ok()) { + status = WriteBatchInternal::InsertInto(write_batch, mem_); + } + mutex_.Lock(); + if (sync_error) { + // The state of the log file is indeterminate: the log record we + // just added may or may not show up when the DB is re-opened. + // So we force the DB into a mode where all future writes fail. + RecordBackgroundError(status); + } + } + if (write_batch == tmp_batch_) tmp_batch_->Clear(); + + versions_->SetLastSequence(last_sequence); + } + + while (true) { + Writer* ready = writers_.front(); + writers_.pop_front(); + if (ready != &w) { + ready->status = status; + ready->done = true; + ready->cv.Signal(); + } + if (ready == last_writer) break; + } + + // Notify new head of write queue + if (!writers_.empty()) { + writers_.front()->cv.Signal(); + } + + return status; +} + +// REQUIRES: Writer list must be non-empty +// REQUIRES: First writer must have a non-null batch +WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) { + mutex_.AssertHeld(); + assert(!writers_.empty()); + Writer* first = writers_.front(); + WriteBatch* result = first->batch; + assert(result != nullptr); + + size_t size = WriteBatchInternal::ByteSize(first->batch); + + // Allow the group to grow up to a maximum size, but if the + // original write is small, limit the growth so we do not slow + // down the small write too much. + size_t max_size = 1 << 20; + if (size <= (128 << 10)) { + max_size = size + (128 << 10); + } + + *last_writer = first; + std::deque::iterator iter = writers_.begin(); + ++iter; // Advance past "first" + for (; iter != writers_.end(); ++iter) { + Writer* w = *iter; + if (w->sync && !first->sync) { + // Do not include a sync write into a batch handled by a non-sync write. + break; + } + + if (w->batch != nullptr) { + size += WriteBatchInternal::ByteSize(w->batch); + if (size > max_size) { + // Do not make batch too big + break; + } + + // Append to *result + if (result == first->batch) { + // Switch to temporary batch instead of disturbing caller's batch + result = tmp_batch_; + assert(WriteBatchInternal::Count(result) == 0); + WriteBatchInternal::Append(result, first->batch); + } + WriteBatchInternal::Append(result, w->batch); + } + *last_writer = w; + } + return result; +} + +// REQUIRES: mutex_ is held +// REQUIRES: this thread is currently at the front of the writer queue +Status DBImpl::MakeRoomForWrite(bool force) { + mutex_.AssertHeld(); + assert(!writers_.empty()); + bool allow_delay = !force; + Status s; + while (true) { + if (!bg_error_.ok()) { + // Yield previous error + s = bg_error_; + break; + } else if (allow_delay && versions_->NumLevelFiles(0) >= + config::kL0_SlowdownWritesTrigger) { + // We are getting close to hitting a hard limit on the number of + // L0 files. Rather than delaying a single write by several + // seconds when we hit the hard limit, start delaying each + // individual write by 1ms to reduce latency variance. Also, + // this delay hands over some CPU to the compaction thread in + // case it is sharing the same core as the writer. + mutex_.Unlock(); + env_->SleepForMicroseconds(1000); + allow_delay = false; // Do not delay a single write more than once + mutex_.Lock(); + } else if (!force && + (mem_->ApproximateMemoryUsage() <= options_.write_buffer_size)) { + // There is room in current memtable + break; + } else if (imm_ != nullptr) { + // We have filled up the current memtable, but the previous + // one is still being compacted, so we wait. + Log(options_.info_log, "Current memtable full; waiting...\n"); + background_work_finished_signal_.Wait(); + } else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) { + // There are too many level-0 files. + Log(options_.info_log, "Too many L0 files; waiting...\n"); + background_work_finished_signal_.Wait(); + } else { + // Attempt to switch to a new memtable and trigger compaction of old + assert(versions_->PrevLogNumber() == 0); + uint64_t new_log_number = versions_->NewFileNumber(); + WritableFile* lfile = nullptr; + s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile); + if (!s.ok()) { + // Avoid chewing through file number space in a tight loop. + versions_->ReuseFileNumber(new_log_number); + break; + } + + delete log_; + + s = logfile_->Close(); + if (!s.ok()) { + // We may have lost some data written to the previous log file. + // Switch to the new log file anyway, but record as a background + // error so we do not attempt any more writes. + // + // We could perhaps attempt to save the memtable corresponding + // to log file and suppress the error if that works, but that + // would add more complexity in a critical code path. + RecordBackgroundError(s); + } + delete logfile_; + + logfile_ = lfile; + logfile_number_ = new_log_number; + log_ = new log::Writer(lfile); + imm_ = mem_; + has_imm_.store(true, std::memory_order_release); + mem_ = new MemTable(internal_comparator_); + mem_->Ref(); + force = false; // Do not force another compaction if have room + MaybeScheduleCompaction(); + } + } + return s; +} + +bool DBImpl::GetProperty(const Slice& property, std::string* value) { + value->clear(); + + MutexLock l(&mutex_); + Slice in = property; + Slice prefix("leveldb."); + if (!in.starts_with(prefix)) return false; + in.remove_prefix(prefix.size()); + + if (in.starts_with("num-files-at-level")) { + in.remove_prefix(strlen("num-files-at-level")); + uint64_t level; + bool ok = ConsumeDecimalNumber(&in, &level) && in.empty(); + if (!ok || level >= config::kNumLevels) { + return false; + } else { + char buf[100]; + std::snprintf(buf, sizeof(buf), "%d", + versions_->NumLevelFiles(static_cast(level))); + *value = buf; + return true; + } + } else if (in == "stats") { + char buf[200]; + std::snprintf(buf, sizeof(buf), + " Compactions\n" + "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n" + "--------------------------------------------------\n"); + value->append(buf); + for (int level = 0; level < config::kNumLevels; level++) { + int files = versions_->NumLevelFiles(level); + if (stats_[level].micros > 0 || files > 0) { + std::snprintf(buf, sizeof(buf), "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", + level, files, versions_->NumLevelBytes(level) / 1048576.0, + stats_[level].micros / 1e6, + stats_[level].bytes_read / 1048576.0, + stats_[level].bytes_written / 1048576.0); + value->append(buf); + } + } + return true; + } else if (in == "sstables") { + *value = versions_->current()->DebugString(); + return true; + } else if (in == "approximate-memory-usage") { + size_t total_usage = options_.block_cache->TotalCharge(); + if (mem_) { + total_usage += mem_->ApproximateMemoryUsage(); + } + if (imm_) { + total_usage += imm_->ApproximateMemoryUsage(); + } + char buf[50]; + std::snprintf(buf, sizeof(buf), "%llu", + static_cast(total_usage)); + value->append(buf); + return true; + } + + return false; +} + +void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) { + // TODO(opt): better implementation + MutexLock l(&mutex_); + Version* v = versions_->current(); + v->Ref(); + + for (int i = 0; i < n; i++) { + // Convert user_key into a corresponding internal key. + InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek); + uint64_t start = versions_->ApproximateOffsetOf(v, k1); + uint64_t limit = versions_->ApproximateOffsetOf(v, k2); + sizes[i] = (limit >= start ? limit - start : 0); + } + + v->Unref(); +} + +// Default implementations of convenience methods that subclasses of DB +// can call if they wish +Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) { + WriteBatch batch; + batch.Put(key, value); + return Write(opt, &batch); +} + +Status DB::Delete(const WriteOptions& opt, const Slice& key) { + WriteBatch batch; + batch.Delete(key); + return Write(opt, &batch); +} + +DB::~DB() = default; + +Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) { + *dbptr = nullptr; + + DBImpl* impl = new DBImpl(options, dbname); + impl->mutex_.Lock(); + VersionEdit edit; + // Recover handles create_if_missing, error_if_exists + bool save_manifest = false; + Status s = impl->Recover(&edit, &save_manifest); + if (s.ok() && impl->mem_ == nullptr) { + // Create new log and a corresponding memtable. + uint64_t new_log_number = impl->versions_->NewFileNumber(); + WritableFile* lfile; + s = options.env->NewWritableFile(LogFileName(dbname, new_log_number), + &lfile); + if (s.ok()) { + edit.SetLogNumber(new_log_number); + impl->logfile_ = lfile; + impl->logfile_number_ = new_log_number; + impl->log_ = new log::Writer(lfile); + impl->mem_ = new MemTable(impl->internal_comparator_); + impl->mem_->Ref(); + } + } + if (s.ok() && save_manifest) { + edit.SetPrevLogNumber(0); // No older logs needed after recovery. + edit.SetLogNumber(impl->logfile_number_); + s = impl->versions_->LogAndApply(&edit, &impl->mutex_); + } + if (s.ok()) { + impl->RemoveObsoleteFiles(); + impl->MaybeScheduleCompaction(); + } + impl->mutex_.Unlock(); + if (s.ok()) { + assert(impl->mem_ != nullptr); + *dbptr = impl; + } else { + delete impl; + } + return s; +} + +Snapshot::~Snapshot() = default; + +Status DestroyDB(const std::string& dbname, const Options& options) { + Env* env = options.env; + std::vector filenames; + Status result = env->GetChildren(dbname, &filenames); + if (!result.ok()) { + // Ignore error in case directory does not exist + return Status::OK(); + } + + FileLock* lock; + const std::string lockname = LockFileName(dbname); + result = env->LockFile(lockname, &lock); + if (result.ok()) { + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && + type != kDBLockFile) { // Lock file will be deleted at end + Status del = env->RemoveFile(dbname + "/" + filenames[i]); + if (result.ok() && !del.ok()) { + result = del; + } + } + } + env->UnlockFile(lock); // Ignore error since state is already gone + env->RemoveFile(lockname); + env->RemoveDir(dbname); // Ignore error in case dir contains other files + } + return result; +} + +} // namespace leveldb diff --git a/leveldb/db/db_impl.h b/leveldb/db/db_impl.h new file mode 100644 index 000000000..c7b01721b --- /dev/null +++ b/leveldb/db/db_impl.h @@ -0,0 +1,217 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_ +#define STORAGE_LEVELDB_DB_DB_IMPL_H_ + +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "db/log_writer.h" +#include "db/snapshot.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "port/port.h" +#include "port/thread_annotations.h" + +namespace leveldb { + +class MemTable; +class TableCache; +class Version; +class VersionEdit; +class VersionSet; + +class DBImpl : public DB { + public: + DBImpl(const Options& options, const std::string& dbname); + + DBImpl(const DBImpl&) = delete; + DBImpl& operator=(const DBImpl&) = delete; + + ~DBImpl() override; + + // Implementations of the DB interface + Status Put(const WriteOptions&, const Slice& key, + const Slice& value) override; + Status Delete(const WriteOptions&, const Slice& key) override; + Status Write(const WriteOptions& options, WriteBatch* updates) override; + Status Get(const ReadOptions& options, const Slice& key, + std::string* value) override; + Iterator* NewIterator(const ReadOptions&) override; + const Snapshot* GetSnapshot() override; + void ReleaseSnapshot(const Snapshot* snapshot) override; + bool GetProperty(const Slice& property, std::string* value) override; + void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) override; + void CompactRange(const Slice* begin, const Slice* end) override; + + // Extra methods (for testing) that are not in the public DB interface + + // Compact any files in the named level that overlap [*begin,*end] + void TEST_CompactRange(int level, const Slice* begin, const Slice* end); + + // Force current memtable contents to be compacted. + Status TEST_CompactMemTable(); + + // Return an internal iterator over the current state of the database. + // The keys of this iterator are internal keys (see format.h). + // The returned iterator should be deleted when no longer needed. + Iterator* TEST_NewInternalIterator(); + + // Return the maximum overlapping data (in bytes) at next level for any + // file at a level >= 1. + int64_t TEST_MaxNextLevelOverlappingBytes(); + + // Record a sample of bytes read at the specified internal key. + // Samples are taken approximately once every config::kReadBytesPeriod + // bytes. + void RecordReadSample(Slice key); + + private: + friend class DB; + struct CompactionState; + struct Writer; + + // Information for a manual compaction + struct ManualCompaction { + int level; + bool done; + const InternalKey* begin; // null means beginning of key range + const InternalKey* end; // null means end of key range + InternalKey tmp_storage; // Used to keep track of compaction progress + }; + + // Per level compaction stats. stats_[level] stores the stats for + // compactions that produced data for the specified "level". + struct CompactionStats { + CompactionStats() : micros(0), bytes_read(0), bytes_written(0) {} + + void Add(const CompactionStats& c) { + this->micros += c.micros; + this->bytes_read += c.bytes_read; + this->bytes_written += c.bytes_written; + } + + int64_t micros; + int64_t bytes_read; + int64_t bytes_written; + }; + + Iterator* NewInternalIterator(const ReadOptions&, + SequenceNumber* latest_snapshot, + uint32_t* seed); + + Status NewDB(); + + // Recover the descriptor from persistent storage. May do a significant + // amount of work to recover recently logged updates. Any changes to + // be made to the descriptor are added to *edit. + Status Recover(VersionEdit* edit, bool* save_manifest) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + void MaybeIgnoreError(Status* s) const; + + // Delete any unneeded files and stale in-memory entries. + void RemoveObsoleteFiles() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Compact the in-memory write buffer to disk. Switches to a new + // log-file/memtable and writes a new descriptor iff successful. + // Errors are recorded in bg_error_. + void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + Status RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest, + VersionEdit* edit, SequenceNumber* max_sequence) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + Status MakeRoomForWrite(bool force /* compact even if there is room? */) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + WriteBatch* BuildBatchGroup(Writer** last_writer) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + void RecordBackgroundError(const Status& s); + + void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + static void BGWork(void* db); + void BackgroundCall(); + void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void CleanupCompaction(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Status DoCompactionWork(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + Status OpenCompactionOutputFile(CompactionState* compact); + Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input); + Status InstallCompactionResults(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + const Comparator* user_comparator() const { + return internal_comparator_.user_comparator(); + } + + // Constant after construction + Env* const env_; + const InternalKeyComparator internal_comparator_; + const InternalFilterPolicy internal_filter_policy_; + const Options options_; // options_.comparator == &internal_comparator_ + const bool owns_info_log_; + const bool owns_cache_; + const std::string dbname_; + + // table_cache_ provides its own synchronization + TableCache* const table_cache_; + + // Lock over the persistent DB state. Non-null iff successfully acquired. + FileLock* db_lock_; + + // State below is protected by mutex_ + port::Mutex mutex_; + std::atomic shutting_down_; + port::CondVar background_work_finished_signal_ GUARDED_BY(mutex_); + MemTable* mem_; + MemTable* imm_ GUARDED_BY(mutex_); // Memtable being compacted + std::atomic has_imm_; // So bg thread can detect non-null imm_ + WritableFile* logfile_; + uint64_t logfile_number_ GUARDED_BY(mutex_); + log::Writer* log_; + uint32_t seed_ GUARDED_BY(mutex_); // For sampling. + + // Queue of writers. + std::deque writers_ GUARDED_BY(mutex_); + WriteBatch* tmp_batch_ GUARDED_BY(mutex_); + + SnapshotList snapshots_ GUARDED_BY(mutex_); + + // Set of table files to protect from deletion because they are + // part of ongoing compactions. + std::set pending_outputs_ GUARDED_BY(mutex_); + + // Has a background compaction been scheduled or is running? + bool background_compaction_scheduled_ GUARDED_BY(mutex_); + + ManualCompaction* manual_compaction_ GUARDED_BY(mutex_); + + VersionSet* const versions_ GUARDED_BY(mutex_); + + // Have we encountered a background error in paranoid mode? + Status bg_error_ GUARDED_BY(mutex_); + + CompactionStats stats_[config::kNumLevels] GUARDED_BY(mutex_); +}; + +// Sanitize db options. The caller should delete result.info_log if +// it is not equal to src.info_log. +Options SanitizeOptions(const std::string& db, + const InternalKeyComparator* icmp, + const InternalFilterPolicy* ipolicy, + const Options& src); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_DB_IMPL_H_ diff --git a/leveldb/db/db_iter.cc b/leveldb/db/db_iter.cc new file mode 100644 index 000000000..532c2db81 --- /dev/null +++ b/leveldb/db/db_iter.cc @@ -0,0 +1,318 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_iter.h" + +#include "db/db_impl.h" +#include "db/dbformat.h" +#include "db/filename.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "port/port.h" +#include "util/logging.h" +#include "util/mutexlock.h" +#include "util/random.h" + +namespace leveldb { + +#if 0 +static void DumpInternalIter(Iterator* iter) { + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey k; + if (!ParseInternalKey(iter->key(), &k)) { + std::fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str()); + } else { + std::fprintf(stderr, "@ '%s'\n", k.DebugString().c_str()); + } + } +} +#endif + +namespace { + +// Memtables and sstables that make the DB representation contain +// (userkey,seq,type) => uservalue entries. DBIter +// combines multiple entries for the same userkey found in the DB +// representation into a single entry while accounting for sequence +// numbers, deletion markers, overwrites, etc. +class DBIter : public Iterator { + public: + // Which direction is the iterator currently moving? + // (1) When moving forward, the internal iterator is positioned at + // the exact entry that yields this->key(), this->value() + // (2) When moving backwards, the internal iterator is positioned + // just before all entries whose user key == this->key(). + enum Direction { kForward, kReverse }; + + DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s, + uint32_t seed) + : db_(db), + user_comparator_(cmp), + iter_(iter), + sequence_(s), + direction_(kForward), + valid_(false), + rnd_(seed), + bytes_until_read_sampling_(RandomCompactionPeriod()) {} + + DBIter(const DBIter&) = delete; + DBIter& operator=(const DBIter&) = delete; + + ~DBIter() override { delete iter_; } + bool Valid() const override { return valid_; } + Slice key() const override { + assert(valid_); + return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_; + } + Slice value() const override { + assert(valid_); + return (direction_ == kForward) ? iter_->value() : saved_value_; + } + Status status() const override { + if (status_.ok()) { + return iter_->status(); + } else { + return status_; + } + } + + void Next() override; + void Prev() override; + void Seek(const Slice& target) override; + void SeekToFirst() override; + void SeekToLast() override; + + private: + void FindNextUserEntry(bool skipping, std::string* skip); + void FindPrevUserEntry(); + bool ParseKey(ParsedInternalKey* key); + + inline void SaveKey(const Slice& k, std::string* dst) { + dst->assign(k.data(), k.size()); + } + + inline void ClearSavedValue() { + if (saved_value_.capacity() > 1048576) { + std::string empty; + swap(empty, saved_value_); + } else { + saved_value_.clear(); + } + } + + // Picks the number of bytes that can be read until a compaction is scheduled. + size_t RandomCompactionPeriod() { + return rnd_.Uniform(2 * config::kReadBytesPeriod); + } + + DBImpl* db_; + const Comparator* const user_comparator_; + Iterator* const iter_; + SequenceNumber const sequence_; + Status status_; + std::string saved_key_; // == current key when direction_==kReverse + std::string saved_value_; // == current raw value when direction_==kReverse + Direction direction_; + bool valid_; + Random rnd_; + size_t bytes_until_read_sampling_; +}; + +inline bool DBIter::ParseKey(ParsedInternalKey* ikey) { + Slice k = iter_->key(); + + size_t bytes_read = k.size() + iter_->value().size(); + while (bytes_until_read_sampling_ < bytes_read) { + bytes_until_read_sampling_ += RandomCompactionPeriod(); + db_->RecordReadSample(k); + } + assert(bytes_until_read_sampling_ >= bytes_read); + bytes_until_read_sampling_ -= bytes_read; + + if (!ParseInternalKey(k, ikey)) { + status_ = Status::Corruption("corrupted internal key in DBIter"); + return false; + } else { + return true; + } +} + +void DBIter::Next() { + assert(valid_); + + if (direction_ == kReverse) { // Switch directions? + direction_ = kForward; + // iter_ is pointing just before the entries for this->key(), + // so advance into the range of entries for this->key() and then + // use the normal skipping code below. + if (!iter_->Valid()) { + iter_->SeekToFirst(); + } else { + iter_->Next(); + } + if (!iter_->Valid()) { + valid_ = false; + saved_key_.clear(); + return; + } + // saved_key_ already contains the key to skip past. + } else { + // Store in saved_key_ the current key so we skip it below. + SaveKey(ExtractUserKey(iter_->key()), &saved_key_); + + // iter_ is pointing to current key. We can now safely move to the next to + // avoid checking current key. + iter_->Next(); + if (!iter_->Valid()) { + valid_ = false; + saved_key_.clear(); + return; + } + } + + FindNextUserEntry(true, &saved_key_); +} + +void DBIter::FindNextUserEntry(bool skipping, std::string* skip) { + // Loop until we hit an acceptable entry to yield + assert(iter_->Valid()); + assert(direction_ == kForward); + do { + ParsedInternalKey ikey; + if (ParseKey(&ikey) && ikey.sequence <= sequence_) { + switch (ikey.type) { + case kTypeDeletion: + // Arrange to skip all upcoming entries for this key since + // they are hidden by this deletion. + SaveKey(ikey.user_key, skip); + skipping = true; + break; + case kTypeValue: + if (skipping && + user_comparator_->Compare(ikey.user_key, *skip) <= 0) { + // Entry hidden + } else { + valid_ = true; + saved_key_.clear(); + return; + } + break; + } + } + iter_->Next(); + } while (iter_->Valid()); + saved_key_.clear(); + valid_ = false; +} + +void DBIter::Prev() { + assert(valid_); + + if (direction_ == kForward) { // Switch directions? + // iter_ is pointing at the current entry. Scan backwards until + // the key changes so we can use the normal reverse scanning code. + assert(iter_->Valid()); // Otherwise valid_ would have been false + SaveKey(ExtractUserKey(iter_->key()), &saved_key_); + while (true) { + iter_->Prev(); + if (!iter_->Valid()) { + valid_ = false; + saved_key_.clear(); + ClearSavedValue(); + return; + } + if (user_comparator_->Compare(ExtractUserKey(iter_->key()), saved_key_) < + 0) { + break; + } + } + direction_ = kReverse; + } + + FindPrevUserEntry(); +} + +void DBIter::FindPrevUserEntry() { + assert(direction_ == kReverse); + + ValueType value_type = kTypeDeletion; + if (iter_->Valid()) { + do { + ParsedInternalKey ikey; + if (ParseKey(&ikey) && ikey.sequence <= sequence_) { + if ((value_type != kTypeDeletion) && + user_comparator_->Compare(ikey.user_key, saved_key_) < 0) { + // We encountered a non-deleted value in entries for previous keys, + break; + } + value_type = ikey.type; + if (value_type == kTypeDeletion) { + saved_key_.clear(); + ClearSavedValue(); + } else { + Slice raw_value = iter_->value(); + if (saved_value_.capacity() > raw_value.size() + 1048576) { + std::string empty; + swap(empty, saved_value_); + } + SaveKey(ExtractUserKey(iter_->key()), &saved_key_); + saved_value_.assign(raw_value.data(), raw_value.size()); + } + } + iter_->Prev(); + } while (iter_->Valid()); + } + + if (value_type == kTypeDeletion) { + // End + valid_ = false; + saved_key_.clear(); + ClearSavedValue(); + direction_ = kForward; + } else { + valid_ = true; + } +} + +void DBIter::Seek(const Slice& target) { + direction_ = kForward; + ClearSavedValue(); + saved_key_.clear(); + AppendInternalKey(&saved_key_, + ParsedInternalKey(target, sequence_, kValueTypeForSeek)); + iter_->Seek(saved_key_); + if (iter_->Valid()) { + FindNextUserEntry(false, &saved_key_ /* temporary storage */); + } else { + valid_ = false; + } +} + +void DBIter::SeekToFirst() { + direction_ = kForward; + ClearSavedValue(); + iter_->SeekToFirst(); + if (iter_->Valid()) { + FindNextUserEntry(false, &saved_key_ /* temporary storage */); + } else { + valid_ = false; + } +} + +void DBIter::SeekToLast() { + direction_ = kReverse; + ClearSavedValue(); + iter_->SeekToLast(); + FindPrevUserEntry(); +} + +} // anonymous namespace + +Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator, + Iterator* internal_iter, SequenceNumber sequence, + uint32_t seed) { + return new DBIter(db, user_key_comparator, internal_iter, sequence, seed); +} + +} // namespace leveldb diff --git a/leveldb/db/db_iter.h b/leveldb/db/db_iter.h new file mode 100644 index 000000000..5977fc893 --- /dev/null +++ b/leveldb/db/db_iter.h @@ -0,0 +1,26 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_DB_ITER_H_ +#define STORAGE_LEVELDB_DB_DB_ITER_H_ + +#include + +#include "db/dbformat.h" +#include "leveldb/db.h" + +namespace leveldb { + +class DBImpl; + +// Return a new iterator that converts internal keys (yielded by +// "*internal_iter") that were live at the specified "sequence" number +// into appropriate user keys. +Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator, + Iterator* internal_iter, SequenceNumber sequence, + uint32_t seed); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_DB_ITER_H_ diff --git a/leveldb/db/db_test.cc b/leveldb/db/db_test.cc new file mode 100644 index 000000000..a4a84cd64 --- /dev/null +++ b/leveldb/db/db_test.cc @@ -0,0 +1,2360 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/db.h" + +#include +#include +#include + +#include "gtest/gtest.h" +#include "db/db_impl.h" +#include "db/filename.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "leveldb/cache.h" +#include "leveldb/env.h" +#include "leveldb/filter_policy.h" +#include "leveldb/table.h" +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/hash.h" +#include "util/logging.h" +#include "util/mutexlock.h" +#include "util/testutil.h" + +namespace leveldb { + +static std::string RandomString(Random* rnd, int len) { + std::string r; + test::RandomString(rnd, len, &r); + return r; +} + +static std::string RandomKey(Random* rnd) { + int len = + (rnd->OneIn(3) ? 1 // Short sometimes to encourage collisions + : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10))); + return test::RandomKey(rnd, len); +} + +namespace { +class AtomicCounter { + public: + AtomicCounter() : count_(0) {} + void Increment() { IncrementBy(1); } + void IncrementBy(int count) LOCKS_EXCLUDED(mu_) { + MutexLock l(&mu_); + count_ += count; + } + int Read() LOCKS_EXCLUDED(mu_) { + MutexLock l(&mu_); + return count_; + } + void Reset() LOCKS_EXCLUDED(mu_) { + MutexLock l(&mu_); + count_ = 0; + } + + private: + port::Mutex mu_; + int count_ GUARDED_BY(mu_); +}; + +void DelayMilliseconds(int millis) { + Env::Default()->SleepForMicroseconds(millis * 1000); +} + +bool IsLdbFile(const std::string& f) { + return strstr(f.c_str(), ".ldb") != nullptr; +} + +bool IsLogFile(const std::string& f) { + return strstr(f.c_str(), ".log") != nullptr; +} + +bool IsManifestFile(const std::string& f) { + return strstr(f.c_str(), "MANIFEST") != nullptr; +} + +} // namespace + +// Test Env to override default Env behavior for testing. +class TestEnv : public EnvWrapper { + public: + explicit TestEnv(Env* base) : EnvWrapper(base), ignore_dot_files_(false) {} + + void SetIgnoreDotFiles(bool ignored) { ignore_dot_files_ = ignored; } + + Status GetChildren(const std::string& dir, + std::vector* result) override { + Status s = target()->GetChildren(dir, result); + if (!s.ok() || !ignore_dot_files_) { + return s; + } + + std::vector::iterator it = result->begin(); + while (it != result->end()) { + if ((*it == ".") || (*it == "..")) { + it = result->erase(it); + } else { + ++it; + } + } + + return s; + } + + private: + bool ignore_dot_files_; +}; + +// Special Env used to delay background operations. +class SpecialEnv : public EnvWrapper { + public: + // For historical reasons, the std::atomic<> fields below are currently + // accessed via acquired loads and release stores. We should switch + // to plain load(), store() calls that provide sequential consistency. + + // sstable/log Sync() calls are blocked while this pointer is non-null. + std::atomic delay_data_sync_; + + // sstable/log Sync() calls return an error. + std::atomic data_sync_error_; + + // Simulate no-space errors while this pointer is non-null. + std::atomic no_space_; + + // Simulate non-writable file system while this pointer is non-null. + std::atomic non_writable_; + + // Force sync of manifest files to fail while this pointer is non-null. + std::atomic manifest_sync_error_; + + // Force write to manifest files to fail while this pointer is non-null. + std::atomic manifest_write_error_; + + // Force log file close to fail while this bool is true. + std::atomic log_file_close_; + + bool count_random_reads_; + AtomicCounter random_read_counter_; + + explicit SpecialEnv(Env* base) + : EnvWrapper(base), + delay_data_sync_(false), + data_sync_error_(false), + no_space_(false), + non_writable_(false), + manifest_sync_error_(false), + manifest_write_error_(false), + log_file_close_(false), + count_random_reads_(false) {} + + Status NewWritableFile(const std::string& f, WritableFile** r) { + class DataFile : public WritableFile { + private: + SpecialEnv* const env_; + WritableFile* const base_; + const std::string fname_; + + public: + DataFile(SpecialEnv* env, WritableFile* base, const std::string& fname) + : env_(env), base_(base), fname_(fname) {} + + ~DataFile() { delete base_; } + Status Append(const Slice& data) { + if (env_->no_space_.load(std::memory_order_acquire)) { + // Drop writes on the floor + return Status::OK(); + } else { + return base_->Append(data); + } + } + Status Close() { + Status s = base_->Close(); + if (s.ok() && IsLogFile(fname_) && + env_->log_file_close_.load(std::memory_order_acquire)) { + s = Status::IOError("simulated log file Close error"); + } + return s; + } + Status Flush() { return base_->Flush(); } + Status Sync() { + if (env_->data_sync_error_.load(std::memory_order_acquire)) { + return Status::IOError("simulated data sync error"); + } + while (env_->delay_data_sync_.load(std::memory_order_acquire)) { + DelayMilliseconds(100); + } + return base_->Sync(); + } + }; + class ManifestFile : public WritableFile { + private: + SpecialEnv* env_; + WritableFile* base_; + + public: + ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) {} + ~ManifestFile() { delete base_; } + Status Append(const Slice& data) { + if (env_->manifest_write_error_.load(std::memory_order_acquire)) { + return Status::IOError("simulated writer error"); + } else { + return base_->Append(data); + } + } + Status Close() { return base_->Close(); } + Status Flush() { return base_->Flush(); } + Status Sync() { + if (env_->manifest_sync_error_.load(std::memory_order_acquire)) { + return Status::IOError("simulated sync error"); + } else { + return base_->Sync(); + } + } + }; + + if (non_writable_.load(std::memory_order_acquire)) { + return Status::IOError("simulated write error"); + } + + Status s = target()->NewWritableFile(f, r); + if (s.ok()) { + if (IsLdbFile(f) || IsLogFile(f)) { + *r = new DataFile(this, *r, f); + } else if (IsManifestFile(f)) { + *r = new ManifestFile(this, *r); + } + } + return s; + } + + Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) { + class CountingFile : public RandomAccessFile { + private: + RandomAccessFile* target_; + AtomicCounter* counter_; + + public: + CountingFile(RandomAccessFile* target, AtomicCounter* counter) + : target_(target), counter_(counter) {} + ~CountingFile() override { delete target_; } + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + counter_->Increment(); + return target_->Read(offset, n, result, scratch); + } + }; + + Status s = target()->NewRandomAccessFile(f, r); + if (s.ok() && count_random_reads_) { + *r = new CountingFile(*r, &random_read_counter_); + } + return s; + } +}; + +class DBTest : public testing::Test { + public: + std::string dbname_; + SpecialEnv* env_; + DB* db_; + + Options last_options_; + + DBTest() : env_(new SpecialEnv(Env::Default())), option_config_(kDefault) { + filter_policy_ = NewBloomFilterPolicy(10); + dbname_ = testing::TempDir() + "db_test"; + DestroyDB(dbname_, Options()); + db_ = nullptr; + Reopen(); + } + + ~DBTest() { + delete db_; + DestroyDB(dbname_, Options()); + delete env_; + delete filter_policy_; + } + + // Switch to a fresh database with the next option configuration to + // test. Return false if there are no more configurations to test. + bool ChangeOptions() { + option_config_++; + if (option_config_ >= kEnd) { + return false; + } else { + DestroyAndReopen(); + return true; + } + } + + // Return the current option configuration. + Options CurrentOptions() { + Options options; + options.reuse_logs = false; + switch (option_config_) { + case kReuse: + options.reuse_logs = true; + break; + case kFilter: + options.filter_policy = filter_policy_; + break; + case kUncompressed: + options.compression = kNoCompression; + break; + default: + break; + } + return options; + } + + DBImpl* dbfull() { return reinterpret_cast(db_); } + + void Reopen(Options* options = nullptr) { + ASSERT_LEVELDB_OK(TryReopen(options)); + } + + void Close() { + delete db_; + db_ = nullptr; + } + + void DestroyAndReopen(Options* options = nullptr) { + delete db_; + db_ = nullptr; + DestroyDB(dbname_, Options()); + ASSERT_LEVELDB_OK(TryReopen(options)); + } + + Status TryReopen(Options* options) { + delete db_; + db_ = nullptr; + Options opts; + if (options != nullptr) { + opts = *options; + } else { + opts = CurrentOptions(); + opts.create_if_missing = true; + } + last_options_ = opts; + + return DB::Open(opts, dbname_, &db_); + } + + Status Put(const std::string& k, const std::string& v) { + return db_->Put(WriteOptions(), k, v); + } + + Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); } + + std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { + ReadOptions options; + options.snapshot = snapshot; + std::string result; + Status s = db_->Get(options, k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + // Return a string that contains all key,value pairs in order, + // formatted like "(k1->v1)(k2->v2)". + std::string Contents() { + std::vector forward; + std::string result; + Iterator* iter = db_->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string s = IterStatus(iter); + result.push_back('('); + result.append(s); + result.push_back(')'); + forward.push_back(s); + } + + // Check reverse iteration results are the reverse of forward results + size_t matched = 0; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + EXPECT_LT(matched, forward.size()); + EXPECT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]); + matched++; + } + EXPECT_EQ(matched, forward.size()); + + delete iter; + return result; + } + + std::string AllEntriesFor(const Slice& user_key) { + Iterator* iter = dbfull()->TEST_NewInternalIterator(); + InternalKey target(user_key, kMaxSequenceNumber, kTypeValue); + iter->Seek(target.Encode()); + std::string result; + if (!iter->status().ok()) { + result = iter->status().ToString(); + } else { + result = "[ "; + bool first = true; + while (iter->Valid()) { + ParsedInternalKey ikey; + if (!ParseInternalKey(iter->key(), &ikey)) { + result += "CORRUPTED"; + } else { + if (last_options_.comparator->Compare(ikey.user_key, user_key) != 0) { + break; + } + if (!first) { + result += ", "; + } + first = false; + switch (ikey.type) { + case kTypeValue: + result += iter->value().ToString(); + break; + case kTypeDeletion: + result += "DEL"; + break; + } + } + iter->Next(); + } + if (!first) { + result += " "; + } + result += "]"; + } + delete iter; + return result; + } + + int NumTableFilesAtLevel(int level) { + std::string property; + EXPECT_TRUE(db_->GetProperty( + "leveldb.num-files-at-level" + NumberToString(level), &property)); + return std::stoi(property); + } + + int TotalTableFiles() { + int result = 0; + for (int level = 0; level < config::kNumLevels; level++) { + result += NumTableFilesAtLevel(level); + } + return result; + } + + // Return spread of files per level + std::string FilesPerLevel() { + std::string result; + int last_non_zero_offset = 0; + for (int level = 0; level < config::kNumLevels; level++) { + int f = NumTableFilesAtLevel(level); + char buf[100]; + std::snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); + result += buf; + if (f > 0) { + last_non_zero_offset = result.size(); + } + } + result.resize(last_non_zero_offset); + return result; + } + + int CountFiles() { + std::vector files; + env_->GetChildren(dbname_, &files); + return static_cast(files.size()); + } + + uint64_t Size(const Slice& start, const Slice& limit) { + Range r(start, limit); + uint64_t size; + db_->GetApproximateSizes(&r, 1, &size); + return size; + } + + void Compact(const Slice& start, const Slice& limit) { + db_->CompactRange(&start, &limit); + } + + // Do n memtable compactions, each of which produces an sstable + // covering the range [small_key,large_key]. + void MakeTables(int n, const std::string& small_key, + const std::string& large_key) { + for (int i = 0; i < n; i++) { + Put(small_key, "begin"); + Put(large_key, "end"); + dbfull()->TEST_CompactMemTable(); + } + } + + // Prevent pushing of new sstables into deeper levels by adding + // tables that cover a specified range to all levels. + void FillLevels(const std::string& smallest, const std::string& largest) { + MakeTables(config::kNumLevels, smallest, largest); + } + + void DumpFileCounts(const char* label) { + std::fprintf(stderr, "---\n%s:\n", label); + std::fprintf( + stderr, "maxoverlap: %lld\n", + static_cast(dbfull()->TEST_MaxNextLevelOverlappingBytes())); + for (int level = 0; level < config::kNumLevels; level++) { + int num = NumTableFilesAtLevel(level); + if (num > 0) { + std::fprintf(stderr, " level %3d : %d files\n", level, num); + } + } + } + + std::string DumpSSTableList() { + std::string property; + db_->GetProperty("leveldb.sstables", &property); + return property; + } + + std::string IterStatus(Iterator* iter) { + std::string result; + if (iter->Valid()) { + result = iter->key().ToString() + "->" + iter->value().ToString(); + } else { + result = "(invalid)"; + } + return result; + } + + bool DeleteAnSSTFile() { + std::vector filenames; + EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) { + EXPECT_LEVELDB_OK(env_->RemoveFile(TableFileName(dbname_, number))); + return true; + } + } + return false; + } + + // Returns number of files renamed. + int RenameLDBToSST() { + std::vector filenames; + EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + int files_renamed = 0; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) { + const std::string from = TableFileName(dbname_, number); + const std::string to = SSTTableFileName(dbname_, number); + EXPECT_LEVELDB_OK(env_->RenameFile(from, to)); + files_renamed++; + } + } + return files_renamed; + } + + private: + // Sequence of option configurations to try + enum OptionConfig { kDefault, kReuse, kFilter, kUncompressed, kEnd }; + + const FilterPolicy* filter_policy_; + int option_config_; +}; + +TEST_F(DBTest, Empty) { + do { + ASSERT_TRUE(db_ != nullptr); + ASSERT_EQ("NOT_FOUND", Get("foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, EmptyKey) { + do { + ASSERT_LEVELDB_OK(Put("", "v1")); + ASSERT_EQ("v1", Get("")); + ASSERT_LEVELDB_OK(Put("", "v2")); + ASSERT_EQ("v2", Get("")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, EmptyValue) { + do { + ASSERT_LEVELDB_OK(Put("key", "v1")); + ASSERT_EQ("v1", Get("key")); + ASSERT_LEVELDB_OK(Put("key", "")); + ASSERT_EQ("", Get("key")); + ASSERT_LEVELDB_OK(Put("key", "v2")); + ASSERT_EQ("v2", Get("key")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, ReadWrite) { + do { + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + ASSERT_LEVELDB_OK(Put("bar", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v3")); + ASSERT_EQ("v3", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, PutDeleteGet) { + do { + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v2")); + ASSERT_EQ("v2", Get("foo")); + ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), "foo")); + ASSERT_EQ("NOT_FOUND", Get("foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetFromImmutableLayer) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + Reopen(&options); + + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + + // Block sync calls. + env_->delay_data_sync_.store(true, std::memory_order_release); + Put("k1", std::string(100000, 'x')); // Fill memtable. + Put("k2", std::string(100000, 'y')); // Trigger compaction. + ASSERT_EQ("v1", Get("foo")); + // Release sync calls. + env_->delay_data_sync_.store(false, std::memory_order_release); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetFromVersions) { + do { + ASSERT_LEVELDB_OK(Put("foo", "v1")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v1", Get("foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetMemUsage) { + do { + ASSERT_LEVELDB_OK(Put("foo", "v1")); + std::string val; + ASSERT_TRUE(db_->GetProperty("leveldb.approximate-memory-usage", &val)); + int mem_usage = std::stoi(val); + ASSERT_GT(mem_usage, 0); + ASSERT_LT(mem_usage, 5 * 1024 * 1024); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetSnapshot) { + do { + // Try with both a short key and a long key + for (int i = 0; i < 2; i++) { + std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x'); + ASSERT_LEVELDB_OK(Put(key, "v1")); + const Snapshot* s1 = db_->GetSnapshot(); + ASSERT_LEVELDB_OK(Put(key, "v2")); + ASSERT_EQ("v2", Get(key)); + ASSERT_EQ("v1", Get(key, s1)); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v2", Get(key)); + ASSERT_EQ("v1", Get(key, s1)); + db_->ReleaseSnapshot(s1); + } + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetIdenticalSnapshots) { + do { + // Try with both a short key and a long key + for (int i = 0; i < 2; i++) { + std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x'); + ASSERT_LEVELDB_OK(Put(key, "v1")); + const Snapshot* s1 = db_->GetSnapshot(); + const Snapshot* s2 = db_->GetSnapshot(); + const Snapshot* s3 = db_->GetSnapshot(); + ASSERT_LEVELDB_OK(Put(key, "v2")); + ASSERT_EQ("v2", Get(key)); + ASSERT_EQ("v1", Get(key, s1)); + ASSERT_EQ("v1", Get(key, s2)); + ASSERT_EQ("v1", Get(key, s3)); + db_->ReleaseSnapshot(s1); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v2", Get(key)); + ASSERT_EQ("v1", Get(key, s2)); + db_->ReleaseSnapshot(s2); + ASSERT_EQ("v1", Get(key, s3)); + db_->ReleaseSnapshot(s3); + } + } while (ChangeOptions()); +} + +TEST_F(DBTest, IterateOverEmptySnapshot) { + do { + const Snapshot* snapshot = db_->GetSnapshot(); + ReadOptions read_options; + read_options.snapshot = snapshot; + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v2")); + + Iterator* iterator1 = db_->NewIterator(read_options); + iterator1->SeekToFirst(); + ASSERT_TRUE(!iterator1->Valid()); + delete iterator1; + + dbfull()->TEST_CompactMemTable(); + + Iterator* iterator2 = db_->NewIterator(read_options); + iterator2->SeekToFirst(); + ASSERT_TRUE(!iterator2->Valid()); + delete iterator2; + + db_->ReleaseSnapshot(snapshot); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetLevel0Ordering) { + do { + // Check that we process level-0 files in correct order. The code + // below generates two level-0 files where the earlier one comes + // before the later one in the level-0 file list since the earlier + // one has a smaller "smallest" key. + ASSERT_LEVELDB_OK(Put("bar", "b")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); + dbfull()->TEST_CompactMemTable(); + ASSERT_LEVELDB_OK(Put("foo", "v2")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v2", Get("foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetOrderedByLevels) { + do { + ASSERT_LEVELDB_OK(Put("foo", "v1")); + Compact("a", "z"); + ASSERT_EQ("v1", Get("foo")); + ASSERT_LEVELDB_OK(Put("foo", "v2")); + ASSERT_EQ("v2", Get("foo")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v2", Get("foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetPicksCorrectFile) { + do { + // Arrange to have multiple files in a non-level-0 level. + ASSERT_LEVELDB_OK(Put("a", "va")); + Compact("a", "b"); + ASSERT_LEVELDB_OK(Put("x", "vx")); + Compact("x", "y"); + ASSERT_LEVELDB_OK(Put("f", "vf")); + Compact("f", "g"); + ASSERT_EQ("va", Get("a")); + ASSERT_EQ("vf", Get("f")); + ASSERT_EQ("vx", Get("x")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetEncountersEmptyLevel) { + do { + // Arrange for the following to happen: + // * sstable A in level 0 + // * nothing in level 1 + // * sstable B in level 2 + // Then do enough Get() calls to arrange for an automatic compaction + // of sstable A. A bug would cause the compaction to be marked as + // occurring at level 1 (instead of the correct level 0). + + // Step 1: First place sstables in levels 0 and 2 + int compaction_count = 0; + while (NumTableFilesAtLevel(0) == 0 || NumTableFilesAtLevel(2) == 0) { + ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2"; + compaction_count++; + Put("a", "begin"); + Put("z", "end"); + dbfull()->TEST_CompactMemTable(); + } + + // Step 2: clear level 1 if necessary. + dbfull()->TEST_CompactRange(1, nullptr, nullptr); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 1); + + // Step 3: read a bunch of times + for (int i = 0; i < 1000; i++) { + ASSERT_EQ("NOT_FOUND", Get("missing")); + } + + // Step 4: Wait for compaction to finish + DelayMilliseconds(1000); + + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + } while (ChangeOptions()); +} + +TEST_F(DBTest, IterEmpty) { + Iterator* iter = db_->NewIterator(ReadOptions()); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("foo"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; +} + +TEST_F(DBTest, IterSingle) { + ASSERT_LEVELDB_OK(Put("a", "va")); + Iterator* iter = db_->NewIterator(ReadOptions()); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek(""); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("a"); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("b"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; +} + +TEST_F(DBTest, IterMulti) { + ASSERT_LEVELDB_OK(Put("a", "va")); + ASSERT_LEVELDB_OK(Put("b", "vb")); + ASSERT_LEVELDB_OK(Put("c", "vc")); + Iterator* iter = db_->NewIterator(ReadOptions()); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek(""); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("a"); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("ax"); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Seek("b"); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Seek("z"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + // Switch from reverse to forward + iter->SeekToLast(); + iter->Prev(); + iter->Prev(); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + + // Switch from forward to reverse + iter->SeekToFirst(); + iter->Next(); + iter->Next(); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + + // Make sure iter stays at snapshot + ASSERT_LEVELDB_OK(Put("a", "va2")); + ASSERT_LEVELDB_OK(Put("a2", "va3")); + ASSERT_LEVELDB_OK(Put("b", "vb2")); + ASSERT_LEVELDB_OK(Put("c", "vc2")); + ASSERT_LEVELDB_OK(Delete("b")); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; +} + +TEST_F(DBTest, IterSmallAndLargeMix) { + ASSERT_LEVELDB_OK(Put("a", "va")); + ASSERT_LEVELDB_OK(Put("b", std::string(100000, 'b'))); + ASSERT_LEVELDB_OK(Put("c", "vc")); + ASSERT_LEVELDB_OK(Put("d", std::string(100000, 'd'))); + ASSERT_LEVELDB_OK(Put("e", std::string(100000, 'e'))); + + Iterator* iter = db_->NewIterator(ReadOptions()); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; +} + +TEST_F(DBTest, IterMultiWithDelete) { + do { + ASSERT_LEVELDB_OK(Put("a", "va")); + ASSERT_LEVELDB_OK(Put("b", "vb")); + ASSERT_LEVELDB_OK(Put("c", "vc")); + ASSERT_LEVELDB_OK(Delete("b")); + ASSERT_EQ("NOT_FOUND", Get("b")); + + Iterator* iter = db_->NewIterator(ReadOptions()); + iter->Seek("c"); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + delete iter; + } while (ChangeOptions()); +} + +TEST_F(DBTest, IterMultiWithDeleteAndCompaction) { + do { + ASSERT_LEVELDB_OK(Put("b", "vb")); + ASSERT_LEVELDB_OK(Put("c", "vc")); + ASSERT_LEVELDB_OK(Put("a", "va")); + dbfull()->TEST_CompactMemTable(); + ASSERT_LEVELDB_OK(Delete("b")); + ASSERT_EQ("NOT_FOUND", Get("b")); + + Iterator* iter = db_->NewIterator(ReadOptions()); + iter->Seek("c"); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("b"); + ASSERT_EQ(IterStatus(iter), "c->vc"); + delete iter; + } while (ChangeOptions()); +} + +TEST_F(DBTest, Recover) { + do { + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("baz", "v5")); + + Reopen(); + ASSERT_EQ("v1", Get("foo")); + + ASSERT_EQ("v1", Get("foo")); + ASSERT_EQ("v5", Get("baz")); + ASSERT_LEVELDB_OK(Put("bar", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v3")); + + Reopen(); + ASSERT_EQ("v3", Get("foo")); + ASSERT_LEVELDB_OK(Put("foo", "v4")); + ASSERT_EQ("v4", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + ASSERT_EQ("v5", Get("baz")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, RecoveryWithEmptyLog) { + do { + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v2")); + Reopen(); + Reopen(); + ASSERT_LEVELDB_OK(Put("foo", "v3")); + Reopen(); + ASSERT_EQ("v3", Get("foo")); + } while (ChangeOptions()); +} + +// Check that writes done during a memtable compaction are recovered +// if the database is shutdown during the memtable compaction. +TEST_F(DBTest, RecoverDuringMemtableCompaction) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 1000000; + Reopen(&options); + + // Trigger a long memtable compaction and reopen the database during it + ASSERT_LEVELDB_OK(Put("foo", "v1")); // Goes to 1st log file + ASSERT_LEVELDB_OK( + Put("big1", std::string(10000000, 'x'))); // Fills memtable + ASSERT_LEVELDB_OK( + Put("big2", std::string(1000, 'y'))); // Triggers compaction + ASSERT_LEVELDB_OK(Put("bar", "v2")); // Goes to new log file + + Reopen(&options); + ASSERT_EQ("v1", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + ASSERT_EQ(std::string(10000000, 'x'), Get("big1")); + ASSERT_EQ(std::string(1000, 'y'), Get("big2")); + } while (ChangeOptions()); +} + +static std::string Key(int i) { + char buf[100]; + std::snprintf(buf, sizeof(buf), "key%06d", i); + return std::string(buf); +} + +TEST_F(DBTest, MinorCompactionsHappen) { + Options options = CurrentOptions(); + options.write_buffer_size = 10000; + Reopen(&options); + + const int N = 500; + + int starting_num_tables = TotalTableFiles(); + for (int i = 0; i < N; i++) { + ASSERT_LEVELDB_OK(Put(Key(i), Key(i) + std::string(1000, 'v'))); + } + int ending_num_tables = TotalTableFiles(); + ASSERT_GT(ending_num_tables, starting_num_tables); + + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i))); + } + + Reopen(); + + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i))); + } +} + +TEST_F(DBTest, RecoverWithLargeLog) { + { + Options options = CurrentOptions(); + Reopen(&options); + ASSERT_LEVELDB_OK(Put("big1", std::string(200000, '1'))); + ASSERT_LEVELDB_OK(Put("big2", std::string(200000, '2'))); + ASSERT_LEVELDB_OK(Put("small3", std::string(10, '3'))); + ASSERT_LEVELDB_OK(Put("small4", std::string(10, '4'))); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + } + + // Make sure that if we re-open with a small write buffer size that + // we flush table files in the middle of a large log file. + Options options = CurrentOptions(); + options.write_buffer_size = 100000; + Reopen(&options); + ASSERT_EQ(NumTableFilesAtLevel(0), 3); + ASSERT_EQ(std::string(200000, '1'), Get("big1")); + ASSERT_EQ(std::string(200000, '2'), Get("big2")); + ASSERT_EQ(std::string(10, '3'), Get("small3")); + ASSERT_EQ(std::string(10, '4'), Get("small4")); + ASSERT_GT(NumTableFilesAtLevel(0), 1); +} + +TEST_F(DBTest, CompactionsGenerateMultipleFiles) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + Reopen(&options); + + Random rnd(301); + + // Write 8MB (80 values, each 100K) + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + std::vector values; + for (int i = 0; i < 80; i++) { + values.push_back(RandomString(&rnd, 100000)); + ASSERT_LEVELDB_OK(Put(Key(i), values[i])); + } + + // Reopening moves updates to level-0 + Reopen(&options); + dbfull()->TEST_CompactRange(0, nullptr, nullptr); + + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_GT(NumTableFilesAtLevel(1), 1); + for (int i = 0; i < 80; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } +} + +TEST_F(DBTest, RepeatedWritesToSameKey) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + Reopen(&options); + + // We must have at most one file per level except for level-0, + // which may have up to kL0_StopWritesTrigger files. + const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger; + + Random rnd(301); + std::string value = RandomString(&rnd, 2 * options.write_buffer_size); + for (int i = 0; i < 5 * kMaxFiles; i++) { + Put("key", value); + ASSERT_LE(TotalTableFiles(), kMaxFiles); + std::fprintf(stderr, "after %d: %d files\n", i + 1, TotalTableFiles()); + } +} + +TEST_F(DBTest, SparseMerge) { + Options options = CurrentOptions(); + options.compression = kNoCompression; + Reopen(&options); + + FillLevels("A", "Z"); + + // Suppose there is: + // small amount of data with prefix A + // large amount of data with prefix B + // small amount of data with prefix C + // and that recent updates have made small changes to all three prefixes. + // Check that we do not do a compaction that merges all of B in one shot. + const std::string value(1000, 'x'); + Put("A", "va"); + // Write approximately 100MB of "B" values + for (int i = 0; i < 100000; i++) { + char key[100]; + std::snprintf(key, sizeof(key), "B%010d", i); + Put(key, value); + } + Put("C", "vc"); + dbfull()->TEST_CompactMemTable(); + dbfull()->TEST_CompactRange(0, nullptr, nullptr); + + // Make sparse update + Put("A", "va2"); + Put("B100", "bvalue2"); + Put("C", "vc2"); + dbfull()->TEST_CompactMemTable(); + + // Compactions should not cause us to create a situation where + // a file overlaps too much data at the next level. + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); + dbfull()->TEST_CompactRange(0, nullptr, nullptr); + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); + dbfull()->TEST_CompactRange(1, nullptr, nullptr); + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); +} + +static bool Between(uint64_t val, uint64_t low, uint64_t high) { + bool result = (val >= low) && (val <= high); + if (!result) { + std::fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", + (unsigned long long)(val), (unsigned long long)(low), + (unsigned long long)(high)); + } + return result; +} + +TEST_F(DBTest, ApproximateSizes) { + do { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + options.compression = kNoCompression; + DestroyAndReopen(); + + ASSERT_TRUE(Between(Size("", "xyz"), 0, 0)); + Reopen(&options); + ASSERT_TRUE(Between(Size("", "xyz"), 0, 0)); + + // Write 8MB (80 values, each 100K) + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + const int N = 80; + static const int S1 = 100000; + static const int S2 = 105000; // Allow some expansion from metadata + Random rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_LEVELDB_OK(Put(Key(i), RandomString(&rnd, S1))); + } + + // 0 because GetApproximateSizes() does not account for memtable space + ASSERT_TRUE(Between(Size("", Key(50)), 0, 0)); + + if (options.reuse_logs) { + // Recovery will reuse memtable, and GetApproximateSizes() does not + // account for memtable usage; + Reopen(&options); + ASSERT_TRUE(Between(Size("", Key(50)), 0, 0)); + continue; + } + + // Check sizes across recovery by reopening a few times + for (int run = 0; run < 3; run++) { + Reopen(&options); + + for (int compact_start = 0; compact_start < N; compact_start += 10) { + for (int i = 0; i < N; i += 10) { + ASSERT_TRUE(Between(Size("", Key(i)), S1 * i, S2 * i)); + ASSERT_TRUE(Between(Size("", Key(i) + ".suffix"), S1 * (i + 1), + S2 * (i + 1))); + ASSERT_TRUE(Between(Size(Key(i), Key(i + 10)), S1 * 10, S2 * 10)); + } + ASSERT_TRUE(Between(Size("", Key(50)), S1 * 50, S2 * 50)); + ASSERT_TRUE(Between(Size("", Key(50) + ".suffix"), S1 * 50, S2 * 50)); + + std::string cstart_str = Key(compact_start); + std::string cend_str = Key(compact_start + 9); + Slice cstart = cstart_str; + Slice cend = cend_str; + dbfull()->TEST_CompactRange(0, &cstart, &cend); + } + + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_GT(NumTableFilesAtLevel(1), 0); + } + } while (ChangeOptions()); +} + +TEST_F(DBTest, ApproximateSizes_MixOfSmallAndLarge) { + do { + Options options = CurrentOptions(); + options.compression = kNoCompression; + Reopen(); + + Random rnd(301); + std::string big1 = RandomString(&rnd, 100000); + ASSERT_LEVELDB_OK(Put(Key(0), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(1), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(2), big1)); + ASSERT_LEVELDB_OK(Put(Key(3), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(4), big1)); + ASSERT_LEVELDB_OK(Put(Key(5), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(6), RandomString(&rnd, 300000))); + ASSERT_LEVELDB_OK(Put(Key(7), RandomString(&rnd, 10000))); + + if (options.reuse_logs) { + // Need to force a memtable compaction since recovery does not do so. + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); + } + + // Check sizes across recovery by reopening a few times + for (int run = 0; run < 3; run++) { + Reopen(&options); + + ASSERT_TRUE(Between(Size("", Key(0)), 0, 0)); + ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000)); + ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000)); + ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000)); + ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000)); + ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000)); + ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000)); + ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000)); + ASSERT_TRUE(Between(Size("", Key(8)), 550000, 560000)); + + ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000)); + + dbfull()->TEST_CompactRange(0, nullptr, nullptr); + } + } while (ChangeOptions()); +} + +TEST_F(DBTest, IteratorPinsRef) { + Put("foo", "hello"); + + // Get iterator that will yield the current contents of the DB. + Iterator* iter = db_->NewIterator(ReadOptions()); + + // Write to force compactions + Put("foo", "newvalue1"); + for (int i = 0; i < 100; i++) { + ASSERT_LEVELDB_OK( + Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values + } + Put("foo", "newvalue2"); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + ASSERT_EQ("hello", iter->value().ToString()); + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + delete iter; +} + +TEST_F(DBTest, Snapshot) { + do { + Put("foo", "v1"); + const Snapshot* s1 = db_->GetSnapshot(); + Put("foo", "v2"); + const Snapshot* s2 = db_->GetSnapshot(); + Put("foo", "v3"); + const Snapshot* s3 = db_->GetSnapshot(); + + Put("foo", "v4"); + ASSERT_EQ("v1", Get("foo", s1)); + ASSERT_EQ("v2", Get("foo", s2)); + ASSERT_EQ("v3", Get("foo", s3)); + ASSERT_EQ("v4", Get("foo")); + + db_->ReleaseSnapshot(s3); + ASSERT_EQ("v1", Get("foo", s1)); + ASSERT_EQ("v2", Get("foo", s2)); + ASSERT_EQ("v4", Get("foo")); + + db_->ReleaseSnapshot(s1); + ASSERT_EQ("v2", Get("foo", s2)); + ASSERT_EQ("v4", Get("foo")); + + db_->ReleaseSnapshot(s2); + ASSERT_EQ("v4", Get("foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, HiddenValuesAreRemoved) { + do { + Random rnd(301); + FillLevels("a", "z"); + + std::string big = RandomString(&rnd, 50000); + Put("foo", big); + Put("pastfoo", "v"); + const Snapshot* snapshot = db_->GetSnapshot(); + Put("foo", "tiny"); + Put("pastfoo2", "v2"); // Advance sequence number one more + + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); + ASSERT_GT(NumTableFilesAtLevel(0), 0); + + ASSERT_EQ(big, Get("foo", snapshot)); + ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000)); + db_->ReleaseSnapshot(snapshot); + ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]"); + Slice x("x"); + dbfull()->TEST_CompactRange(0, nullptr, &x); + ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]"); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_GE(NumTableFilesAtLevel(1), 1); + dbfull()->TEST_CompactRange(1, nullptr, &x); + ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]"); + + ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000)); + } while (ChangeOptions()); +} + +TEST_F(DBTest, DeletionMarkers1) { + Put("foo", "v1"); + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level + + // Place a table at level last-1 to prevent merging with preceding mutation + Put("a", "begin"); + Put("z", "end"); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ(NumTableFilesAtLevel(last), 1); + ASSERT_EQ(NumTableFilesAtLevel(last - 1), 1); + + Delete("foo"); + Put("foo", "v2"); + ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]"); + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 + ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]"); + Slice z("z"); + dbfull()->TEST_CompactRange(last - 2, nullptr, &z); + // DEL eliminated, but v1 remains because we aren't compacting that level + // (DEL can be eliminated because v2 hides v1). + ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]"); + dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr); + // Merging last-1 w/ last, so we are the base level for "foo", so + // DEL is removed. (as is v1). + ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]"); +} + +TEST_F(DBTest, DeletionMarkers2) { + Put("foo", "v1"); + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level + + // Place a table at level last-1 to prevent merging with preceding mutation + Put("a", "begin"); + Put("z", "end"); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ(NumTableFilesAtLevel(last), 1); + ASSERT_EQ(NumTableFilesAtLevel(last - 1), 1); + + Delete("foo"); + ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 + ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); + dbfull()->TEST_CompactRange(last - 2, nullptr, nullptr); + // DEL kept: "last" file overlaps + ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); + dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr); + // Merging last-1 w/ last, so we are the base level for "foo", so + // DEL is removed. (as is v1). + ASSERT_EQ(AllEntriesFor("foo"), "[ ]"); +} + +TEST_F(DBTest, OverlapInLevel0) { + do { + ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config"; + + // Fill levels 1 and 2 to disable the pushing of new memtables to levels > + // 0. + ASSERT_LEVELDB_OK(Put("100", "v100")); + ASSERT_LEVELDB_OK(Put("999", "v999")); + dbfull()->TEST_CompactMemTable(); + ASSERT_LEVELDB_OK(Delete("100")); + ASSERT_LEVELDB_OK(Delete("999")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("0,1,1", FilesPerLevel()); + + // Make files spanning the following ranges in level-0: + // files[0] 200 .. 900 + // files[1] 300 .. 500 + // Note that files are sorted by smallest key. + ASSERT_LEVELDB_OK(Put("300", "v300")); + ASSERT_LEVELDB_OK(Put("500", "v500")); + dbfull()->TEST_CompactMemTable(); + ASSERT_LEVELDB_OK(Put("200", "v200")); + ASSERT_LEVELDB_OK(Put("600", "v600")); + ASSERT_LEVELDB_OK(Put("900", "v900")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("2,1,1", FilesPerLevel()); + + // Compact away the placeholder files we created initially + dbfull()->TEST_CompactRange(1, nullptr, nullptr); + dbfull()->TEST_CompactRange(2, nullptr, nullptr); + ASSERT_EQ("2", FilesPerLevel()); + + // Do a memtable compaction. Before bug-fix, the compaction would + // not detect the overlap with level-0 files and would incorrectly place + // the deletion in a deeper level. + ASSERT_LEVELDB_OK(Delete("600")); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("3", FilesPerLevel()); + ASSERT_EQ("NOT_FOUND", Get("600")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, L0_CompactionBug_Issue44_a) { + Reopen(); + ASSERT_LEVELDB_OK(Put("b", "v")); + Reopen(); + ASSERT_LEVELDB_OK(Delete("b")); + ASSERT_LEVELDB_OK(Delete("a")); + Reopen(); + ASSERT_LEVELDB_OK(Delete("a")); + Reopen(); + ASSERT_LEVELDB_OK(Put("a", "v")); + Reopen(); + Reopen(); + ASSERT_EQ("(a->v)", Contents()); + DelayMilliseconds(1000); // Wait for compaction to finish + ASSERT_EQ("(a->v)", Contents()); +} + +TEST_F(DBTest, L0_CompactionBug_Issue44_b) { + Reopen(); + Put("", ""); + Reopen(); + Delete("e"); + Put("", ""); + Reopen(); + Put("c", "cv"); + Reopen(); + Put("", ""); + Reopen(); + Put("", ""); + DelayMilliseconds(1000); // Wait for compaction to finish + Reopen(); + Put("d", "dv"); + Reopen(); + Put("", ""); + Reopen(); + Delete("d"); + Delete("b"); + Reopen(); + ASSERT_EQ("(->)(c->cv)", Contents()); + DelayMilliseconds(1000); // Wait for compaction to finish + ASSERT_EQ("(->)(c->cv)", Contents()); +} + +TEST_F(DBTest, Fflush_Issue474) { + static const int kNum = 100000; + Random rnd(test::RandomSeed()); + for (int i = 0; i < kNum; i++) { + std::fflush(nullptr); + ASSERT_LEVELDB_OK(Put(RandomKey(&rnd), RandomString(&rnd, 100))); + } +} + +TEST_F(DBTest, ComparatorCheck) { + class NewComparator : public Comparator { + public: + const char* Name() const override { return "leveldb.NewComparator"; } + int Compare(const Slice& a, const Slice& b) const override { + return BytewiseComparator()->Compare(a, b); + } + void FindShortestSeparator(std::string* s, const Slice& l) const override { + BytewiseComparator()->FindShortestSeparator(s, l); + } + void FindShortSuccessor(std::string* key) const override { + BytewiseComparator()->FindShortSuccessor(key); + } + }; + NewComparator cmp; + Options new_options = CurrentOptions(); + new_options.comparator = &cmp; + Status s = TryReopen(&new_options); + ASSERT_TRUE(!s.ok()); + ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos) + << s.ToString(); +} + +TEST_F(DBTest, CustomComparator) { + class NumberComparator : public Comparator { + public: + const char* Name() const override { return "test.NumberComparator"; } + int Compare(const Slice& a, const Slice& b) const override { + return ToNumber(a) - ToNumber(b); + } + void FindShortestSeparator(std::string* s, const Slice& l) const override { + ToNumber(*s); // Check format + ToNumber(l); // Check format + } + void FindShortSuccessor(std::string* key) const override { + ToNumber(*key); // Check format + } + + private: + static int ToNumber(const Slice& x) { + // Check that there are no extra characters. + EXPECT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size() - 1] == ']') + << EscapeString(x); + int val; + char ignored; + EXPECT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1) + << EscapeString(x); + return val; + } + }; + NumberComparator cmp; + Options new_options = CurrentOptions(); + new_options.create_if_missing = true; + new_options.comparator = &cmp; + new_options.filter_policy = nullptr; // Cannot use bloom filters + new_options.write_buffer_size = 1000; // Compact more often + DestroyAndReopen(&new_options); + ASSERT_LEVELDB_OK(Put("[10]", "ten")); + ASSERT_LEVELDB_OK(Put("[0x14]", "twenty")); + for (int i = 0; i < 2; i++) { + ASSERT_EQ("ten", Get("[10]")); + ASSERT_EQ("ten", Get("[0xa]")); + ASSERT_EQ("twenty", Get("[20]")); + ASSERT_EQ("twenty", Get("[0x14]")); + ASSERT_EQ("NOT_FOUND", Get("[15]")); + ASSERT_EQ("NOT_FOUND", Get("[0xf]")); + Compact("[0]", "[9999]"); + } + + for (int run = 0; run < 2; run++) { + for (int i = 0; i < 1000; i++) { + char buf[100]; + std::snprintf(buf, sizeof(buf), "[%d]", i * 10); + ASSERT_LEVELDB_OK(Put(buf, buf)); + } + Compact("[0]", "[1000000]"); + } +} + +TEST_F(DBTest, ManualCompaction) { + ASSERT_EQ(config::kMaxMemCompactLevel, 2) + << "Need to update this test to match kMaxMemCompactLevel"; + + MakeTables(3, "p", "q"); + ASSERT_EQ("1,1,1", FilesPerLevel()); + + // Compaction range falls before files + Compact("", "c"); + ASSERT_EQ("1,1,1", FilesPerLevel()); + + // Compaction range falls after files + Compact("r", "z"); + ASSERT_EQ("1,1,1", FilesPerLevel()); + + // Compaction range overlaps files + Compact("p1", "p9"); + ASSERT_EQ("0,0,1", FilesPerLevel()); + + // Populate a different range + MakeTables(3, "c", "e"); + ASSERT_EQ("1,1,2", FilesPerLevel()); + + // Compact just the new range + Compact("b", "f"); + ASSERT_EQ("0,0,2", FilesPerLevel()); + + // Compact all + MakeTables(1, "a", "z"); + ASSERT_EQ("0,1,2", FilesPerLevel()); + db_->CompactRange(nullptr, nullptr); + ASSERT_EQ("0,0,1", FilesPerLevel()); +} + +TEST_F(DBTest, DBOpen_Options) { + std::string dbname = testing::TempDir() + "db_options_test"; + DestroyDB(dbname, Options()); + + // Does not exist, and create_if_missing == false: error + DB* db = nullptr; + Options opts; + opts.create_if_missing = false; + Status s = DB::Open(opts, dbname, &db); + ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr); + ASSERT_TRUE(db == nullptr); + + // Does not exist, and create_if_missing == true: OK + opts.create_if_missing = true; + s = DB::Open(opts, dbname, &db); + ASSERT_LEVELDB_OK(s); + ASSERT_TRUE(db != nullptr); + + delete db; + db = nullptr; + + // Does exist, and error_if_exists == true: error + opts.create_if_missing = false; + opts.error_if_exists = true; + s = DB::Open(opts, dbname, &db); + ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr); + ASSERT_TRUE(db == nullptr); + + // Does exist, and error_if_exists == false: OK + opts.create_if_missing = true; + opts.error_if_exists = false; + s = DB::Open(opts, dbname, &db); + ASSERT_LEVELDB_OK(s); + ASSERT_TRUE(db != nullptr); + + delete db; + db = nullptr; +} + +TEST_F(DBTest, DestroyEmptyDir) { + std::string dbname = testing::TempDir() + "db_empty_dir"; + TestEnv env(Env::Default()); + env.RemoveDir(dbname); + ASSERT_TRUE(!env.FileExists(dbname)); + + Options opts; + opts.env = &env; + + ASSERT_LEVELDB_OK(env.CreateDir(dbname)); + ASSERT_TRUE(env.FileExists(dbname)); + std::vector children; + ASSERT_LEVELDB_OK(env.GetChildren(dbname, &children)); +#if defined(LEVELDB_PLATFORM_CHROMIUM) + // TODO(https://crbug.com/1428746): Chromium's file system abstraction always + // filters out '.' and '..'. + ASSERT_EQ(0, children.size()); +#else + // The stock Env's do not filter out '.' and '..' special files. + ASSERT_EQ(2, children.size()); +#endif // defined(LEVELDB_PLATFORM_CHROMIUM) + ASSERT_LEVELDB_OK(DestroyDB(dbname, opts)); + ASSERT_TRUE(!env.FileExists(dbname)); + + // Should also be destroyed if Env is filtering out dot files. + env.SetIgnoreDotFiles(true); + ASSERT_LEVELDB_OK(env.CreateDir(dbname)); + ASSERT_TRUE(env.FileExists(dbname)); + ASSERT_LEVELDB_OK(env.GetChildren(dbname, &children)); + ASSERT_EQ(0, children.size()); + ASSERT_LEVELDB_OK(DestroyDB(dbname, opts)); + ASSERT_TRUE(!env.FileExists(dbname)); +} + +TEST_F(DBTest, DestroyOpenDB) { + std::string dbname = testing::TempDir() + "open_db_dir"; + env_->RemoveDir(dbname); + ASSERT_TRUE(!env_->FileExists(dbname)); + + Options opts; + opts.create_if_missing = true; + DB* db = nullptr; + ASSERT_LEVELDB_OK(DB::Open(opts, dbname, &db)); + ASSERT_TRUE(db != nullptr); + + // Must fail to destroy an open db. + ASSERT_TRUE(env_->FileExists(dbname)); + ASSERT_TRUE(!DestroyDB(dbname, Options()).ok()); + ASSERT_TRUE(env_->FileExists(dbname)); + + delete db; + db = nullptr; + + // Should succeed destroying a closed db. + ASSERT_LEVELDB_OK(DestroyDB(dbname, Options())); + ASSERT_TRUE(!env_->FileExists(dbname)); +} + +TEST_F(DBTest, Locking) { + DB* db2 = nullptr; + Status s = DB::Open(CurrentOptions(), dbname_, &db2); + ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db"; +} + +// Check that number of files does not grow when we are out of space +TEST_F(DBTest, NoSpace) { + Options options = CurrentOptions(); + options.env = env_; + Reopen(&options); + + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + Compact("a", "z"); + const int num_files = CountFiles(); + // Force out-of-space errors. + env_->no_space_.store(true, std::memory_order_release); + for (int i = 0; i < 10; i++) { + for (int level = 0; level < config::kNumLevels - 1; level++) { + dbfull()->TEST_CompactRange(level, nullptr, nullptr); + } + } + env_->no_space_.store(false, std::memory_order_release); + ASSERT_LT(CountFiles(), num_files + 3); +} + +TEST_F(DBTest, NonWritableFileSystem) { + Options options = CurrentOptions(); + options.write_buffer_size = 1000; + options.env = env_; + Reopen(&options); + ASSERT_LEVELDB_OK(Put("foo", "v1")); + // Force errors for new files. + env_->non_writable_.store(true, std::memory_order_release); + std::string big(100000, 'x'); + int errors = 0; + for (int i = 0; i < 20; i++) { + std::fprintf(stderr, "iter %d; errors %d\n", i, errors); + if (!Put("foo", big).ok()) { + errors++; + DelayMilliseconds(100); + } + } + ASSERT_GT(errors, 0); + env_->non_writable_.store(false, std::memory_order_release); +} + +TEST_F(DBTest, WriteSyncError) { + // Check that log sync errors cause the DB to disallow future writes. + + // (a) Cause log sync calls to fail + Options options = CurrentOptions(); + options.env = env_; + Reopen(&options); + env_->data_sync_error_.store(true, std::memory_order_release); + + // (b) Normal write should succeed + WriteOptions w; + ASSERT_LEVELDB_OK(db_->Put(w, "k1", "v1")); + ASSERT_EQ("v1", Get("k1")); + + // (c) Do a sync write; should fail + w.sync = true; + ASSERT_TRUE(!db_->Put(w, "k2", "v2").ok()); + ASSERT_EQ("v1", Get("k1")); + ASSERT_EQ("NOT_FOUND", Get("k2")); + + // (d) make sync behave normally + env_->data_sync_error_.store(false, std::memory_order_release); + + // (e) Do a non-sync write; should fail + w.sync = false; + ASSERT_TRUE(!db_->Put(w, "k3", "v3").ok()); + ASSERT_EQ("v1", Get("k1")); + ASSERT_EQ("NOT_FOUND", Get("k2")); + ASSERT_EQ("NOT_FOUND", Get("k3")); +} + +TEST_F(DBTest, ManifestWriteError) { + // Test for the following problem: + // (a) Compaction produces file F + // (b) Log record containing F is written to MANIFEST file, but Sync() fails + // (c) GC deletes F + // (d) After reopening DB, reads fail since deleted F is named in log record + + // We iterate twice. In the second iteration, everything is the + // same except the log record never makes it to the MANIFEST file. + for (int iter = 0; iter < 2; iter++) { + std::atomic* error_type = (iter == 0) ? &env_->manifest_sync_error_ + : &env_->manifest_write_error_; + + // Insert foo=>bar mapping + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + DestroyAndReopen(&options); + ASSERT_LEVELDB_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Memtable compaction (will succeed) + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level + + // Merging compaction (will fail) + error_type->store(true, std::memory_order_release); + dbfull()->TEST_CompactRange(last, nullptr, nullptr); // Should fail + ASSERT_EQ("bar", Get("foo")); + + // Recovery: should not lose data + error_type->store(false, std::memory_order_release); + Reopen(&options); + ASSERT_EQ("bar", Get("foo")); + } +} + +TEST_F(DBTest, MissingSSTFile) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Dump the memtable to disk. + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + + Close(); + ASSERT_TRUE(DeleteAnSSTFile()); + Options options = CurrentOptions(); + options.paranoid_checks = true; + Status s = TryReopen(&options); + ASSERT_TRUE(!s.ok()); + ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) << s.ToString(); +} + +TEST_F(DBTest, StillReadSST) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Dump the memtable to disk. + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + Close(); + ASSERT_GT(RenameLDBToSST(), 0); + Options options = CurrentOptions(); + options.paranoid_checks = true; + Status s = TryReopen(&options); + ASSERT_TRUE(s.ok()); + ASSERT_EQ("bar", Get("foo")); +} + +TEST_F(DBTest, FilesDeletedAfterCompaction) { + ASSERT_LEVELDB_OK(Put("foo", "v2")); + Compact("a", "z"); + const int num_files = CountFiles(); + for (int i = 0; i < 10; i++) { + ASSERT_LEVELDB_OK(Put("foo", "v2")); + Compact("a", "z"); + } + ASSERT_EQ(CountFiles(), num_files); +} + +TEST_F(DBTest, BloomFilter) { + env_->count_random_reads_ = true; + Options options = CurrentOptions(); + options.env = env_; + options.block_cache = NewLRUCache(0); // Prevent cache hits + options.filter_policy = NewBloomFilterPolicy(10); + Reopen(&options); + + // Populate multiple layers + const int N = 10000; + for (int i = 0; i < N; i++) { + ASSERT_LEVELDB_OK(Put(Key(i), Key(i))); + } + Compact("a", "z"); + for (int i = 0; i < N; i += 100) { + ASSERT_LEVELDB_OK(Put(Key(i), Key(i))); + } + dbfull()->TEST_CompactMemTable(); + + // Prevent auto compactions triggered by seeks + env_->delay_data_sync_.store(true, std::memory_order_release); + + // Lookup present keys. Should rarely read from small sstable. + env_->random_read_counter_.Reset(); + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i), Get(Key(i))); + } + int reads = env_->random_read_counter_.Read(); + std::fprintf(stderr, "%d present => %d reads\n", N, reads); + ASSERT_GE(reads, N); + ASSERT_LE(reads, N + 2 * N / 100); + + // Lookup present keys. Should rarely read from either sstable. + env_->random_read_counter_.Reset(); + for (int i = 0; i < N; i++) { + ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing")); + } + reads = env_->random_read_counter_.Read(); + std::fprintf(stderr, "%d missing => %d reads\n", N, reads); + ASSERT_LE(reads, 3 * N / 100); + + env_->delay_data_sync_.store(false, std::memory_order_release); + Close(); + delete options.block_cache; + delete options.filter_policy; +} + +TEST_F(DBTest, LogCloseError) { + // Regression test for bug where we could ignore log file + // Close() error when switching to a new log file. + const int kValueSize = 20000; + const int kWriteCount = 10; + const int kWriteBufferSize = (kValueSize * kWriteCount) / 2; + + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = kWriteBufferSize; // Small write buffer + Reopen(&options); + env_->log_file_close_.store(true, std::memory_order_release); + + std::string value(kValueSize, 'x'); + Status s; + for (int i = 0; i < kWriteCount && s.ok(); i++) { + s = Put(Key(i), value); + } + ASSERT_TRUE(!s.ok()) << "succeeded even after log file Close failure"; + + // Future writes should also fail after an earlier error. + s = Put("hello", "world"); + ASSERT_TRUE(!s.ok()) << "write succeeded after log file Close failure"; + + env_->log_file_close_.store(false, std::memory_order_release); +} + +// Multi-threaded test: +namespace { + +static const int kNumThreads = 4; +static const int kTestSeconds = 10; +static const int kNumKeys = 1000; + +struct MTState { + DBTest* test; + std::atomic stop; + std::atomic counter[kNumThreads]; + std::atomic thread_done[kNumThreads]; +}; + +struct MTThread { + MTState* state; + int id; +}; + +static void MTThreadBody(void* arg) { + MTThread* t = reinterpret_cast(arg); + int id = t->id; + DB* db = t->state->test->db_; + int counter = 0; + std::fprintf(stderr, "... starting thread %d\n", id); + Random rnd(1000 + id); + std::string value; + char valbuf[1500]; + while (!t->state->stop.load(std::memory_order_acquire)) { + t->state->counter[id].store(counter, std::memory_order_release); + + int key = rnd.Uniform(kNumKeys); + char keybuf[20]; + std::snprintf(keybuf, sizeof(keybuf), "%016d", key); + + if (rnd.OneIn(2)) { + // Write values of the form . + // We add some padding for force compactions. + std::snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d", key, id, + static_cast(counter)); + ASSERT_LEVELDB_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf))); + } else { + // Read a value and verify that it matches the pattern written above. + Status s = db->Get(ReadOptions(), Slice(keybuf), &value); + if (s.IsNotFound()) { + // Key has not yet been written + } else { + // Check that the writer thread counter is >= the counter in the value + ASSERT_LEVELDB_OK(s); + int k, w, c; + ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value; + ASSERT_EQ(k, key); + ASSERT_GE(w, 0); + ASSERT_LT(w, kNumThreads); + ASSERT_LE(c, t->state->counter[w].load(std::memory_order_acquire)); + } + } + counter++; + } + t->state->thread_done[id].store(true, std::memory_order_release); + std::fprintf(stderr, "... stopping thread %d after %d ops\n", id, counter); +} + +} // namespace + +TEST_F(DBTest, MultiThreaded) { + do { + // Initialize state + MTState mt; + mt.test = this; + mt.stop.store(false, std::memory_order_release); + for (int id = 0; id < kNumThreads; id++) { + mt.counter[id].store(false, std::memory_order_release); + mt.thread_done[id].store(false, std::memory_order_release); + } + + // Start threads + MTThread thread[kNumThreads]; + for (int id = 0; id < kNumThreads; id++) { + thread[id].state = &mt; + thread[id].id = id; + env_->StartThread(MTThreadBody, &thread[id]); + } + + // Let them run for a while + DelayMilliseconds(kTestSeconds * 1000); + + // Stop the threads and wait for them to finish + mt.stop.store(true, std::memory_order_release); + for (int id = 0; id < kNumThreads; id++) { + while (!mt.thread_done[id].load(std::memory_order_acquire)) { + DelayMilliseconds(100); + } + } + } while (ChangeOptions()); +} + +namespace { +typedef std::map KVMap; +} + +class ModelDB : public DB { + public: + class ModelSnapshot : public Snapshot { + public: + KVMap map_; + }; + + explicit ModelDB(const Options& options) : options_(options) {} + ~ModelDB() override = default; + Status Put(const WriteOptions& o, const Slice& k, const Slice& v) override { + return DB::Put(o, k, v); + } + Status Delete(const WriteOptions& o, const Slice& key) override { + return DB::Delete(o, key); + } + Status Get(const ReadOptions& options, const Slice& key, + std::string* value) override { + assert(false); // Not implemented + return Status::NotFound(key); + } + Iterator* NewIterator(const ReadOptions& options) override { + if (options.snapshot == nullptr) { + KVMap* saved = new KVMap; + *saved = map_; + return new ModelIter(saved, true); + } else { + const KVMap* snapshot_state = + &(reinterpret_cast(options.snapshot)->map_); + return new ModelIter(snapshot_state, false); + } + } + const Snapshot* GetSnapshot() override { + ModelSnapshot* snapshot = new ModelSnapshot; + snapshot->map_ = map_; + return snapshot; + } + + void ReleaseSnapshot(const Snapshot* snapshot) override { + delete reinterpret_cast(snapshot); + } + Status Write(const WriteOptions& options, WriteBatch* batch) override { + class Handler : public WriteBatch::Handler { + public: + KVMap* map_; + void Put(const Slice& key, const Slice& value) override { + (*map_)[key.ToString()] = value.ToString(); + } + void Delete(const Slice& key) override { map_->erase(key.ToString()); } + }; + Handler handler; + handler.map_ = &map_; + return batch->Iterate(&handler); + } + + bool GetProperty(const Slice& property, std::string* value) override { + return false; + } + void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) override { + for (int i = 0; i < n; i++) { + sizes[i] = 0; + } + } + void CompactRange(const Slice* start, const Slice* end) override {} + + private: + class ModelIter : public Iterator { + public: + ModelIter(const KVMap* map, bool owned) + : map_(map), owned_(owned), iter_(map_->end()) {} + ~ModelIter() override { + if (owned_) delete map_; + } + bool Valid() const override { return iter_ != map_->end(); } + void SeekToFirst() override { iter_ = map_->begin(); } + void SeekToLast() override { + if (map_->empty()) { + iter_ = map_->end(); + } else { + iter_ = map_->find(map_->rbegin()->first); + } + } + void Seek(const Slice& k) override { + iter_ = map_->lower_bound(k.ToString()); + } + void Next() override { ++iter_; } + void Prev() override { --iter_; } + Slice key() const override { return iter_->first; } + Slice value() const override { return iter_->second; } + Status status() const override { return Status::OK(); } + + private: + const KVMap* const map_; + const bool owned_; // Do we own map_ + KVMap::const_iterator iter_; + }; + const Options options_; + KVMap map_; +}; + +static bool CompareIterators(int step, DB* model, DB* db, + const Snapshot* model_snap, + const Snapshot* db_snap) { + ReadOptions options; + options.snapshot = model_snap; + Iterator* miter = model->NewIterator(options); + options.snapshot = db_snap; + Iterator* dbiter = db->NewIterator(options); + bool ok = true; + int count = 0; + std::vector seek_keys; + // Compare equality of all elements using Next(). Save some of the keys for + // comparing Seek equality. + for (miter->SeekToFirst(), dbiter->SeekToFirst(); + ok && miter->Valid() && dbiter->Valid(); miter->Next(), dbiter->Next()) { + count++; + if (miter->key().compare(dbiter->key()) != 0) { + std::fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", step, + EscapeString(miter->key()).c_str(), + EscapeString(dbiter->key()).c_str()); + ok = false; + break; + } + + if (miter->value().compare(dbiter->value()) != 0) { + std::fprintf(stderr, + "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n", + step, EscapeString(miter->key()).c_str(), + EscapeString(miter->value()).c_str(), + EscapeString(miter->value()).c_str()); + ok = false; + break; + } + + if (count % 10 == 0) { + seek_keys.push_back(miter->key().ToString()); + } + } + + if (ok) { + if (miter->Valid() != dbiter->Valid()) { + std::fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n", + step, miter->Valid(), dbiter->Valid()); + ok = false; + } + } + + if (ok) { + // Validate iterator equality when performing seeks. + for (auto kiter = seek_keys.begin(); ok && kiter != seek_keys.end(); + ++kiter) { + miter->Seek(*kiter); + dbiter->Seek(*kiter); + if (!miter->Valid() || !dbiter->Valid()) { + std::fprintf(stderr, "step %d: Seek iterators invalid: %d vs. %d\n", + step, miter->Valid(), dbiter->Valid()); + ok = false; + } + if (miter->key().compare(dbiter->key()) != 0) { + std::fprintf(stderr, "step %d: Seek key mismatch: '%s' vs. '%s'\n", + step, EscapeString(miter->key()).c_str(), + EscapeString(dbiter->key()).c_str()); + ok = false; + break; + } + + if (miter->value().compare(dbiter->value()) != 0) { + std::fprintf( + stderr, + "step %d: Seek value mismatch for key '%s': '%s' vs. '%s'\n", step, + EscapeString(miter->key()).c_str(), + EscapeString(miter->value()).c_str(), + EscapeString(miter->value()).c_str()); + ok = false; + break; + } + } + } + + std::fprintf(stderr, "%d entries compared: ok=%d\n", count, ok); + delete miter; + delete dbiter; + return ok; +} + +TEST_F(DBTest, Randomized) { + Random rnd(test::RandomSeed()); + do { + ModelDB model(CurrentOptions()); + const int N = 10000; + const Snapshot* model_snap = nullptr; + const Snapshot* db_snap = nullptr; + std::string k, v; + for (int step = 0; step < N; step++) { + if (step % 100 == 0) { + std::fprintf(stderr, "Step %d of %d\n", step, N); + } + // TODO(sanjay): Test Get() works + int p = rnd.Uniform(100); + if (p < 45) { // Put + k = RandomKey(&rnd); + v = RandomString( + &rnd, rnd.OneIn(20) ? 100 + rnd.Uniform(100) : rnd.Uniform(8)); + ASSERT_LEVELDB_OK(model.Put(WriteOptions(), k, v)); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), k, v)); + + } else if (p < 90) { // Delete + k = RandomKey(&rnd); + ASSERT_LEVELDB_OK(model.Delete(WriteOptions(), k)); + ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), k)); + + } else { // Multi-element batch + WriteBatch b; + const int num = rnd.Uniform(8); + for (int i = 0; i < num; i++) { + if (i == 0 || !rnd.OneIn(10)) { + k = RandomKey(&rnd); + } else { + // Periodically re-use the same key from the previous iter, so + // we have multiple entries in the write batch for the same key + } + if (rnd.OneIn(2)) { + v = RandomString(&rnd, rnd.Uniform(10)); + b.Put(k, v); + } else { + b.Delete(k); + } + } + ASSERT_LEVELDB_OK(model.Write(WriteOptions(), &b)); + ASSERT_LEVELDB_OK(db_->Write(WriteOptions(), &b)); + } + + if ((step % 100) == 0) { + ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr)); + ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap)); + // Save a snapshot from each DB this time that we'll use next + // time we compare things, to make sure the current state is + // preserved with the snapshot + if (model_snap != nullptr) model.ReleaseSnapshot(model_snap); + if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap); + + Reopen(); + ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr)); + + model_snap = model.GetSnapshot(); + db_snap = db_->GetSnapshot(); + } + } + if (model_snap != nullptr) model.ReleaseSnapshot(model_snap); + if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap); + } while (ChangeOptions()); +} + +} // namespace leveldb diff --git a/leveldb/db/dbformat.cc b/leveldb/db/dbformat.cc new file mode 100644 index 000000000..2a5749f8b --- /dev/null +++ b/leveldb/db/dbformat.cc @@ -0,0 +1,136 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/dbformat.h" + +#include +#include + +#include "port/port.h" +#include "util/coding.h" + +namespace leveldb { + +static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) { + assert(seq <= kMaxSequenceNumber); + assert(t <= kValueTypeForSeek); + return (seq << 8) | t; +} + +void AppendInternalKey(std::string* result, const ParsedInternalKey& key) { + result->append(key.user_key.data(), key.user_key.size()); + PutFixed64(result, PackSequenceAndType(key.sequence, key.type)); +} + +std::string ParsedInternalKey::DebugString() const { + std::ostringstream ss; + ss << '\'' << EscapeString(user_key.ToString()) << "' @ " << sequence << " : " + << static_cast(type); + return ss.str(); +} + +std::string InternalKey::DebugString() const { + ParsedInternalKey parsed; + if (ParseInternalKey(rep_, &parsed)) { + return parsed.DebugString(); + } + std::ostringstream ss; + ss << "(bad)" << EscapeString(rep_); + return ss.str(); +} + +const char* InternalKeyComparator::Name() const { + return "leveldb.InternalKeyComparator"; +} + +int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const { + // Order by: + // increasing user key (according to user-supplied comparator) + // decreasing sequence number + // decreasing type (though sequence# should be enough to disambiguate) + int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey)); + if (r == 0) { + const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8); + const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8); + if (anum > bnum) { + r = -1; + } else if (anum < bnum) { + r = +1; + } + } + return r; +} + +void InternalKeyComparator::FindShortestSeparator(std::string* start, + const Slice& limit) const { + // Attempt to shorten the user portion of the key + Slice user_start = ExtractUserKey(*start); + Slice user_limit = ExtractUserKey(limit); + std::string tmp(user_start.data(), user_start.size()); + user_comparator_->FindShortestSeparator(&tmp, user_limit); + if (tmp.size() < user_start.size() && + user_comparator_->Compare(user_start, tmp) < 0) { + // User key has become shorter physically, but larger logically. + // Tack on the earliest possible number to the shortened user key. + PutFixed64(&tmp, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); + assert(this->Compare(*start, tmp) < 0); + assert(this->Compare(tmp, limit) < 0); + start->swap(tmp); + } +} + +void InternalKeyComparator::FindShortSuccessor(std::string* key) const { + Slice user_key = ExtractUserKey(*key); + std::string tmp(user_key.data(), user_key.size()); + user_comparator_->FindShortSuccessor(&tmp); + if (tmp.size() < user_key.size() && + user_comparator_->Compare(user_key, tmp) < 0) { + // User key has become shorter physically, but larger logically. + // Tack on the earliest possible number to the shortened user key. + PutFixed64(&tmp, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); + assert(this->Compare(*key, tmp) < 0); + key->swap(tmp); + } +} + +const char* InternalFilterPolicy::Name() const { return user_policy_->Name(); } + +void InternalFilterPolicy::CreateFilter(const Slice* keys, int n, + std::string* dst) const { + // We rely on the fact that the code in table.cc does not mind us + // adjusting keys[]. + Slice* mkey = const_cast(keys); + for (int i = 0; i < n; i++) { + mkey[i] = ExtractUserKey(keys[i]); + // TODO(sanjay): Suppress dups? + } + user_policy_->CreateFilter(keys, n, dst); +} + +bool InternalFilterPolicy::KeyMayMatch(const Slice& key, const Slice& f) const { + return user_policy_->KeyMayMatch(ExtractUserKey(key), f); +} + +LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) { + size_t usize = user_key.size(); + size_t needed = usize + 13; // A conservative estimate + char* dst; + if (needed <= sizeof(space_)) { + dst = space_; + } else { + dst = new char[needed]; + } + start_ = dst; + dst = EncodeVarint32(dst, usize + 8); + kstart_ = dst; + std::memcpy(dst, user_key.data(), usize); + dst += usize; + EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek)); + dst += 8; + end_ = dst; +} + +} // namespace leveldb diff --git a/leveldb/db/dbformat.h b/leveldb/db/dbformat.h new file mode 100644 index 000000000..a1c30ed88 --- /dev/null +++ b/leveldb/db/dbformat.h @@ -0,0 +1,224 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_ +#define STORAGE_LEVELDB_DB_DBFORMAT_H_ + +#include +#include +#include + +#include "leveldb/comparator.h" +#include "leveldb/db.h" +#include "leveldb/filter_policy.h" +#include "leveldb/slice.h" +#include "leveldb/table_builder.h" +#include "util/coding.h" +#include "util/logging.h" + +namespace leveldb { + +// Grouping of constants. We may want to make some of these +// parameters set via options. +namespace config { +static const int kNumLevels = 7; + +// Level-0 compaction is started when we hit this many files. +static const int kL0_CompactionTrigger = 4; + +// Soft limit on number of level-0 files. We slow down writes at this point. +static const int kL0_SlowdownWritesTrigger = 8; + +// Maximum number of level-0 files. We stop writes at this point. +static const int kL0_StopWritesTrigger = 12; + +// Maximum level to which a new compacted memtable is pushed if it +// does not create overlap. We try to push to level 2 to avoid the +// relatively expensive level 0=>1 compactions and to avoid some +// expensive manifest file operations. We do not push all the way to +// the largest level since that can generate a lot of wasted disk +// space if the same key space is being repeatedly overwritten. +static const int kMaxMemCompactLevel = 2; + +// Approximate gap in bytes between samples of data read during iteration. +static const int kReadBytesPeriod = 1048576; + +} // namespace config + +class InternalKey; + +// Value types encoded as the last component of internal keys. +// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk +// data structures. +enum ValueType { kTypeDeletion = 0x0, kTypeValue = 0x1 }; +// kValueTypeForSeek defines the ValueType that should be passed when +// constructing a ParsedInternalKey object for seeking to a particular +// sequence number (since we sort sequence numbers in decreasing order +// and the value type is embedded as the low 8 bits in the sequence +// number in internal keys, we need to use the highest-numbered +// ValueType, not the lowest). +static const ValueType kValueTypeForSeek = kTypeValue; + +typedef uint64_t SequenceNumber; + +// We leave eight bits empty at the bottom so a type and sequence# +// can be packed together into 64-bits. +static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1); + +struct ParsedInternalKey { + Slice user_key; + SequenceNumber sequence; + ValueType type; + + ParsedInternalKey() {} // Intentionally left uninitialized (for speed) + ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t) + : user_key(u), sequence(seq), type(t) {} + std::string DebugString() const; +}; + +// Return the length of the encoding of "key". +inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) { + return key.user_key.size() + 8; +} + +// Append the serialization of "key" to *result. +void AppendInternalKey(std::string* result, const ParsedInternalKey& key); + +// Attempt to parse an internal key from "internal_key". On success, +// stores the parsed data in "*result", and returns true. +// +// On error, returns false, leaves "*result" in an undefined state. +bool ParseInternalKey(const Slice& internal_key, ParsedInternalKey* result); + +// Returns the user key portion of an internal key. +inline Slice ExtractUserKey(const Slice& internal_key) { + assert(internal_key.size() >= 8); + return Slice(internal_key.data(), internal_key.size() - 8); +} + +// A comparator for internal keys that uses a specified comparator for +// the user key portion and breaks ties by decreasing sequence number. +class InternalKeyComparator : public Comparator { + private: + const Comparator* user_comparator_; + + public: + explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {} + const char* Name() const override; + int Compare(const Slice& a, const Slice& b) const override; + void FindShortestSeparator(std::string* start, + const Slice& limit) const override; + void FindShortSuccessor(std::string* key) const override; + + const Comparator* user_comparator() const { return user_comparator_; } + + int Compare(const InternalKey& a, const InternalKey& b) const; +}; + +// Filter policy wrapper that converts from internal keys to user keys +class InternalFilterPolicy : public FilterPolicy { + private: + const FilterPolicy* const user_policy_; + + public: + explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) {} + const char* Name() const override; + void CreateFilter(const Slice* keys, int n, std::string* dst) const override; + bool KeyMayMatch(const Slice& key, const Slice& filter) const override; +}; + +// Modules in this directory should keep internal keys wrapped inside +// the following class instead of plain strings so that we do not +// incorrectly use string comparisons instead of an InternalKeyComparator. +class InternalKey { + private: + std::string rep_; + + public: + InternalKey() {} // Leave rep_ as empty to indicate it is invalid + InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) { + AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t)); + } + + bool DecodeFrom(const Slice& s) { + rep_.assign(s.data(), s.size()); + return !rep_.empty(); + } + + Slice Encode() const { + assert(!rep_.empty()); + return rep_; + } + + Slice user_key() const { return ExtractUserKey(rep_); } + + void SetFrom(const ParsedInternalKey& p) { + rep_.clear(); + AppendInternalKey(&rep_, p); + } + + void Clear() { rep_.clear(); } + + std::string DebugString() const; +}; + +inline int InternalKeyComparator::Compare(const InternalKey& a, + const InternalKey& b) const { + return Compare(a.Encode(), b.Encode()); +} + +inline bool ParseInternalKey(const Slice& internal_key, + ParsedInternalKey* result) { + const size_t n = internal_key.size(); + if (n < 8) return false; + uint64_t num = DecodeFixed64(internal_key.data() + n - 8); + uint8_t c = num & 0xff; + result->sequence = num >> 8; + result->type = static_cast(c); + result->user_key = Slice(internal_key.data(), n - 8); + return (c <= static_cast(kTypeValue)); +} + +// A helper class useful for DBImpl::Get() +class LookupKey { + public: + // Initialize *this for looking up user_key at a snapshot with + // the specified sequence number. + LookupKey(const Slice& user_key, SequenceNumber sequence); + + LookupKey(const LookupKey&) = delete; + LookupKey& operator=(const LookupKey&) = delete; + + ~LookupKey(); + + // Return a key suitable for lookup in a MemTable. + Slice memtable_key() const { return Slice(start_, end_ - start_); } + + // Return an internal key (suitable for passing to an internal iterator) + Slice internal_key() const { return Slice(kstart_, end_ - kstart_); } + + // Return the user key + Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); } + + private: + // We construct a char array of the form: + // klength varint32 <-- start_ + // userkey char[klength] <-- kstart_ + // tag uint64 + // <-- end_ + // The array is a suitable MemTable key. + // The suffix starting with "userkey" can be used as an InternalKey. + const char* start_; + const char* kstart_; + const char* end_; + char space_[200]; // Avoid allocation for short keys +}; + +inline LookupKey::~LookupKey() { + if (start_ != space_) delete[] start_; +} + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_DBFORMAT_H_ diff --git a/leveldb/db/dbformat_test.cc b/leveldb/db/dbformat_test.cc new file mode 100644 index 000000000..7f3f81a5d --- /dev/null +++ b/leveldb/db/dbformat_test.cc @@ -0,0 +1,128 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/dbformat.h" + +#include "gtest/gtest.h" +#include "util/logging.h" + +namespace leveldb { + +static std::string IKey(const std::string& user_key, uint64_t seq, + ValueType vt) { + std::string encoded; + AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); + return encoded; +} + +static std::string Shorten(const std::string& s, const std::string& l) { + std::string result = s; + InternalKeyComparator(BytewiseComparator()).FindShortestSeparator(&result, l); + return result; +} + +static std::string ShortSuccessor(const std::string& s) { + std::string result = s; + InternalKeyComparator(BytewiseComparator()).FindShortSuccessor(&result); + return result; +} + +static void TestKey(const std::string& key, uint64_t seq, ValueType vt) { + std::string encoded = IKey(key, seq, vt); + + Slice in(encoded); + ParsedInternalKey decoded("", 0, kTypeValue); + + ASSERT_TRUE(ParseInternalKey(in, &decoded)); + ASSERT_EQ(key, decoded.user_key.ToString()); + ASSERT_EQ(seq, decoded.sequence); + ASSERT_EQ(vt, decoded.type); + + ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); +} + +TEST(FormatTest, InternalKey_EncodeDecode) { + const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"}; + const uint64_t seq[] = {1, + 2, + 3, + (1ull << 8) - 1, + 1ull << 8, + (1ull << 8) + 1, + (1ull << 16) - 1, + 1ull << 16, + (1ull << 16) + 1, + (1ull << 32) - 1, + 1ull << 32, + (1ull << 32) + 1}; + for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { + for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { + TestKey(keys[k], seq[s], kTypeValue); + TestKey("hello", 1, kTypeDeletion); + } + } +} + +TEST(FormatTest, InternalKey_DecodeFromEmpty) { + InternalKey internal_key; + + ASSERT_TRUE(!internal_key.DecodeFrom("")); +} + +TEST(FormatTest, InternalKeyShortSeparator) { + // When user keys are same + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion))); + + // When user keys are misordered + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue))); + + // When user keys are different, but correctly ordered + ASSERT_EQ( + IKey("g", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue))); + + // When start user key is prefix of limit user key + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue))); + + // When limit user key is prefix of start user key + ASSERT_EQ( + IKey("foobar", 100, kTypeValue), + Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue))); +} + +TEST(FormatTest, InternalKeyShortestSuccessor) { + ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), + ShortSuccessor(IKey("foo", 100, kTypeValue))); + ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue), + ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); +} + +TEST(FormatTest, ParsedInternalKeyDebugString) { + ParsedInternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue); + + ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString()); +} + +TEST(FormatTest, InternalKeyDebugString) { + InternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue); + ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString()); + + InternalKey invalid_key; + ASSERT_EQ("(bad)", invalid_key.DebugString()); +} + +} // namespace leveldb diff --git a/leveldb/db/dumpfile.cc b/leveldb/db/dumpfile.cc new file mode 100644 index 000000000..6085475fd --- /dev/null +++ b/leveldb/db/dumpfile.cc @@ -0,0 +1,232 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/dumpfile.h" + +#include + +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/version_edit.h" +#include "db/write_batch_internal.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" +#include "leveldb/status.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" + +namespace leveldb { + +namespace { + +bool GuessType(const std::string& fname, FileType* type) { + size_t pos = fname.rfind('/'); + std::string basename; + if (pos == std::string::npos) { + basename = fname; + } else { + basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); + } + uint64_t ignored; + return ParseFileName(basename, &ignored, type); +} + +// Notified when log reader encounters corruption. +class CorruptionReporter : public log::Reader::Reporter { + public: + void Corruption(size_t bytes, const Status& status) override { + std::string r = "corruption: "; + AppendNumberTo(&r, bytes); + r += " bytes; "; + r += status.ToString(); + r.push_back('\n'); + dst_->Append(r); + } + + WritableFile* dst_; +}; + +// Print contents of a log file. (*func)() is called on every record. +Status PrintLogContents(Env* env, const std::string& fname, + void (*func)(uint64_t, Slice, WritableFile*), + WritableFile* dst) { + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + return s; + } + CorruptionReporter reporter; + reporter.dst_ = dst; + log::Reader reader(file, &reporter, true, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch)) { + (*func)(reader.LastRecordOffset(), record, dst); + } + delete file; + return Status::OK(); +} + +// Called on every item found in a WriteBatch. +class WriteBatchItemPrinter : public WriteBatch::Handler { + public: + void Put(const Slice& key, const Slice& value) override { + std::string r = " put '"; + AppendEscapedStringTo(&r, key); + r += "' '"; + AppendEscapedStringTo(&r, value); + r += "'\n"; + dst_->Append(r); + } + void Delete(const Slice& key) override { + std::string r = " del '"; + AppendEscapedStringTo(&r, key); + r += "'\n"; + dst_->Append(r); + } + + WritableFile* dst_; +}; + +// Called on every log record (each one of which is a WriteBatch) +// found in a kLogFile. +static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) { + std::string r = "--- offset "; + AppendNumberTo(&r, pos); + r += "; "; + if (record.size() < 12) { + r += "log record length "; + AppendNumberTo(&r, record.size()); + r += " is too small\n"; + dst->Append(r); + return; + } + WriteBatch batch; + WriteBatchInternal::SetContents(&batch, record); + r += "sequence "; + AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch)); + r.push_back('\n'); + dst->Append(r); + WriteBatchItemPrinter batch_item_printer; + batch_item_printer.dst_ = dst; + Status s = batch.Iterate(&batch_item_printer); + if (!s.ok()) { + dst->Append(" error: " + s.ToString() + "\n"); + } +} + +Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) { + return PrintLogContents(env, fname, WriteBatchPrinter, dst); +} + +// Called on every log record (each one of which is a WriteBatch) +// found in a kDescriptorFile. +static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) { + std::string r = "--- offset "; + AppendNumberTo(&r, pos); + r += "; "; + VersionEdit edit; + Status s = edit.DecodeFrom(record); + if (!s.ok()) { + r += s.ToString(); + r.push_back('\n'); + } else { + r += edit.DebugString(); + } + dst->Append(r); +} + +Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) { + return PrintLogContents(env, fname, VersionEditPrinter, dst); +} + +Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) { + uint64_t file_size; + RandomAccessFile* file = nullptr; + Table* table = nullptr; + Status s = env->GetFileSize(fname, &file_size); + if (s.ok()) { + s = env->NewRandomAccessFile(fname, &file); + } + if (s.ok()) { + // We use the default comparator, which may or may not match the + // comparator used in this database. However this should not cause + // problems since we only use Table operations that do not require + // any comparisons. In particular, we do not call Seek or Prev. + s = Table::Open(Options(), file, file_size, &table); + } + if (!s.ok()) { + delete table; + delete file; + return s; + } + + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = table->NewIterator(ro); + std::string r; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + r.clear(); + ParsedInternalKey key; + if (!ParseInternalKey(iter->key(), &key)) { + r = "badkey '"; + AppendEscapedStringTo(&r, iter->key()); + r += "' => '"; + AppendEscapedStringTo(&r, iter->value()); + r += "'\n"; + dst->Append(r); + } else { + r = "'"; + AppendEscapedStringTo(&r, key.user_key); + r += "' @ "; + AppendNumberTo(&r, key.sequence); + r += " : "; + if (key.type == kTypeDeletion) { + r += "del"; + } else if (key.type == kTypeValue) { + r += "val"; + } else { + AppendNumberTo(&r, key.type); + } + r += " => '"; + AppendEscapedStringTo(&r, iter->value()); + r += "'\n"; + dst->Append(r); + } + } + s = iter->status(); + if (!s.ok()) { + dst->Append("iterator error: " + s.ToString() + "\n"); + } + + delete iter; + delete table; + delete file; + return Status::OK(); +} + +} // namespace + +Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) { + FileType ftype; + if (!GuessType(fname, &ftype)) { + return Status::InvalidArgument(fname + ": unknown file type"); + } + switch (ftype) { + case kLogFile: + return DumpLog(env, fname, dst); + case kDescriptorFile: + return DumpDescriptor(env, fname, dst); + case kTableFile: + return DumpTable(env, fname, dst); + default: + break; + } + return Status::InvalidArgument(fname + ": not a dump-able file type"); +} + +} // namespace leveldb diff --git a/leveldb/db/fault_injection_test.cc b/leveldb/db/fault_injection_test.cc new file mode 100644 index 000000000..ef864a4f0 --- /dev/null +++ b/leveldb/db/fault_injection_test.cc @@ -0,0 +1,550 @@ +// Copyright 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// This test uses a custom Env to keep track of the state of a filesystem as of +// the last "sync". It then checks for data loss errors by purposely dropping +// file data (or entire files) not protected by a "sync". + +#include +#include + +#include "gtest/gtest.h" +#include "db/db_impl.h" +#include "db/filename.h" +#include "db/log_format.h" +#include "db/version_set.h" +#include "leveldb/cache.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/logging.h" +#include "util/mutexlock.h" +#include "util/testutil.h" + +namespace leveldb { + +static const int kValueSize = 1000; +static const int kMaxNumValues = 2000; +static const size_t kNumIterations = 3; + +class FaultInjectionTestEnv; + +namespace { + +// Assume a filename, and not a directory name like "/foo/bar/" +static std::string GetDirName(const std::string& filename) { + size_t found = filename.find_last_of("/\\"); + if (found == std::string::npos) { + return ""; + } else { + return filename.substr(0, found); + } +} + +Status SyncDir(const std::string& dir) { + // As this is a test it isn't required to *actually* sync this directory. + return Status::OK(); +} + +// A basic file truncation function suitable for this test. +Status Truncate(const std::string& filename, uint64_t length) { + leveldb::Env* env = leveldb::Env::Default(); + + SequentialFile* orig_file; + Status s = env->NewSequentialFile(filename, &orig_file); + if (!s.ok()) return s; + + char* scratch = new char[length]; + leveldb::Slice result; + s = orig_file->Read(length, &result, scratch); + delete orig_file; + if (s.ok()) { + std::string tmp_name = GetDirName(filename) + "/truncate.tmp"; + WritableFile* tmp_file; + s = env->NewWritableFile(tmp_name, &tmp_file); + if (s.ok()) { + s = tmp_file->Append(result); + delete tmp_file; + if (s.ok()) { + s = env->RenameFile(tmp_name, filename); + } else { + env->RemoveFile(tmp_name); + } + } + } + + delete[] scratch; + + return s; +} + +struct FileState { + std::string filename_; + int64_t pos_; + int64_t pos_at_last_sync_; + int64_t pos_at_last_flush_; + + FileState(const std::string& filename) + : filename_(filename), + pos_(-1), + pos_at_last_sync_(-1), + pos_at_last_flush_(-1) {} + + FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} + + bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; } + + Status DropUnsyncedData() const; +}; + +} // anonymous namespace + +// A wrapper around WritableFile which informs another Env whenever this file +// is written to or sync'ed. +class TestWritableFile : public WritableFile { + public: + TestWritableFile(const FileState& state, WritableFile* f, + FaultInjectionTestEnv* env); + ~TestWritableFile() override; + Status Append(const Slice& data) override; + Status Close() override; + Status Flush() override; + Status Sync() override; + + private: + FileState state_; + WritableFile* target_; + bool writable_file_opened_; + FaultInjectionTestEnv* env_; + + Status SyncParent(); +}; + +class FaultInjectionTestEnv : public EnvWrapper { + public: + FaultInjectionTestEnv() + : EnvWrapper(Env::Default()), filesystem_active_(true) {} + ~FaultInjectionTestEnv() override = default; + Status NewWritableFile(const std::string& fname, + WritableFile** result) override; + Status NewAppendableFile(const std::string& fname, + WritableFile** result) override; + Status RemoveFile(const std::string& f) override; + Status RenameFile(const std::string& s, const std::string& t) override; + + void WritableFileClosed(const FileState& state); + Status DropUnsyncedFileData(); + Status RemoveFilesCreatedAfterLastDirSync(); + void DirWasSynced(); + bool IsFileCreatedSinceLastDirSync(const std::string& filename); + void ResetState(); + void UntrackFile(const std::string& f); + // Setting the filesystem to inactive is the test equivalent to simulating a + // system reset. Setting to inactive will freeze our saved filesystem state so + // that it will stop being recorded. It can then be reset back to the state at + // the time of the reset. + bool IsFilesystemActive() LOCKS_EXCLUDED(mutex_) { + MutexLock l(&mutex_); + return filesystem_active_; + } + void SetFilesystemActive(bool active) LOCKS_EXCLUDED(mutex_) { + MutexLock l(&mutex_); + filesystem_active_ = active; + } + + private: + port::Mutex mutex_; + std::map db_file_state_ GUARDED_BY(mutex_); + std::set new_files_since_last_dir_sync_ GUARDED_BY(mutex_); + bool filesystem_active_ GUARDED_BY(mutex_); // Record flushes, syncs, writes +}; + +TestWritableFile::TestWritableFile(const FileState& state, WritableFile* f, + FaultInjectionTestEnv* env) + : state_(state), target_(f), writable_file_opened_(true), env_(env) { + assert(f != nullptr); +} + +TestWritableFile::~TestWritableFile() { + if (writable_file_opened_) { + Close(); + } + delete target_; +} + +Status TestWritableFile::Append(const Slice& data) { + Status s = target_->Append(data); + if (s.ok() && env_->IsFilesystemActive()) { + state_.pos_ += data.size(); + } + return s; +} + +Status TestWritableFile::Close() { + writable_file_opened_ = false; + Status s = target_->Close(); + if (s.ok()) { + env_->WritableFileClosed(state_); + } + return s; +} + +Status TestWritableFile::Flush() { + Status s = target_->Flush(); + if (s.ok() && env_->IsFilesystemActive()) { + state_.pos_at_last_flush_ = state_.pos_; + } + return s; +} + +Status TestWritableFile::SyncParent() { + Status s = SyncDir(GetDirName(state_.filename_)); + if (s.ok()) { + env_->DirWasSynced(); + } + return s; +} + +Status TestWritableFile::Sync() { + if (!env_->IsFilesystemActive()) { + return Status::OK(); + } + // Ensure new files referred to by the manifest are in the filesystem. + Status s = target_->Sync(); + if (s.ok()) { + state_.pos_at_last_sync_ = state_.pos_; + } + if (env_->IsFileCreatedSinceLastDirSync(state_.filename_)) { + Status ps = SyncParent(); + if (s.ok() && !ps.ok()) { + s = ps; + } + } + return s; +} + +Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname, + WritableFile** result) { + WritableFile* actual_writable_file; + Status s = target()->NewWritableFile(fname, &actual_writable_file); + if (s.ok()) { + FileState state(fname); + state.pos_ = 0; + *result = new TestWritableFile(state, actual_writable_file, this); + // NewWritableFile doesn't append to files, so if the same file is + // opened again then it will be truncated - so forget our saved + // state. + UntrackFile(fname); + MutexLock l(&mutex_); + new_files_since_last_dir_sync_.insert(fname); + } + return s; +} + +Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname, + WritableFile** result) { + WritableFile* actual_writable_file; + Status s = target()->NewAppendableFile(fname, &actual_writable_file); + if (s.ok()) { + FileState state(fname); + state.pos_ = 0; + { + MutexLock l(&mutex_); + if (db_file_state_.count(fname) == 0) { + new_files_since_last_dir_sync_.insert(fname); + } else { + state = db_file_state_[fname]; + } + } + *result = new TestWritableFile(state, actual_writable_file, this); + } + return s; +} + +Status FaultInjectionTestEnv::DropUnsyncedFileData() { + Status s; + MutexLock l(&mutex_); + for (const auto& kvp : db_file_state_) { + if (!s.ok()) { + break; + } + const FileState& state = kvp.second; + if (!state.IsFullySynced()) { + s = state.DropUnsyncedData(); + } + } + return s; +} + +void FaultInjectionTestEnv::DirWasSynced() { + MutexLock l(&mutex_); + new_files_since_last_dir_sync_.clear(); +} + +bool FaultInjectionTestEnv::IsFileCreatedSinceLastDirSync( + const std::string& filename) { + MutexLock l(&mutex_); + return new_files_since_last_dir_sync_.find(filename) != + new_files_since_last_dir_sync_.end(); +} + +void FaultInjectionTestEnv::UntrackFile(const std::string& f) { + MutexLock l(&mutex_); + db_file_state_.erase(f); + new_files_since_last_dir_sync_.erase(f); +} + +Status FaultInjectionTestEnv::RemoveFile(const std::string& f) { + Status s = EnvWrapper::RemoveFile(f); + EXPECT_LEVELDB_OK(s); + if (s.ok()) { + UntrackFile(f); + } + return s; +} + +Status FaultInjectionTestEnv::RenameFile(const std::string& s, + const std::string& t) { + Status ret = EnvWrapper::RenameFile(s, t); + + if (ret.ok()) { + MutexLock l(&mutex_); + if (db_file_state_.find(s) != db_file_state_.end()) { + db_file_state_[t] = db_file_state_[s]; + db_file_state_.erase(s); + } + + if (new_files_since_last_dir_sync_.erase(s) != 0) { + assert(new_files_since_last_dir_sync_.find(t) == + new_files_since_last_dir_sync_.end()); + new_files_since_last_dir_sync_.insert(t); + } + } + + return ret; +} + +void FaultInjectionTestEnv::ResetState() { + // Since we are not destroying the database, the existing files + // should keep their recorded synced/flushed state. Therefore + // we do not reset db_file_state_ and new_files_since_last_dir_sync_. + SetFilesystemActive(true); +} + +Status FaultInjectionTestEnv::RemoveFilesCreatedAfterLastDirSync() { + // Because RemoveFile access this container make a copy to avoid deadlock + mutex_.Lock(); + std::set new_files(new_files_since_last_dir_sync_.begin(), + new_files_since_last_dir_sync_.end()); + mutex_.Unlock(); + Status status; + for (const auto& new_file : new_files) { + Status remove_status = RemoveFile(new_file); + if (!remove_status.ok() && status.ok()) { + status = std::move(remove_status); + } + } + return status; +} + +void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { + MutexLock l(&mutex_); + db_file_state_[state.filename_] = state; +} + +Status FileState::DropUnsyncedData() const { + int64_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; + return Truncate(filename_, sync_pos); +} + +class FaultInjectionTest : public testing::Test { + public: + enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR }; + enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES }; + + FaultInjectionTestEnv* env_; + std::string dbname_; + Cache* tiny_cache_; + Options options_; + DB* db_; + + FaultInjectionTest() + : env_(new FaultInjectionTestEnv), + tiny_cache_(NewLRUCache(100)), + db_(nullptr) { + dbname_ = testing::TempDir() + "fault_test"; + DestroyDB(dbname_, Options()); // Destroy any db from earlier run + options_.reuse_logs = true; + options_.env = env_; + options_.paranoid_checks = true; + options_.block_cache = tiny_cache_; + options_.create_if_missing = true; + } + + ~FaultInjectionTest() { + CloseDB(); + DestroyDB(dbname_, Options()); + delete tiny_cache_; + delete env_; + } + + void ReuseLogs(bool reuse) { options_.reuse_logs = reuse; } + + void Build(int start_idx, int num_vals) { + std::string key_space, value_space; + WriteBatch batch; + for (int i = start_idx; i < start_idx + num_vals; i++) { + Slice key = Key(i, &key_space); + batch.Clear(); + batch.Put(key, Value(i, &value_space)); + WriteOptions options; + ASSERT_LEVELDB_OK(db_->Write(options, &batch)); + } + } + + Status ReadValue(int i, std::string* val) const { + std::string key_space, value_space; + Slice key = Key(i, &key_space); + Value(i, &value_space); + ReadOptions options; + return db_->Get(options, key, val); + } + + Status Verify(int start_idx, int num_vals, + ExpectedVerifResult expected) const { + std::string val; + std::string value_space; + Status s; + for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) { + Value(i, &value_space); + s = ReadValue(i, &val); + if (expected == VAL_EXPECT_NO_ERROR) { + if (s.ok()) { + EXPECT_EQ(value_space, val); + } + } else if (s.ok()) { + std::fprintf(stderr, "Expected an error at %d, but was OK\n", i); + s = Status::IOError(dbname_, "Expected value error:"); + } else { + s = Status::OK(); // An expected error + } + } + return s; + } + + // Return the ith key + Slice Key(int i, std::string* storage) const { + char buf[100]; + std::snprintf(buf, sizeof(buf), "%016d", i); + storage->assign(buf, strlen(buf)); + return Slice(*storage); + } + + // Return the value to associate with the specified key + Slice Value(int k, std::string* storage) const { + Random r(k); + return test::RandomString(&r, kValueSize, storage); + } + + Status OpenDB() { + delete db_; + db_ = nullptr; + env_->ResetState(); + return DB::Open(options_, dbname_, &db_); + } + + void CloseDB() { + delete db_; + db_ = nullptr; + } + + void DeleteAllData() { + Iterator* iter = db_->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), iter->key())); + } + + delete iter; + } + + void ResetDBState(ResetMethod reset_method) { + switch (reset_method) { + case RESET_DROP_UNSYNCED_DATA: + ASSERT_LEVELDB_OK(env_->DropUnsyncedFileData()); + break; + case RESET_DELETE_UNSYNCED_FILES: + ASSERT_LEVELDB_OK(env_->RemoveFilesCreatedAfterLastDirSync()); + break; + default: + assert(false); + } + } + + void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) { + DeleteAllData(); + Build(0, num_pre_sync); + db_->CompactRange(nullptr, nullptr); + Build(num_pre_sync, num_post_sync); + } + + void PartialCompactTestReopenWithFault(ResetMethod reset_method, + int num_pre_sync, int num_post_sync) { + env_->SetFilesystemActive(false); + CloseDB(); + ResetDBState(reset_method); + ASSERT_LEVELDB_OK(OpenDB()); + ASSERT_LEVELDB_OK( + Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR)); + ASSERT_LEVELDB_OK(Verify(num_pre_sync, num_post_sync, + FaultInjectionTest::VAL_EXPECT_ERROR)); + } + + void NoWriteTestPreFault() {} + + void NoWriteTestReopenWithFault(ResetMethod reset_method) { + CloseDB(); + ResetDBState(reset_method); + ASSERT_LEVELDB_OK(OpenDB()); + } + + void DoTest() { + Random rnd(0); + ASSERT_LEVELDB_OK(OpenDB()); + for (size_t idx = 0; idx < kNumIterations; idx++) { + int num_pre_sync = rnd.Uniform(kMaxNumValues); + int num_post_sync = rnd.Uniform(kMaxNumValues); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, num_pre_sync, + num_post_sync); + + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + // No new files created so we expect all values since no files will be + // dropped. + PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES, + num_pre_sync + num_post_sync, 0); + + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES); + } + } +}; + +TEST_F(FaultInjectionTest, FaultTestNoLogReuse) { + ReuseLogs(false); + DoTest(); +} + +TEST_F(FaultInjectionTest, FaultTestWithLogReuse) { + ReuseLogs(true); + DoTest(); +} + +} // namespace leveldb diff --git a/leveldb/db/filename.cc b/leveldb/db/filename.cc new file mode 100644 index 000000000..e52624958 --- /dev/null +++ b/leveldb/db/filename.cc @@ -0,0 +1,141 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/filename.h" + +#include +#include + +#include "db/dbformat.h" +#include "leveldb/env.h" +#include "util/logging.h" + +namespace leveldb { + +// A utility routine: write "data" to the named file and Sync() it. +Status WriteStringToFileSync(Env* env, const Slice& data, + const std::string& fname); + +static std::string MakeFileName(const std::string& dbname, uint64_t number, + const char* suffix) { + char buf[100]; + std::snprintf(buf, sizeof(buf), "/%06llu.%s", + static_cast(number), suffix); + return dbname + buf; +} + +std::string LogFileName(const std::string& dbname, uint64_t number) { + assert(number > 0); + return MakeFileName(dbname, number, "log"); +} + +std::string TableFileName(const std::string& dbname, uint64_t number) { + assert(number > 0); + return MakeFileName(dbname, number, "ldb"); +} + +std::string SSTTableFileName(const std::string& dbname, uint64_t number) { + assert(number > 0); + return MakeFileName(dbname, number, "sst"); +} + +std::string DescriptorFileName(const std::string& dbname, uint64_t number) { + assert(number > 0); + char buf[100]; + std::snprintf(buf, sizeof(buf), "/MANIFEST-%06llu", + static_cast(number)); + return dbname + buf; +} + +std::string CurrentFileName(const std::string& dbname) { + return dbname + "/CURRENT"; +} + +std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; } + +std::string TempFileName(const std::string& dbname, uint64_t number) { + assert(number > 0); + return MakeFileName(dbname, number, "dbtmp"); +} + +std::string InfoLogFileName(const std::string& dbname) { + return dbname + "/LOG"; +} + +// Return the name of the old info log file for "dbname". +std::string OldInfoLogFileName(const std::string& dbname) { + return dbname + "/LOG.old"; +} + +// Owned filenames have the form: +// dbname/CURRENT +// dbname/LOCK +// dbname/LOG +// dbname/LOG.old +// dbname/MANIFEST-[0-9]+ +// dbname/[0-9]+.(log|sst|ldb) +bool ParseFileName(const std::string& filename, uint64_t* number, + FileType* type) { + Slice rest(filename); + if (rest == "CURRENT") { + *number = 0; + *type = kCurrentFile; + } else if (rest == "LOCK") { + *number = 0; + *type = kDBLockFile; + } else if (rest == "LOG" || rest == "LOG.old") { + *number = 0; + *type = kInfoLogFile; + } else if (rest.starts_with("MANIFEST-")) { + rest.remove_prefix(strlen("MANIFEST-")); + uint64_t num; + if (!ConsumeDecimalNumber(&rest, &num)) { + return false; + } + if (!rest.empty()) { + return false; + } + *type = kDescriptorFile; + *number = num; + } else { + // Avoid strtoull() to keep filename format independent of the + // current locale + uint64_t num; + if (!ConsumeDecimalNumber(&rest, &num)) { + return false; + } + Slice suffix = rest; + if (suffix == Slice(".log")) { + *type = kLogFile; + } else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) { + *type = kTableFile; + } else if (suffix == Slice(".dbtmp")) { + *type = kTempFile; + } else { + return false; + } + *number = num; + } + return true; +} + +Status SetCurrentFile(Env* env, const std::string& dbname, + uint64_t descriptor_number) { + // Remove leading "dbname/" and add newline to manifest file name + std::string manifest = DescriptorFileName(dbname, descriptor_number); + Slice contents = manifest; + assert(contents.starts_with(dbname + "/")); + contents.remove_prefix(dbname.size() + 1); + std::string tmp = TempFileName(dbname, descriptor_number); + Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp); + if (s.ok()) { + s = env->RenameFile(tmp, CurrentFileName(dbname)); + } + if (!s.ok()) { + env->RemoveFile(tmp); + } + return s; +} + +} // namespace leveldb diff --git a/leveldb/db/filename.h b/leveldb/db/filename.h new file mode 100644 index 000000000..563c6d82f --- /dev/null +++ b/leveldb/db/filename.h @@ -0,0 +1,83 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// File names used by DB code + +#ifndef STORAGE_LEVELDB_DB_FILENAME_H_ +#define STORAGE_LEVELDB_DB_FILENAME_H_ + +#include +#include + +#include "leveldb/slice.h" +#include "leveldb/status.h" +#include "port/port.h" + +namespace leveldb { + +class Env; + +enum FileType { + kLogFile, + kDBLockFile, + kTableFile, + kDescriptorFile, + kCurrentFile, + kTempFile, + kInfoLogFile // Either the current one, or an old one +}; + +// Return the name of the log file with the specified number +// in the db named by "dbname". The result will be prefixed with +// "dbname". +std::string LogFileName(const std::string& dbname, uint64_t number); + +// Return the name of the sstable with the specified number +// in the db named by "dbname". The result will be prefixed with +// "dbname". +std::string TableFileName(const std::string& dbname, uint64_t number); + +// Return the legacy file name for an sstable with the specified number +// in the db named by "dbname". The result will be prefixed with +// "dbname". +std::string SSTTableFileName(const std::string& dbname, uint64_t number); + +// Return the name of the descriptor file for the db named by +// "dbname" and the specified incarnation number. The result will be +// prefixed with "dbname". +std::string DescriptorFileName(const std::string& dbname, uint64_t number); + +// Return the name of the current file. This file contains the name +// of the current manifest file. The result will be prefixed with +// "dbname". +std::string CurrentFileName(const std::string& dbname); + +// Return the name of the lock file for the db named by +// "dbname". The result will be prefixed with "dbname". +std::string LockFileName(const std::string& dbname); + +// Return the name of a temporary file owned by the db named "dbname". +// The result will be prefixed with "dbname". +std::string TempFileName(const std::string& dbname, uint64_t number); + +// Return the name of the info log file for "dbname". +std::string InfoLogFileName(const std::string& dbname); + +// Return the name of the old info log file for "dbname". +std::string OldInfoLogFileName(const std::string& dbname); + +// If filename is a leveldb file, store the type of the file in *type. +// The number encoded in the filename is stored in *number. If the +// filename was successfully parsed, returns true. Else return false. +bool ParseFileName(const std::string& filename, uint64_t* number, + FileType* type); + +// Make the CURRENT file point to the descriptor file with the +// specified number. +Status SetCurrentFile(Env* env, const std::string& dbname, + uint64_t descriptor_number); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_FILENAME_H_ diff --git a/leveldb/db/filename_test.cc b/leveldb/db/filename_test.cc new file mode 100644 index 000000000..9ac011155 --- /dev/null +++ b/leveldb/db/filename_test.cc @@ -0,0 +1,127 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/filename.h" + +#include "gtest/gtest.h" +#include "db/dbformat.h" +#include "port/port.h" +#include "util/logging.h" + +namespace leveldb { + +TEST(FileNameTest, Parse) { + Slice db; + FileType type; + uint64_t number; + + // Successful parses + static struct { + const char* fname; + uint64_t number; + FileType type; + } cases[] = { + {"100.log", 100, kLogFile}, + {"0.log", 0, kLogFile}, + {"0.sst", 0, kTableFile}, + {"0.ldb", 0, kTableFile}, + {"CURRENT", 0, kCurrentFile}, + {"LOCK", 0, kDBLockFile}, + {"MANIFEST-2", 2, kDescriptorFile}, + {"MANIFEST-7", 7, kDescriptorFile}, + {"LOG", 0, kInfoLogFile}, + {"LOG.old", 0, kInfoLogFile}, + {"18446744073709551615.log", 18446744073709551615ull, kLogFile}, + }; + for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + std::string f = cases[i].fname; + ASSERT_TRUE(ParseFileName(f, &number, &type)) << f; + ASSERT_EQ(cases[i].type, type) << f; + ASSERT_EQ(cases[i].number, number) << f; + } + + // Errors + static const char* errors[] = {"", + "foo", + "foo-dx-100.log", + ".log", + "", + "manifest", + "CURREN", + "CURRENTX", + "MANIFES", + "MANIFEST", + "MANIFEST-", + "XMANIFEST-3", + "MANIFEST-3x", + "LOC", + "LOCKx", + "LO", + "LOGx", + "18446744073709551616.log", + "184467440737095516150.log", + "100", + "100.", + "100.lop"}; + for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { + std::string f = errors[i]; + ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; + } +} + +TEST(FileNameTest, Construction) { + uint64_t number; + FileType type; + std::string fname; + + fname = CurrentFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0, number); + ASSERT_EQ(kCurrentFile, type); + + fname = LockFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0, number); + ASSERT_EQ(kDBLockFile, type); + + fname = LogFileName("foo", 192); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(192, number); + ASSERT_EQ(kLogFile, type); + + fname = TableFileName("bar", 200); + ASSERT_EQ("bar/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(200, number); + ASSERT_EQ(kTableFile, type); + + fname = DescriptorFileName("bar", 100); + ASSERT_EQ("bar/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(100, number); + ASSERT_EQ(kDescriptorFile, type); + + fname = TempFileName("tmp", 999); + ASSERT_EQ("tmp/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(999, number); + ASSERT_EQ(kTempFile, type); + + fname = InfoLogFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0, number); + ASSERT_EQ(kInfoLogFile, type); + + fname = OldInfoLogFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0, number); + ASSERT_EQ(kInfoLogFile, type); +} + +} // namespace leveldb diff --git a/leveldb/db/leveldbutil.cc b/leveldb/db/leveldbutil.cc new file mode 100644 index 000000000..95ee8976d --- /dev/null +++ b/leveldb/db/leveldbutil.cc @@ -0,0 +1,64 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "leveldb/dumpfile.h" +#include "leveldb/env.h" +#include "leveldb/status.h" + +namespace leveldb { +namespace { + +class StdoutPrinter : public WritableFile { + public: + Status Append(const Slice& data) override { + fwrite(data.data(), 1, data.size(), stdout); + return Status::OK(); + } + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } +}; + +bool HandleDumpCommand(Env* env, char** files, int num) { + StdoutPrinter printer; + bool ok = true; + for (int i = 0; i < num; i++) { + Status s = DumpFile(env, files[i], &printer); + if (!s.ok()) { + std::fprintf(stderr, "%s\n", s.ToString().c_str()); + ok = false; + } + } + return ok; +} + +} // namespace +} // namespace leveldb + +static void Usage() { + std::fprintf( + stderr, + "Usage: leveldbutil command...\n" + " dump files... -- dump contents of specified files\n"); +} + +int main(int argc, char** argv) { + leveldb::Env* env = leveldb::Env::Default(); + bool ok = true; + if (argc < 2) { + Usage(); + ok = false; + } else { + std::string command = argv[1]; + if (command == "dump") { + ok = leveldb::HandleDumpCommand(env, argv + 2, argc - 2); + } else { + Usage(); + ok = false; + } + } + return (ok ? 0 : 1); +} diff --git a/leveldb/db/log_format.h b/leveldb/db/log_format.h new file mode 100644 index 000000000..356e69fca --- /dev/null +++ b/leveldb/db/log_format.h @@ -0,0 +1,35 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Log format information shared by reader and writer. +// See ../doc/log_format.md for more detail. + +#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_ +#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_ + +namespace leveldb { +namespace log { + +enum RecordType { + // Zero is reserved for preallocated files + kZeroType = 0, + + kFullType = 1, + + // For fragments + kFirstType = 2, + kMiddleType = 3, + kLastType = 4 +}; +static const int kMaxRecordType = kLastType; + +static const int kBlockSize = 32768; + +// Header is checksum (4 bytes), length (2 bytes), type (1 byte). +static const int kHeaderSize = 4 + 2 + 1; + +} // namespace log +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_ diff --git a/leveldb/db/log_reader.cc b/leveldb/db/log_reader.cc new file mode 100644 index 000000000..988027919 --- /dev/null +++ b/leveldb/db/log_reader.cc @@ -0,0 +1,274 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/log_reader.h" + +#include + +#include "leveldb/env.h" +#include "util/coding.h" +#include "util/crc32c.h" + +namespace leveldb { +namespace log { + +Reader::Reporter::~Reporter() = default; + +Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, + uint64_t initial_offset) + : file_(file), + reporter_(reporter), + checksum_(checksum), + backing_store_(new char[kBlockSize]), + buffer_(), + eof_(false), + last_record_offset_(0), + end_of_buffer_offset_(0), + initial_offset_(initial_offset), + resyncing_(initial_offset > 0) {} + +Reader::~Reader() { delete[] backing_store_; } + +bool Reader::SkipToInitialBlock() { + const size_t offset_in_block = initial_offset_ % kBlockSize; + uint64_t block_start_location = initial_offset_ - offset_in_block; + + // Don't search a block if we'd be in the trailer + if (offset_in_block > kBlockSize - 6) { + block_start_location += kBlockSize; + } + + end_of_buffer_offset_ = block_start_location; + + // Skip to start of first block that can contain the initial record + if (block_start_location > 0) { + Status skip_status = file_->Skip(block_start_location); + if (!skip_status.ok()) { + ReportDrop(block_start_location, skip_status); + return false; + } + } + + return true; +} + +bool Reader::ReadRecord(Slice* record, std::string* scratch) { + if (last_record_offset_ < initial_offset_) { + if (!SkipToInitialBlock()) { + return false; + } + } + + scratch->clear(); + record->clear(); + bool in_fragmented_record = false; + // Record offset of the logical record that we're reading + // 0 is a dummy value to make compilers happy + uint64_t prospective_record_offset = 0; + + Slice fragment; + while (true) { + const unsigned int record_type = ReadPhysicalRecord(&fragment); + + // ReadPhysicalRecord may have only had an empty trailer remaining in its + // internal buffer. Calculate the offset of the next physical record now + // that it has returned, properly accounting for its header size. + uint64_t physical_record_offset = + end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size(); + + if (resyncing_) { + if (record_type == kMiddleType) { + continue; + } else if (record_type == kLastType) { + resyncing_ = false; + continue; + } else { + resyncing_ = false; + } + } + + switch (record_type) { + case kFullType: + if (in_fragmented_record) { + // Handle bug in earlier versions of log::Writer where + // it could emit an empty kFirstType record at the tail end + // of a block followed by a kFullType or kFirstType record + // at the beginning of the next block. + if (!scratch->empty()) { + ReportCorruption(scratch->size(), "partial record without end(1)"); + } + } + prospective_record_offset = physical_record_offset; + scratch->clear(); + *record = fragment; + last_record_offset_ = prospective_record_offset; + return true; + + case kFirstType: + if (in_fragmented_record) { + // Handle bug in earlier versions of log::Writer where + // it could emit an empty kFirstType record at the tail end + // of a block followed by a kFullType or kFirstType record + // at the beginning of the next block. + if (!scratch->empty()) { + ReportCorruption(scratch->size(), "partial record without end(2)"); + } + } + prospective_record_offset = physical_record_offset; + scratch->assign(fragment.data(), fragment.size()); + in_fragmented_record = true; + break; + + case kMiddleType: + if (!in_fragmented_record) { + ReportCorruption(fragment.size(), + "missing start of fragmented record(1)"); + } else { + scratch->append(fragment.data(), fragment.size()); + } + break; + + case kLastType: + if (!in_fragmented_record) { + ReportCorruption(fragment.size(), + "missing start of fragmented record(2)"); + } else { + scratch->append(fragment.data(), fragment.size()); + *record = Slice(*scratch); + last_record_offset_ = prospective_record_offset; + return true; + } + break; + + case kEof: + if (in_fragmented_record) { + // This can be caused by the writer dying immediately after + // writing a physical record but before completing the next; don't + // treat it as a corruption, just ignore the entire logical record. + scratch->clear(); + } + return false; + + case kBadRecord: + if (in_fragmented_record) { + ReportCorruption(scratch->size(), "error in middle of record"); + in_fragmented_record = false; + scratch->clear(); + } + break; + + default: { + char buf[40]; + std::snprintf(buf, sizeof(buf), "unknown record type %u", record_type); + ReportCorruption( + (fragment.size() + (in_fragmented_record ? scratch->size() : 0)), + buf); + in_fragmented_record = false; + scratch->clear(); + break; + } + } + } + return false; +} + +uint64_t Reader::LastRecordOffset() { return last_record_offset_; } + +void Reader::ReportCorruption(uint64_t bytes, const char* reason) { + ReportDrop(bytes, Status::Corruption(reason)); +} + +void Reader::ReportDrop(uint64_t bytes, const Status& reason) { + if (reporter_ != nullptr && + end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) { + reporter_->Corruption(static_cast(bytes), reason); + } +} + +unsigned int Reader::ReadPhysicalRecord(Slice* result) { + while (true) { + if (buffer_.size() < kHeaderSize) { + if (!eof_) { + // Last read was a full read, so this is a trailer to skip + buffer_.clear(); + Status status = file_->Read(kBlockSize, &buffer_, backing_store_); + end_of_buffer_offset_ += buffer_.size(); + if (!status.ok()) { + buffer_.clear(); + ReportDrop(kBlockSize, status); + eof_ = true; + return kEof; + } else if (buffer_.size() < kBlockSize) { + eof_ = true; + } + continue; + } else { + // Note that if buffer_ is non-empty, we have a truncated header at the + // end of the file, which can be caused by the writer crashing in the + // middle of writing the header. Instead of considering this an error, + // just report EOF. + buffer_.clear(); + return kEof; + } + } + + // Parse the header + const char* header = buffer_.data(); + const uint32_t a = static_cast(header[4]) & 0xff; + const uint32_t b = static_cast(header[5]) & 0xff; + const unsigned int type = header[6]; + const uint32_t length = a | (b << 8); + if (kHeaderSize + length > buffer_.size()) { + size_t drop_size = buffer_.size(); + buffer_.clear(); + if (!eof_) { + ReportCorruption(drop_size, "bad record length"); + return kBadRecord; + } + // If the end of the file has been reached without reading |length| bytes + // of payload, assume the writer died in the middle of writing the record. + // Don't report a corruption. + return kEof; + } + + if (type == kZeroType && length == 0) { + // Skip zero length record without reporting any drops since + // such records are produced by the mmap based writing code in + // env_posix.cc that preallocates file regions. + buffer_.clear(); + return kBadRecord; + } + + // Check crc + if (checksum_) { + uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header)); + uint32_t actual_crc = crc32c::Value(header + 6, 1 + length); + if (actual_crc != expected_crc) { + // Drop the rest of the buffer since "length" itself may have + // been corrupted and if we trust it, we could find some + // fragment of a real log record that just happens to look + // like a valid log record. + size_t drop_size = buffer_.size(); + buffer_.clear(); + ReportCorruption(drop_size, "checksum mismatch"); + return kBadRecord; + } + } + + buffer_.remove_prefix(kHeaderSize + length); + + // Skip physical record that started before initial_offset_ + if (end_of_buffer_offset_ - buffer_.size() - kHeaderSize - length < + initial_offset_) { + result->clear(); + return kBadRecord; + } + + *result = Slice(header + kHeaderSize, length); + return type; + } +} + +} // namespace log +} // namespace leveldb diff --git a/leveldb/db/log_reader.h b/leveldb/db/log_reader.h new file mode 100644 index 000000000..ba711f88c --- /dev/null +++ b/leveldb/db/log_reader.h @@ -0,0 +1,112 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_LOG_READER_H_ +#define STORAGE_LEVELDB_DB_LOG_READER_H_ + +#include + +#include "db/log_format.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" + +namespace leveldb { + +class SequentialFile; + +namespace log { + +class Reader { + public: + // Interface for reporting errors. + class Reporter { + public: + virtual ~Reporter(); + + // Some corruption was detected. "bytes" is the approximate number + // of bytes dropped due to the corruption. + virtual void Corruption(size_t bytes, const Status& status) = 0; + }; + + // Create a reader that will return log records from "*file". + // "*file" must remain live while this Reader is in use. + // + // If "reporter" is non-null, it is notified whenever some data is + // dropped due to a detected corruption. "*reporter" must remain + // live while this Reader is in use. + // + // If "checksum" is true, verify checksums if available. + // + // The Reader will start reading at the first record located at physical + // position >= initial_offset within the file. + Reader(SequentialFile* file, Reporter* reporter, bool checksum, + uint64_t initial_offset); + + Reader(const Reader&) = delete; + Reader& operator=(const Reader&) = delete; + + ~Reader(); + + // Read the next record into *record. Returns true if read + // successfully, false if we hit end of the input. May use + // "*scratch" as temporary storage. The contents filled in *record + // will only be valid until the next mutating operation on this + // reader or the next mutation to *scratch. + bool ReadRecord(Slice* record, std::string* scratch); + + // Returns the physical offset of the last record returned by ReadRecord. + // + // Undefined before the first call to ReadRecord. + uint64_t LastRecordOffset(); + + private: + // Extend record types with the following special values + enum { + kEof = kMaxRecordType + 1, + // Returned whenever we find an invalid physical record. + // Currently there are three situations in which this happens: + // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) + // * The record is a 0-length record (No drop is reported) + // * The record is below constructor's initial_offset (No drop is reported) + kBadRecord = kMaxRecordType + 2 + }; + + // Skips all blocks that are completely before "initial_offset_". + // + // Returns true on success. Handles reporting. + bool SkipToInitialBlock(); + + // Return type, or one of the preceding special values + unsigned int ReadPhysicalRecord(Slice* result); + + // Reports dropped bytes to the reporter. + // buffer_ must be updated to remove the dropped bytes prior to invocation. + void ReportCorruption(uint64_t bytes, const char* reason); + void ReportDrop(uint64_t bytes, const Status& reason); + + SequentialFile* const file_; + Reporter* const reporter_; + bool const checksum_; + char* const backing_store_; + Slice buffer_; + bool eof_; // Last Read() indicated EOF by returning < kBlockSize + + // Offset of the last record returned by ReadRecord. + uint64_t last_record_offset_; + // Offset of the first location past the end of buffer_. + uint64_t end_of_buffer_offset_; + + // Offset at which to start looking for the first record to return + uint64_t const initial_offset_; + + // True if we are resynchronizing after a seek (initial_offset_ > 0). In + // particular, a run of kMiddleType and kLastType records can be silently + // skipped in this mode + bool resyncing_; +}; + +} // namespace log +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_LOG_READER_H_ diff --git a/leveldb/db/log_test.cc b/leveldb/db/log_test.cc new file mode 100644 index 000000000..d55d4dd70 --- /dev/null +++ b/leveldb/db/log_test.cc @@ -0,0 +1,558 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "gtest/gtest.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "leveldb/env.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/random.h" + +namespace leveldb { +namespace log { + +// Construct a string of the specified length made out of the supplied +// partial string. +static std::string BigString(const std::string& partial_string, size_t n) { + std::string result; + while (result.size() < n) { + result.append(partial_string); + } + result.resize(n); + return result; +} + +// Construct a string from a number +static std::string NumberString(int n) { + char buf[50]; + std::snprintf(buf, sizeof(buf), "%d.", n); + return std::string(buf); +} + +// Return a skewed potentially long string +static std::string RandomSkewedString(int i, Random* rnd) { + return BigString(NumberString(i), rnd->Skewed(17)); +} + +class LogTest : public testing::Test { + public: + LogTest() + : reading_(false), + writer_(new Writer(&dest_)), + reader_(new Reader(&source_, &report_, true /*checksum*/, + 0 /*initial_offset*/)) {} + + ~LogTest() { + delete writer_; + delete reader_; + } + + void ReopenForAppend() { + delete writer_; + writer_ = new Writer(&dest_, dest_.contents_.size()); + } + + void Write(const std::string& msg) { + ASSERT_TRUE(!reading_) << "Write() after starting to read"; + writer_->AddRecord(Slice(msg)); + } + + size_t WrittenBytes() const { return dest_.contents_.size(); } + + std::string Read() { + if (!reading_) { + reading_ = true; + source_.contents_ = Slice(dest_.contents_); + } + std::string scratch; + Slice record; + if (reader_->ReadRecord(&record, &scratch)) { + return record.ToString(); + } else { + return "EOF"; + } + } + + void IncrementByte(int offset, int delta) { + dest_.contents_[offset] += delta; + } + + void SetByte(int offset, char new_byte) { + dest_.contents_[offset] = new_byte; + } + + void ShrinkSize(int bytes) { + dest_.contents_.resize(dest_.contents_.size() - bytes); + } + + void FixChecksum(int header_offset, int len) { + // Compute crc of type/len/data + uint32_t crc = crc32c::Value(&dest_.contents_[header_offset + 6], 1 + len); + crc = crc32c::Mask(crc); + EncodeFixed32(&dest_.contents_[header_offset], crc); + } + + void ForceError() { source_.force_error_ = true; } + + size_t DroppedBytes() const { return report_.dropped_bytes_; } + + std::string ReportMessage() const { return report_.message_; } + + // Returns OK iff recorded error message contains "msg" + std::string MatchError(const std::string& msg) const { + if (report_.message_.find(msg) == std::string::npos) { + return report_.message_; + } else { + return "OK"; + } + } + + void WriteInitialOffsetLog() { + for (int i = 0; i < num_initial_offset_records_; i++) { + std::string record(initial_offset_record_sizes_[i], + static_cast('a' + i)); + Write(record); + } + } + + void StartReadingAt(uint64_t initial_offset) { + delete reader_; + reader_ = new Reader(&source_, &report_, true /*checksum*/, initial_offset); + } + + void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) { + WriteInitialOffsetLog(); + reading_ = true; + source_.contents_ = Slice(dest_.contents_); + Reader* offset_reader = new Reader(&source_, &report_, true /*checksum*/, + WrittenBytes() + offset_past_end); + Slice record; + std::string scratch; + ASSERT_TRUE(!offset_reader->ReadRecord(&record, &scratch)); + delete offset_reader; + } + + void CheckInitialOffsetRecord(uint64_t initial_offset, + int expected_record_offset) { + WriteInitialOffsetLog(); + reading_ = true; + source_.contents_ = Slice(dest_.contents_); + Reader* offset_reader = + new Reader(&source_, &report_, true /*checksum*/, initial_offset); + + // Read all records from expected_record_offset through the last one. + ASSERT_LT(expected_record_offset, num_initial_offset_records_); + for (; expected_record_offset < num_initial_offset_records_; + ++expected_record_offset) { + Slice record; + std::string scratch; + ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch)); + ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset], + record.size()); + ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset], + offset_reader->LastRecordOffset()); + ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]); + } + delete offset_reader; + } + + private: + class StringDest : public WritableFile { + public: + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + Status Append(const Slice& slice) override { + contents_.append(slice.data(), slice.size()); + return Status::OK(); + } + + std::string contents_; + }; + + class StringSource : public SequentialFile { + public: + StringSource() : force_error_(false), returned_partial_(false) {} + + Status Read(size_t n, Slice* result, char* scratch) override { + EXPECT_TRUE(!returned_partial_) << "must not Read() after eof/error"; + + if (force_error_) { + force_error_ = false; + returned_partial_ = true; + return Status::Corruption("read error"); + } + + if (contents_.size() < n) { + n = contents_.size(); + returned_partial_ = true; + } + *result = Slice(contents_.data(), n); + contents_.remove_prefix(n); + return Status::OK(); + } + + Status Skip(uint64_t n) override { + if (n > contents_.size()) { + contents_.clear(); + return Status::NotFound("in-memory file skipped past end"); + } + + contents_.remove_prefix(n); + + return Status::OK(); + } + + Slice contents_; + bool force_error_; + bool returned_partial_; + }; + + class ReportCollector : public Reader::Reporter { + public: + ReportCollector() : dropped_bytes_(0) {} + void Corruption(size_t bytes, const Status& status) override { + dropped_bytes_ += bytes; + message_.append(status.ToString()); + } + + size_t dropped_bytes_; + std::string message_; + }; + + // Record metadata for testing initial offset functionality + static size_t initial_offset_record_sizes_[]; + static uint64_t initial_offset_last_record_offsets_[]; + static int num_initial_offset_records_; + + StringDest dest_; + StringSource source_; + ReportCollector report_; + bool reading_; + Writer* writer_; + Reader* reader_; +}; + +size_t LogTest::initial_offset_record_sizes_[] = { + 10000, // Two sizable records in first block + 10000, + 2 * log::kBlockSize - 1000, // Span three blocks + 1, + 13716, // Consume all but two bytes of block 3. + log::kBlockSize - kHeaderSize, // Consume the entirety of block 4. +}; + +uint64_t LogTest::initial_offset_last_record_offsets_[] = { + 0, + kHeaderSize + 10000, + 2 * (kHeaderSize + 10000), + 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, + 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize + + kHeaderSize + 1, + 3 * log::kBlockSize, +}; + +// LogTest::initial_offset_last_record_offsets_ must be defined before this. +int LogTest::num_initial_offset_records_ = + sizeof(LogTest::initial_offset_last_record_offsets_) / sizeof(uint64_t); + +TEST_F(LogTest, Empty) { ASSERT_EQ("EOF", Read()); } + +TEST_F(LogTest, ReadWrite) { + Write("foo"); + Write("bar"); + Write(""); + Write("xxxx"); + ASSERT_EQ("foo", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("xxxx", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("EOF", Read()); // Make sure reads at eof work +} + +TEST_F(LogTest, ManyBlocks) { + for (int i = 0; i < 100000; i++) { + Write(NumberString(i)); + } + for (int i = 0; i < 100000; i++) { + ASSERT_EQ(NumberString(i), Read()); + } + ASSERT_EQ("EOF", Read()); +} + +TEST_F(LogTest, Fragmentation) { + Write("small"); + Write(BigString("medium", 50000)); + Write(BigString("large", 100000)); + ASSERT_EQ("small", Read()); + ASSERT_EQ(BigString("medium", 50000), Read()); + ASSERT_EQ(BigString("large", 100000), Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_F(LogTest, MarginalTrailer) { + // Make a trailer that is exactly the same length as an empty record. + const int n = kBlockSize - 2 * kHeaderSize; + Write(BigString("foo", n)); + ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes()); + Write(""); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_F(LogTest, MarginalTrailer2) { + // Make a trailer that is exactly the same length as an empty record. + const int n = kBlockSize - 2 * kHeaderSize; + Write(BigString("foo", n)); + ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes()); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); +} + +TEST_F(LogTest, ShortTrailer) { + const int n = kBlockSize - 2 * kHeaderSize + 4; + Write(BigString("foo", n)); + ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); + Write(""); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_F(LogTest, AlignedEof) { + const int n = kBlockSize - 2 * kHeaderSize + 4; + Write(BigString("foo", n)); + ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_F(LogTest, OpenForAppend) { + Write("hello"); + ReopenForAppend(); + Write("world"); + ASSERT_EQ("hello", Read()); + ASSERT_EQ("world", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_F(LogTest, RandomRead) { + const int N = 500; + Random write_rnd(301); + for (int i = 0; i < N; i++) { + Write(RandomSkewedString(i, &write_rnd)); + } + Random read_rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_EQ(RandomSkewedString(i, &read_rnd), Read()); + } + ASSERT_EQ("EOF", Read()); +} + +// Tests of all the error paths in log_reader.cc follow: + +TEST_F(LogTest, ReadError) { + Write("foo"); + ForceError(); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(kBlockSize, DroppedBytes()); + ASSERT_EQ("OK", MatchError("read error")); +} + +TEST_F(LogTest, BadRecordType) { + Write("foo"); + // Type is stored in header[6] + IncrementByte(6, 100); + FixChecksum(0, 3); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("unknown record type")); +} + +TEST_F(LogTest, TruncatedTrailingRecordIsIgnored) { + Write("foo"); + ShrinkSize(4); // Drop all payload as well as a header byte + ASSERT_EQ("EOF", Read()); + // Truncated last record is ignored, not treated as an error. + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); +} + +TEST_F(LogTest, BadLength) { + const int kPayloadSize = kBlockSize - kHeaderSize; + Write(BigString("bar", kPayloadSize)); + Write("foo"); + // Least significant size byte is stored in header[4]. + IncrementByte(4, 1); + ASSERT_EQ("foo", Read()); + ASSERT_EQ(kBlockSize, DroppedBytes()); + ASSERT_EQ("OK", MatchError("bad record length")); +} + +TEST_F(LogTest, BadLengthAtEndIsIgnored) { + Write("foo"); + ShrinkSize(1); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); +} + +TEST_F(LogTest, ChecksumMismatch) { + Write("foo"); + IncrementByte(0, 10); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(10, DroppedBytes()); + ASSERT_EQ("OK", MatchError("checksum mismatch")); +} + +TEST_F(LogTest, UnexpectedMiddleType) { + Write("foo"); + SetByte(6, kMiddleType); + FixChecksum(0, 3); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("missing start")); +} + +TEST_F(LogTest, UnexpectedLastType) { + Write("foo"); + SetByte(6, kLastType); + FixChecksum(0, 3); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("missing start")); +} + +TEST_F(LogTest, UnexpectedFullType) { + Write("foo"); + Write("bar"); + SetByte(6, kFirstType); + FixChecksum(0, 3); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("partial record without end")); +} + +TEST_F(LogTest, UnexpectedFirstType) { + Write("foo"); + Write(BigString("bar", 100000)); + SetByte(6, kFirstType); + FixChecksum(0, 3); + ASSERT_EQ(BigString("bar", 100000), Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3, DroppedBytes()); + ASSERT_EQ("OK", MatchError("partial record without end")); +} + +TEST_F(LogTest, MissingLastIsIgnored) { + Write(BigString("bar", kBlockSize)); + // Remove the LAST block, including header. + ShrinkSize(14); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0, DroppedBytes()); +} + +TEST_F(LogTest, PartialLastIsIgnored) { + Write(BigString("bar", kBlockSize)); + // Cause a bad record length in the LAST block. + ShrinkSize(1); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0, DroppedBytes()); +} + +TEST_F(LogTest, SkipIntoMultiRecord) { + // Consider a fragmented record: + // first(R1), middle(R1), last(R1), first(R2) + // If initial_offset points to a record after first(R1) but before first(R2) + // incomplete fragment errors are not actual errors, and must be suppressed + // until a new first or full record is encountered. + Write(BigString("foo", 3 * kBlockSize)); + Write("correct"); + StartReadingAt(kBlockSize); + + ASSERT_EQ("correct", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("EOF", Read()); +} + +TEST_F(LogTest, ErrorJoinsRecords) { + // Consider two fragmented records: + // first(R1) last(R1) first(R2) last(R2) + // where the middle two fragments disappear. We do not want + // first(R1),last(R2) to get joined and returned as a valid record. + + // Write records that span two blocks + Write(BigString("foo", kBlockSize)); + Write(BigString("bar", kBlockSize)); + Write("correct"); + + // Wipe the middle block + for (int offset = kBlockSize; offset < 2 * kBlockSize; offset++) { + SetByte(offset, 'x'); + } + + ASSERT_EQ("correct", Read()); + ASSERT_EQ("EOF", Read()); + const size_t dropped = DroppedBytes(); + ASSERT_LE(dropped, 2 * kBlockSize + 100); + ASSERT_GE(dropped, 2 * kBlockSize); +} + +TEST_F(LogTest, ReadStart) { CheckInitialOffsetRecord(0, 0); } + +TEST_F(LogTest, ReadSecondOneOff) { CheckInitialOffsetRecord(1, 1); } + +TEST_F(LogTest, ReadSecondTenThousand) { CheckInitialOffsetRecord(10000, 1); } + +TEST_F(LogTest, ReadSecondStart) { CheckInitialOffsetRecord(10007, 1); } + +TEST_F(LogTest, ReadThirdOneOff) { CheckInitialOffsetRecord(10008, 2); } + +TEST_F(LogTest, ReadThirdStart) { CheckInitialOffsetRecord(20014, 2); } + +TEST_F(LogTest, ReadFourthOneOff) { CheckInitialOffsetRecord(20015, 3); } + +TEST_F(LogTest, ReadFourthFirstBlockTrailer) { + CheckInitialOffsetRecord(log::kBlockSize - 4, 3); +} + +TEST_F(LogTest, ReadFourthMiddleBlock) { + CheckInitialOffsetRecord(log::kBlockSize + 1, 3); +} + +TEST_F(LogTest, ReadFourthLastBlock) { + CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3); +} + +TEST_F(LogTest, ReadFourthStart) { + CheckInitialOffsetRecord( + 2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, + 3); +} + +TEST_F(LogTest, ReadInitialOffsetIntoBlockPadding) { + CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5); +} + +TEST_F(LogTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); } + +TEST_F(LogTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); } + +} // namespace log +} // namespace leveldb diff --git a/leveldb/db/log_writer.cc b/leveldb/db/log_writer.cc new file mode 100644 index 000000000..ad66bfb8a --- /dev/null +++ b/leveldb/db/log_writer.cc @@ -0,0 +1,111 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/log_writer.h" + +#include + +#include "leveldb/env.h" +#include "util/coding.h" +#include "util/crc32c.h" + +namespace leveldb { +namespace log { + +static void InitTypeCrc(uint32_t* type_crc) { + for (int i = 0; i <= kMaxRecordType; i++) { + char t = static_cast(i); + type_crc[i] = crc32c::Value(&t, 1); + } +} + +Writer::Writer(WritableFile* dest) : dest_(dest), block_offset_(0) { + InitTypeCrc(type_crc_); +} + +Writer::Writer(WritableFile* dest, uint64_t dest_length) + : dest_(dest), block_offset_(dest_length % kBlockSize) { + InitTypeCrc(type_crc_); +} + +Writer::~Writer() = default; + +Status Writer::AddRecord(const Slice& slice) { + const char* ptr = slice.data(); + size_t left = slice.size(); + + // Fragment the record if necessary and emit it. Note that if slice + // is empty, we still want to iterate once to emit a single + // zero-length record + Status s; + bool begin = true; + do { + const int leftover = kBlockSize - block_offset_; + assert(leftover >= 0); + if (leftover < kHeaderSize) { + // Switch to a new block + if (leftover > 0) { + // Fill the trailer (literal below relies on kHeaderSize being 7) + static_assert(kHeaderSize == 7, ""); + dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover)); + } + block_offset_ = 0; + } + + // Invariant: we never leave < kHeaderSize bytes in a block. + assert(kBlockSize - block_offset_ - kHeaderSize >= 0); + + const size_t avail = kBlockSize - block_offset_ - kHeaderSize; + const size_t fragment_length = (left < avail) ? left : avail; + + RecordType type; + const bool end = (left == fragment_length); + if (begin && end) { + type = kFullType; + } else if (begin) { + type = kFirstType; + } else if (end) { + type = kLastType; + } else { + type = kMiddleType; + } + + s = EmitPhysicalRecord(type, ptr, fragment_length); + ptr += fragment_length; + left -= fragment_length; + begin = false; + } while (s.ok() && left > 0); + return s; +} + +Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, + size_t length) { + assert(length <= 0xffff); // Must fit in two bytes + assert(block_offset_ + kHeaderSize + length <= kBlockSize); + + // Format the header + char buf[kHeaderSize]; + buf[4] = static_cast(length & 0xff); + buf[5] = static_cast(length >> 8); + buf[6] = static_cast(t); + + // Compute the crc of the record type and the payload. + uint32_t crc = crc32c::Extend(type_crc_[t], ptr, length); + crc = crc32c::Mask(crc); // Adjust for storage + EncodeFixed32(buf, crc); + + // Write the header and the payload + Status s = dest_->Append(Slice(buf, kHeaderSize)); + if (s.ok()) { + s = dest_->Append(Slice(ptr, length)); + if (s.ok()) { + s = dest_->Flush(); + } + } + block_offset_ += kHeaderSize + length; + return s; +} + +} // namespace log +} // namespace leveldb diff --git a/leveldb/db/log_writer.h b/leveldb/db/log_writer.h new file mode 100644 index 000000000..ad36794db --- /dev/null +++ b/leveldb/db/log_writer.h @@ -0,0 +1,54 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_ +#define STORAGE_LEVELDB_DB_LOG_WRITER_H_ + +#include + +#include "db/log_format.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" + +namespace leveldb { + +class WritableFile; + +namespace log { + +class Writer { + public: + // Create a writer that will append data to "*dest". + // "*dest" must be initially empty. + // "*dest" must remain live while this Writer is in use. + explicit Writer(WritableFile* dest); + + // Create a writer that will append data to "*dest". + // "*dest" must have initial length "dest_length". + // "*dest" must remain live while this Writer is in use. + Writer(WritableFile* dest, uint64_t dest_length); + + Writer(const Writer&) = delete; + Writer& operator=(const Writer&) = delete; + + ~Writer(); + + Status AddRecord(const Slice& slice); + + private: + Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); + + WritableFile* dest_; + int block_offset_; // Current offset in block + + // crc32c values for all supported record types. These are + // pre-computed to reduce the overhead of computing the crc of the + // record type stored in the header. + uint32_t type_crc_[kMaxRecordType + 1]; +}; + +} // namespace log +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_ diff --git a/leveldb/db/memtable.cc b/leveldb/db/memtable.cc new file mode 100644 index 000000000..4f09340e0 --- /dev/null +++ b/leveldb/db/memtable.cc @@ -0,0 +1,138 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/memtable.h" +#include "db/dbformat.h" +#include "leveldb/comparator.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "util/coding.h" + +namespace leveldb { + +static Slice GetLengthPrefixedSlice(const char* data) { + uint32_t len; + const char* p = data; + p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted + return Slice(p, len); +} + +MemTable::MemTable(const InternalKeyComparator& comparator) + : comparator_(comparator), refs_(0), table_(comparator_, &arena_) {} + +MemTable::~MemTable() { assert(refs_ == 0); } + +size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); } + +int MemTable::KeyComparator::operator()(const char* aptr, + const char* bptr) const { + // Internal keys are encoded as length-prefixed strings. + Slice a = GetLengthPrefixedSlice(aptr); + Slice b = GetLengthPrefixedSlice(bptr); + return comparator.Compare(a, b); +} + +// Encode a suitable internal key target for "target" and return it. +// Uses *scratch as scratch space, and the returned pointer will point +// into this scratch space. +static const char* EncodeKey(std::string* scratch, const Slice& target) { + scratch->clear(); + PutVarint32(scratch, target.size()); + scratch->append(target.data(), target.size()); + return scratch->data(); +} + +class MemTableIterator : public Iterator { + public: + explicit MemTableIterator(MemTable::Table* table) : iter_(table) {} + + MemTableIterator(const MemTableIterator&) = delete; + MemTableIterator& operator=(const MemTableIterator&) = delete; + + ~MemTableIterator() override = default; + + bool Valid() const override { return iter_.Valid(); } + void Seek(const Slice& k) override { iter_.Seek(EncodeKey(&tmp_, k)); } + void SeekToFirst() override { iter_.SeekToFirst(); } + void SeekToLast() override { iter_.SeekToLast(); } + void Next() override { iter_.Next(); } + void Prev() override { iter_.Prev(); } + Slice key() const override { return GetLengthPrefixedSlice(iter_.key()); } + Slice value() const override { + Slice key_slice = GetLengthPrefixedSlice(iter_.key()); + return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); + } + + Status status() const override { return Status::OK(); } + + private: + MemTable::Table::Iterator iter_; + std::string tmp_; // For passing to EncodeKey +}; + +Iterator* MemTable::NewIterator() { return new MemTableIterator(&table_); } + +void MemTable::Add(SequenceNumber s, ValueType type, const Slice& key, + const Slice& value) { + // Format of an entry is concatenation of: + // key_size : varint32 of internal_key.size() + // key bytes : char[internal_key.size()] + // tag : uint64((sequence << 8) | type) + // value_size : varint32 of value.size() + // value bytes : char[value.size()] + size_t key_size = key.size(); + size_t val_size = value.size(); + size_t internal_key_size = key_size + 8; + const size_t encoded_len = VarintLength(internal_key_size) + + internal_key_size + VarintLength(val_size) + + val_size; + char* buf = arena_.Allocate(encoded_len); + char* p = EncodeVarint32(buf, internal_key_size); + std::memcpy(p, key.data(), key_size); + p += key_size; + EncodeFixed64(p, (s << 8) | type); + p += 8; + p = EncodeVarint32(p, val_size); + std::memcpy(p, value.data(), val_size); + assert(p + val_size == buf + encoded_len); + table_.Insert(buf); +} + +bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) { + Slice memkey = key.memtable_key(); + Table::Iterator iter(&table_); + iter.Seek(memkey.data()); + if (iter.Valid()) { + // entry format is: + // klength varint32 + // userkey char[klength] + // tag uint64 + // vlength varint32 + // value char[vlength] + // Check that it belongs to same user key. We do not check the + // sequence number since the Seek() call above should have skipped + // all entries with overly large sequence numbers. + const char* entry = iter.key(); + uint32_t key_length; + const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + if (comparator_.comparator.user_comparator()->Compare( + Slice(key_ptr, key_length - 8), key.user_key()) == 0) { + // Correct user key + const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); + switch (static_cast(tag & 0xff)) { + case kTypeValue: { + Slice v = GetLengthPrefixedSlice(key_ptr + key_length); + value->assign(v.data(), v.size()); + return true; + } + case kTypeDeletion: + *s = Status::NotFound(Slice()); + return true; + } + } + } + return false; +} + +} // namespace leveldb diff --git a/leveldb/db/memtable.h b/leveldb/db/memtable.h new file mode 100644 index 000000000..9d986b107 --- /dev/null +++ b/leveldb/db/memtable.h @@ -0,0 +1,87 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_ +#define STORAGE_LEVELDB_DB_MEMTABLE_H_ + +#include + +#include "db/dbformat.h" +#include "db/skiplist.h" +#include "leveldb/db.h" +#include "util/arena.h" + +namespace leveldb { + +class InternalKeyComparator; +class MemTableIterator; + +class MemTable { + public: + // MemTables are reference counted. The initial reference count + // is zero and the caller must call Ref() at least once. + explicit MemTable(const InternalKeyComparator& comparator); + + MemTable(const MemTable&) = delete; + MemTable& operator=(const MemTable&) = delete; + + // Increase reference count. + void Ref() { ++refs_; } + + // Drop reference count. Delete if no more references exist. + void Unref() { + --refs_; + assert(refs_ >= 0); + if (refs_ <= 0) { + delete this; + } + } + + // Returns an estimate of the number of bytes of data in use by this + // data structure. It is safe to call when MemTable is being modified. + size_t ApproximateMemoryUsage(); + + // Return an iterator that yields the contents of the memtable. + // + // The caller must ensure that the underlying MemTable remains live + // while the returned iterator is live. The keys returned by this + // iterator are internal keys encoded by AppendInternalKey in the + // db/format.{h,cc} module. + Iterator* NewIterator(); + + // Add an entry into memtable that maps key to value at the + // specified sequence number and with the specified type. + // Typically value will be empty if type==kTypeDeletion. + void Add(SequenceNumber seq, ValueType type, const Slice& key, + const Slice& value); + + // If memtable contains a value for key, store it in *value and return true. + // If memtable contains a deletion for key, store a NotFound() error + // in *status and return true. + // Else, return false. + bool Get(const LookupKey& key, std::string* value, Status* s); + + private: + friend class MemTableIterator; + friend class MemTableBackwardIterator; + + struct KeyComparator { + const InternalKeyComparator comparator; + explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {} + int operator()(const char* a, const char* b) const; + }; + + typedef SkipList Table; + + ~MemTable(); // Private since only Unref() should be used to delete it + + KeyComparator comparator_; + int refs_; + Arena arena_; + Table table_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_MEMTABLE_H_ diff --git a/leveldb/db/recovery_test.cc b/leveldb/db/recovery_test.cc new file mode 100644 index 000000000..8dc039ac7 --- /dev/null +++ b/leveldb/db/recovery_test.cc @@ -0,0 +1,339 @@ +// Copyright (c) 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "gtest/gtest.h" +#include "db/db_impl.h" +#include "db/filename.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" +#include "util/testutil.h" + +namespace leveldb { + +class RecoveryTest : public testing::Test { + public: + RecoveryTest() : env_(Env::Default()), db_(nullptr) { + dbname_ = testing::TempDir() + "recovery_test"; + DestroyDB(dbname_, Options()); + Open(); + } + + ~RecoveryTest() { + Close(); + DestroyDB(dbname_, Options()); + } + + DBImpl* dbfull() const { return reinterpret_cast(db_); } + Env* env() const { return env_; } + + bool CanAppend() { + WritableFile* tmp; + Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp); + delete tmp; + if (s.IsNotSupportedError()) { + return false; + } else { + return true; + } + } + + void Close() { + delete db_; + db_ = nullptr; + } + + Status OpenWithStatus(Options* options = nullptr) { + Close(); + Options opts; + if (options != nullptr) { + opts = *options; + } else { + opts.reuse_logs = true; // TODO(sanjay): test both ways + opts.create_if_missing = true; + } + if (opts.env == nullptr) { + opts.env = env_; + } + return DB::Open(opts, dbname_, &db_); + } + + void Open(Options* options = nullptr) { + ASSERT_LEVELDB_OK(OpenWithStatus(options)); + ASSERT_EQ(1, NumLogs()); + } + + Status Put(const std::string& k, const std::string& v) { + return db_->Put(WriteOptions(), k, v); + } + + std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { + std::string result; + Status s = db_->Get(ReadOptions(), k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + std::string ManifestFileName() { + std::string current; + EXPECT_LEVELDB_OK( + ReadFileToString(env_, CurrentFileName(dbname_), ¤t)); + size_t len = current.size(); + if (len > 0 && current[len - 1] == '\n') { + current.resize(len - 1); + } + return dbname_ + "/" + current; + } + + std::string LogName(uint64_t number) { return LogFileName(dbname_, number); } + + size_t RemoveLogFiles() { + // Linux allows unlinking open files, but Windows does not. + // Closing the db allows for file deletion. + Close(); + std::vector logs = GetFiles(kLogFile); + for (size_t i = 0; i < logs.size(); i++) { + EXPECT_LEVELDB_OK(env_->RemoveFile(LogName(logs[i]))) << LogName(logs[i]); + } + return logs.size(); + } + + void RemoveManifestFile() { + ASSERT_LEVELDB_OK(env_->RemoveFile(ManifestFileName())); + } + + uint64_t FirstLogFile() { return GetFiles(kLogFile)[0]; } + + std::vector GetFiles(FileType t) { + std::vector filenames; + EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames)); + std::vector result; + for (size_t i = 0; i < filenames.size(); i++) { + uint64_t number; + FileType type; + if (ParseFileName(filenames[i], &number, &type) && type == t) { + result.push_back(number); + } + } + return result; + } + + int NumLogs() { return GetFiles(kLogFile).size(); } + + int NumTables() { return GetFiles(kTableFile).size(); } + + uint64_t FileSize(const std::string& fname) { + uint64_t result; + EXPECT_LEVELDB_OK(env_->GetFileSize(fname, &result)) << fname; + return result; + } + + void CompactMemTable() { dbfull()->TEST_CompactMemTable(); } + + // Directly construct a log file that sets key to val. + void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) { + std::string fname = LogFileName(dbname_, lognum); + WritableFile* file; + ASSERT_LEVELDB_OK(env_->NewWritableFile(fname, &file)); + log::Writer writer(file); + WriteBatch batch; + batch.Put(key, val); + WriteBatchInternal::SetSequence(&batch, seq); + ASSERT_LEVELDB_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); + ASSERT_LEVELDB_OK(file->Flush()); + delete file; + } + + private: + std::string dbname_; + Env* env_; + DB* db_; +}; + +TEST_F(RecoveryTest, ManifestReused) { + if (!CanAppend()) { + std::fprintf(stderr, + "skipping test because env does not support appending\n"); + return; + } + ASSERT_LEVELDB_OK(Put("foo", "bar")); + Close(); + std::string old_manifest = ManifestFileName(); + Open(); + ASSERT_EQ(old_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); + Open(); + ASSERT_EQ(old_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); +} + +TEST_F(RecoveryTest, LargeManifestCompacted) { + if (!CanAppend()) { + std::fprintf(stderr, + "skipping test because env does not support appending\n"); + return; + } + ASSERT_LEVELDB_OK(Put("foo", "bar")); + Close(); + std::string old_manifest = ManifestFileName(); + + // Pad with zeroes to make manifest file very big. + { + uint64_t len = FileSize(old_manifest); + WritableFile* file; + ASSERT_LEVELDB_OK(env()->NewAppendableFile(old_manifest, &file)); + std::string zeroes(3 * 1048576 - static_cast(len), 0); + ASSERT_LEVELDB_OK(file->Append(zeroes)); + ASSERT_LEVELDB_OK(file->Flush()); + delete file; + } + + Open(); + std::string new_manifest = ManifestFileName(); + ASSERT_NE(old_manifest, new_manifest); + ASSERT_GT(10000, FileSize(new_manifest)); + ASSERT_EQ("bar", Get("foo")); + + Open(); + ASSERT_EQ(new_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); +} + +TEST_F(RecoveryTest, NoLogFiles) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); + ASSERT_EQ(1, RemoveLogFiles()); + Open(); + ASSERT_EQ("NOT_FOUND", Get("foo")); + Open(); + ASSERT_EQ("NOT_FOUND", Get("foo")); +} + +TEST_F(RecoveryTest, LogFileReuse) { + if (!CanAppend()) { + std::fprintf(stderr, + "skipping test because env does not support appending\n"); + return; + } + for (int i = 0; i < 2; i++) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); + if (i == 0) { + // Compact to ensure current log is empty + CompactMemTable(); + } + Close(); + ASSERT_EQ(1, NumLogs()); + uint64_t number = FirstLogFile(); + if (i == 0) { + ASSERT_EQ(0, FileSize(LogName(number))); + } else { + ASSERT_LT(0, FileSize(LogName(number))); + } + Open(); + ASSERT_EQ(1, NumLogs()); + ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; + ASSERT_EQ("bar", Get("foo")); + Open(); + ASSERT_EQ(1, NumLogs()); + ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; + ASSERT_EQ("bar", Get("foo")); + } +} + +TEST_F(RecoveryTest, MultipleMemTables) { + // Make a large log. + const int kNum = 1000; + for (int i = 0; i < kNum; i++) { + char buf[100]; + std::snprintf(buf, sizeof(buf), "%050d", i); + ASSERT_LEVELDB_OK(Put(buf, buf)); + } + ASSERT_EQ(0, NumTables()); + Close(); + ASSERT_EQ(0, NumTables()); + ASSERT_EQ(1, NumLogs()); + uint64_t old_log_file = FirstLogFile(); + + // Force creation of multiple memtables by reducing the write buffer size. + Options opt; + opt.reuse_logs = true; + opt.write_buffer_size = (kNum * 100) / 2; + Open(&opt); + ASSERT_LE(2, NumTables()); + ASSERT_EQ(1, NumLogs()); + ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log"; + for (int i = 0; i < kNum; i++) { + char buf[100]; + std::snprintf(buf, sizeof(buf), "%050d", i); + ASSERT_EQ(buf, Get(buf)); + } +} + +TEST_F(RecoveryTest, MultipleLogFiles) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); + Close(); + ASSERT_EQ(1, NumLogs()); + + // Make a bunch of uncompacted log files. + uint64_t old_log = FirstLogFile(); + MakeLogFile(old_log + 1, 1000, "hello", "world"); + MakeLogFile(old_log + 2, 1001, "hi", "there"); + MakeLogFile(old_log + 3, 1002, "foo", "bar2"); + + // Recover and check that all log files were processed. + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + uint64_t new_log = FirstLogFile(); + ASSERT_LE(old_log + 3, new_log); + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); + + // Test that previous recovery produced recoverable state. + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + if (CanAppend()) { + ASSERT_EQ(new_log, FirstLogFile()); + } + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); + + // Check that introducing an older log file does not cause it to be re-read. + Close(); + MakeLogFile(old_log + 1, 2000, "hello", "stale write"); + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + if (CanAppend()) { + ASSERT_EQ(new_log, FirstLogFile()); + } + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); +} + +TEST_F(RecoveryTest, ManifestMissing) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); + Close(); + RemoveManifestFile(); + + Status status = OpenWithStatus(); +#if defined(LEVELDB_PLATFORM_CHROMIUM) + // TODO(crbug.com/760362): See comment in MakeIOError() from env_chromium.cc. + ASSERT_TRUE(status.IsIOError()); +#else + ASSERT_TRUE(status.IsCorruption()); +#endif // defined(LEVELDB_PLATFORM_CHROMIUM) +} + +} // namespace leveldb diff --git a/leveldb/db/repair.cc b/leveldb/db/repair.cc new file mode 100644 index 000000000..97a27c669 --- /dev/null +++ b/leveldb/db/repair.cc @@ -0,0 +1,451 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// We recover the contents of the descriptor from the other files we find. +// (1) Any log files are first converted to tables +// (2) We scan every table to compute +// (a) smallest/largest for the table +// (b) largest sequence number in the table +// (3) We generate descriptor contents: +// - log number is set to zero +// - next-file-number is set to 1 + largest file number we found +// - last-sequence-number is set to largest sequence# found across +// all tables (see 2c) +// - compaction pointers are cleared +// - every table file is added at level 0 +// +// Possible optimization 1: +// (a) Compute total size and use to pick appropriate max-level M +// (b) Sort tables by largest sequence# in the table +// (c) For each table: if it overlaps earlier table, place in level-0, +// else place in level-M. +// Possible optimization 2: +// Store per-table metadata (smallest, largest, largest-seq#, ...) +// in the table's meta section to speed up ScanTable. + +#include "db/builder.h" +#include "db/db_impl.h" +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/memtable.h" +#include "db/table_cache.h" +#include "db/version_edit.h" +#include "db/write_batch_internal.h" +#include "leveldb/comparator.h" +#include "leveldb/db.h" +#include "leveldb/env.h" + +namespace leveldb { + +namespace { + +class Repairer { + public: + Repairer(const std::string& dbname, const Options& options) + : dbname_(dbname), + env_(options.env), + icmp_(options.comparator), + ipolicy_(options.filter_policy), + options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)), + owns_info_log_(options_.info_log != options.info_log), + owns_cache_(options_.block_cache != options.block_cache), + next_file_number_(1) { + // TableCache can be small since we expect each table to be opened once. + table_cache_ = new TableCache(dbname_, options_, 10); + } + + ~Repairer() { + delete table_cache_; + if (owns_info_log_) { + delete options_.info_log; + } + if (owns_cache_) { + delete options_.block_cache; + } + } + + Status Run() { + Status status = FindFiles(); + if (status.ok()) { + ConvertLogFilesToTables(); + ExtractMetaData(); + status = WriteDescriptor(); + } + if (status.ok()) { + unsigned long long bytes = 0; + for (size_t i = 0; i < tables_.size(); i++) { + bytes += tables_[i].meta.file_size; + } + Log(options_.info_log, + "**** Repaired leveldb %s; " + "recovered %d files; %llu bytes. " + "Some data may have been lost. " + "****", + dbname_.c_str(), static_cast(tables_.size()), bytes); + } + return status; + } + + private: + struct TableInfo { + FileMetaData meta; + SequenceNumber max_sequence; + }; + + Status FindFiles() { + std::vector filenames; + Status status = env_->GetChildren(dbname_, &filenames); + if (!status.ok()) { + return status; + } + if (filenames.empty()) { + return Status::IOError(dbname_, "repair found no files"); + } + + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type)) { + if (type == kDescriptorFile) { + manifests_.push_back(filenames[i]); + } else { + if (number + 1 > next_file_number_) { + next_file_number_ = number + 1; + } + if (type == kLogFile) { + logs_.push_back(number); + } else if (type == kTableFile) { + table_numbers_.push_back(number); + } else { + // Ignore other files + } + } + } + } + return status; + } + + void ConvertLogFilesToTables() { + for (size_t i = 0; i < logs_.size(); i++) { + std::string logname = LogFileName(dbname_, logs_[i]); + Status status = ConvertLogToTable(logs_[i]); + if (!status.ok()) { + Log(options_.info_log, "Log #%llu: ignoring conversion error: %s", + (unsigned long long)logs_[i], status.ToString().c_str()); + } + ArchiveFile(logname); + } + } + + Status ConvertLogToTable(uint64_t log) { + struct LogReporter : public log::Reader::Reporter { + Env* env; + Logger* info_log; + uint64_t lognum; + void Corruption(size_t bytes, const Status& s) override { + // We print error messages for corruption, but continue repairing. + Log(info_log, "Log #%llu: dropping %d bytes; %s", + (unsigned long long)lognum, static_cast(bytes), + s.ToString().c_str()); + } + }; + + // Open the log file + std::string logname = LogFileName(dbname_, log); + SequentialFile* lfile; + Status status = env_->NewSequentialFile(logname, &lfile); + if (!status.ok()) { + return status; + } + + // Create the log reader. + LogReporter reporter; + reporter.env = env_; + reporter.info_log = options_.info_log; + reporter.lognum = log; + // We intentionally make log::Reader do checksumming so that + // corruptions cause entire commits to be skipped instead of + // propagating bad information (like overly large sequence + // numbers). + log::Reader reader(lfile, &reporter, false /*do not checksum*/, + 0 /*initial_offset*/); + + // Read all the records and add to a memtable + std::string scratch; + Slice record; + WriteBatch batch; + MemTable* mem = new MemTable(icmp_); + mem->Ref(); + int counter = 0; + while (reader.ReadRecord(&record, &scratch)) { + if (record.size() < 12) { + reporter.Corruption(record.size(), + Status::Corruption("log record too small")); + continue; + } + WriteBatchInternal::SetContents(&batch, record); + status = WriteBatchInternal::InsertInto(&batch, mem); + if (status.ok()) { + counter += WriteBatchInternal::Count(&batch); + } else { + Log(options_.info_log, "Log #%llu: ignoring %s", + (unsigned long long)log, status.ToString().c_str()); + status = Status::OK(); // Keep going with rest of file + } + } + delete lfile; + + // Do not record a version edit for this conversion to a Table + // since ExtractMetaData() will also generate edits. + FileMetaData meta; + meta.number = next_file_number_++; + Iterator* iter = mem->NewIterator(); + status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta); + delete iter; + mem->Unref(); + mem = nullptr; + if (status.ok()) { + if (meta.file_size > 0) { + table_numbers_.push_back(meta.number); + } + } + Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s", + (unsigned long long)log, counter, (unsigned long long)meta.number, + status.ToString().c_str()); + return status; + } + + void ExtractMetaData() { + for (size_t i = 0; i < table_numbers_.size(); i++) { + ScanTable(table_numbers_[i]); + } + } + + Iterator* NewTableIterator(const FileMetaData& meta) { + // Same as compaction iterators: if paranoid_checks are on, turn + // on checksum verification. + ReadOptions r; + r.verify_checksums = options_.paranoid_checks; + return table_cache_->NewIterator(r, meta.number, meta.file_size); + } + + void ScanTable(uint64_t number) { + TableInfo t; + t.meta.number = number; + std::string fname = TableFileName(dbname_, number); + Status status = env_->GetFileSize(fname, &t.meta.file_size); + if (!status.ok()) { + // Try alternate file name. + fname = SSTTableFileName(dbname_, number); + Status s2 = env_->GetFileSize(fname, &t.meta.file_size); + if (s2.ok()) { + status = Status::OK(); + } + } + if (!status.ok()) { + ArchiveFile(TableFileName(dbname_, number)); + ArchiveFile(SSTTableFileName(dbname_, number)); + Log(options_.info_log, "Table #%llu: dropped: %s", + (unsigned long long)t.meta.number, status.ToString().c_str()); + return; + } + + // Extract metadata by scanning through table. + int counter = 0; + Iterator* iter = NewTableIterator(t.meta); + bool empty = true; + ParsedInternalKey parsed; + t.max_sequence = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + Slice key = iter->key(); + if (!ParseInternalKey(key, &parsed)) { + Log(options_.info_log, "Table #%llu: unparsable key %s", + (unsigned long long)t.meta.number, EscapeString(key).c_str()); + continue; + } + + counter++; + if (empty) { + empty = false; + t.meta.smallest.DecodeFrom(key); + } + t.meta.largest.DecodeFrom(key); + if (parsed.sequence > t.max_sequence) { + t.max_sequence = parsed.sequence; + } + } + if (!iter->status().ok()) { + status = iter->status(); + } + delete iter; + Log(options_.info_log, "Table #%llu: %d entries %s", + (unsigned long long)t.meta.number, counter, status.ToString().c_str()); + + if (status.ok()) { + tables_.push_back(t); + } else { + RepairTable(fname, t); // RepairTable archives input file. + } + } + + void RepairTable(const std::string& src, TableInfo t) { + // We will copy src contents to a new table and then rename the + // new table over the source. + + // Create builder. + std::string copy = TableFileName(dbname_, next_file_number_++); + WritableFile* file; + Status s = env_->NewWritableFile(copy, &file); + if (!s.ok()) { + return; + } + TableBuilder* builder = new TableBuilder(options_, file); + + // Copy data. + Iterator* iter = NewTableIterator(t.meta); + int counter = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + builder->Add(iter->key(), iter->value()); + counter++; + } + delete iter; + + ArchiveFile(src); + if (counter == 0) { + builder->Abandon(); // Nothing to save + } else { + s = builder->Finish(); + if (s.ok()) { + t.meta.file_size = builder->FileSize(); + } + } + delete builder; + builder = nullptr; + + if (s.ok()) { + s = file->Close(); + } + delete file; + file = nullptr; + + if (counter > 0 && s.ok()) { + std::string orig = TableFileName(dbname_, t.meta.number); + s = env_->RenameFile(copy, orig); + if (s.ok()) { + Log(options_.info_log, "Table #%llu: %d entries repaired", + (unsigned long long)t.meta.number, counter); + tables_.push_back(t); + } + } + if (!s.ok()) { + env_->RemoveFile(copy); + } + } + + Status WriteDescriptor() { + std::string tmp = TempFileName(dbname_, 1); + WritableFile* file; + Status status = env_->NewWritableFile(tmp, &file); + if (!status.ok()) { + return status; + } + + SequenceNumber max_sequence = 0; + for (size_t i = 0; i < tables_.size(); i++) { + if (max_sequence < tables_[i].max_sequence) { + max_sequence = tables_[i].max_sequence; + } + } + + edit_.SetComparatorName(icmp_.user_comparator()->Name()); + edit_.SetLogNumber(0); + edit_.SetNextFile(next_file_number_); + edit_.SetLastSequence(max_sequence); + + for (size_t i = 0; i < tables_.size(); i++) { + // TODO(opt): separate out into multiple levels + const TableInfo& t = tables_[i]; + edit_.AddFile(0, t.meta.number, t.meta.file_size, t.meta.smallest, + t.meta.largest); + } + + // std::fprintf(stderr, + // "NewDescriptor:\n%s\n", edit_.DebugString().c_str()); + { + log::Writer log(file); + std::string record; + edit_.EncodeTo(&record); + status = log.AddRecord(record); + } + if (status.ok()) { + status = file->Close(); + } + delete file; + file = nullptr; + + if (!status.ok()) { + env_->RemoveFile(tmp); + } else { + // Discard older manifests + for (size_t i = 0; i < manifests_.size(); i++) { + ArchiveFile(dbname_ + "/" + manifests_[i]); + } + + // Install new manifest + status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1)); + if (status.ok()) { + status = SetCurrentFile(env_, dbname_, 1); + } else { + env_->RemoveFile(tmp); + } + } + return status; + } + + void ArchiveFile(const std::string& fname) { + // Move into another directory. E.g., for + // dir/foo + // rename to + // dir/lost/foo + const char* slash = strrchr(fname.c_str(), '/'); + std::string new_dir; + if (slash != nullptr) { + new_dir.assign(fname.data(), slash - fname.data()); + } + new_dir.append("/lost"); + env_->CreateDir(new_dir); // Ignore error + std::string new_file = new_dir; + new_file.append("/"); + new_file.append((slash == nullptr) ? fname.c_str() : slash + 1); + Status s = env_->RenameFile(fname, new_file); + Log(options_.info_log, "Archiving %s: %s\n", fname.c_str(), + s.ToString().c_str()); + } + + const std::string dbname_; + Env* const env_; + InternalKeyComparator const icmp_; + InternalFilterPolicy const ipolicy_; + const Options options_; + bool owns_info_log_; + bool owns_cache_; + TableCache* table_cache_; + VersionEdit edit_; + + std::vector manifests_; + std::vector table_numbers_; + std::vector logs_; + std::vector tables_; + uint64_t next_file_number_; +}; +} // namespace + +Status RepairDB(const std::string& dbname, const Options& options) { + Repairer repairer(dbname, options); + return repairer.Run(); +} + +} // namespace leveldb diff --git a/leveldb/db/skiplist.h b/leveldb/db/skiplist.h new file mode 100644 index 000000000..1140e59a6 --- /dev/null +++ b/leveldb/db/skiplist.h @@ -0,0 +1,380 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_ +#define STORAGE_LEVELDB_DB_SKIPLIST_H_ + +// Thread safety +// ------------- +// +// Writes require external synchronization, most likely a mutex. +// Reads require a guarantee that the SkipList will not be destroyed +// while the read is in progress. Apart from that, reads progress +// without any internal locking or synchronization. +// +// Invariants: +// +// (1) Allocated nodes are never deleted until the SkipList is +// destroyed. This is trivially guaranteed by the code since we +// never delete any skip list nodes. +// +// (2) The contents of a Node except for the next/prev pointers are +// immutable after the Node has been linked into the SkipList. +// Only Insert() modifies the list, and it is careful to initialize +// a node and use release-stores to publish the nodes in one or +// more lists. +// +// ... prev vs. next pointer ordering ... + +#include +#include +#include + +#include "util/arena.h" +#include "util/random.h" + +namespace leveldb { + +template +class SkipList { + private: + struct Node; + + public: + // Create a new SkipList object that will use "cmp" for comparing keys, + // and will allocate memory using "*arena". Objects allocated in the arena + // must remain allocated for the lifetime of the skiplist object. + explicit SkipList(Comparator cmp, Arena* arena); + + SkipList(const SkipList&) = delete; + SkipList& operator=(const SkipList&) = delete; + + // Insert key into the list. + // REQUIRES: nothing that compares equal to key is currently in the list. + void Insert(const Key& key); + + // Returns true iff an entry that compares equal to key is in the list. + bool Contains(const Key& key) const; + + // Iteration over the contents of a skip list + class Iterator { + public: + // Initialize an iterator over the specified list. + // The returned iterator is not valid. + explicit Iterator(const SkipList* list); + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const; + + // Returns the key at the current position. + // REQUIRES: Valid() + const Key& key() const; + + // Advances to the next position. + // REQUIRES: Valid() + void Next(); + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev(); + + // Advance to the first entry with a key >= target + void Seek(const Key& target); + + // Position at the first entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToFirst(); + + // Position at the last entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToLast(); + + private: + const SkipList* list_; + Node* node_; + // Intentionally copyable + }; + + private: + enum { kMaxHeight = 12 }; + + inline int GetMaxHeight() const { + return max_height_.load(std::memory_order_relaxed); + } + + Node* NewNode(const Key& key, int height); + int RandomHeight(); + bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } + + // Return true if key is greater than the data stored in "n" + bool KeyIsAfterNode(const Key& key, Node* n) const; + + // Return the earliest node that comes at or after key. + // Return nullptr if there is no such node. + // + // If prev is non-null, fills prev[level] with pointer to previous + // node at "level" for every level in [0..max_height_-1]. + Node* FindGreaterOrEqual(const Key& key, Node** prev) const; + + // Return the latest node with a key < key. + // Return head_ if there is no such node. + Node* FindLessThan(const Key& key) const; + + // Return the last node in the list. + // Return head_ if list is empty. + Node* FindLast() const; + + // Immutable after construction + Comparator const compare_; + Arena* const arena_; // Arena used for allocations of nodes + + Node* const head_; + + // Modified only by Insert(). Read racily by readers, but stale + // values are ok. + std::atomic max_height_; // Height of the entire list + + // Read/written only by Insert(). + Random rnd_; +}; + +// Implementation details follow +template +struct SkipList::Node { + explicit Node(const Key& k) : key(k) {} + + Key const key; + + // Accessors/mutators for links. Wrapped in methods so we can + // add the appropriate barriers as necessary. + Node* Next(int n) { + assert(n >= 0); + // Use an 'acquire load' so that we observe a fully initialized + // version of the returned Node. + return next_[n].load(std::memory_order_acquire); + } + void SetNext(int n, Node* x) { + assert(n >= 0); + // Use a 'release store' so that anybody who reads through this + // pointer observes a fully initialized version of the inserted node. + next_[n].store(x, std::memory_order_release); + } + + // No-barrier variants that can be safely used in a few locations. + Node* NoBarrier_Next(int n) { + assert(n >= 0); + return next_[n].load(std::memory_order_relaxed); + } + void NoBarrier_SetNext(int n, Node* x) { + assert(n >= 0); + next_[n].store(x, std::memory_order_relaxed); + } + + private: + // Array of length equal to the node height. next_[0] is lowest level link. + std::atomic next_[1]; +}; + +template +typename SkipList::Node* SkipList::NewNode( + const Key& key, int height) { + char* const node_memory = arena_->AllocateAligned( + sizeof(Node) + sizeof(std::atomic) * (height - 1)); + return new (node_memory) Node(key); +} + +template +inline SkipList::Iterator::Iterator(const SkipList* list) { + list_ = list; + node_ = nullptr; +} + +template +inline bool SkipList::Iterator::Valid() const { + return node_ != nullptr; +} + +template +inline const Key& SkipList::Iterator::key() const { + assert(Valid()); + return node_->key; +} + +template +inline void SkipList::Iterator::Next() { + assert(Valid()); + node_ = node_->Next(0); +} + +template +inline void SkipList::Iterator::Prev() { + // Instead of using explicit "prev" links, we just search for the + // last node that falls before key. + assert(Valid()); + node_ = list_->FindLessThan(node_->key); + if (node_ == list_->head_) { + node_ = nullptr; + } +} + +template +inline void SkipList::Iterator::Seek(const Key& target) { + node_ = list_->FindGreaterOrEqual(target, nullptr); +} + +template +inline void SkipList::Iterator::SeekToFirst() { + node_ = list_->head_->Next(0); +} + +template +inline void SkipList::Iterator::SeekToLast() { + node_ = list_->FindLast(); + if (node_ == list_->head_) { + node_ = nullptr; + } +} + +template +int SkipList::RandomHeight() { + // Increase height with probability 1 in kBranching + static const unsigned int kBranching = 4; + int height = 1; + while (height < kMaxHeight && rnd_.OneIn(kBranching)) { + height++; + } + assert(height > 0); + assert(height <= kMaxHeight); + return height; +} + +template +bool SkipList::KeyIsAfterNode(const Key& key, Node* n) const { + // null n is considered infinite + return (n != nullptr) && (compare_(n->key, key) < 0); +} + +template +typename SkipList::Node* +SkipList::FindGreaterOrEqual(const Key& key, + Node** prev) const { + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + Node* next = x->Next(level); + if (KeyIsAfterNode(key, next)) { + // Keep searching in this list + x = next; + } else { + if (prev != nullptr) prev[level] = x; + if (level == 0) { + return next; + } else { + // Switch to next list + level--; + } + } + } +} + +template +typename SkipList::Node* +SkipList::FindLessThan(const Key& key) const { + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + assert(x == head_ || compare_(x->key, key) < 0); + Node* next = x->Next(level); + if (next == nullptr || compare_(next->key, key) >= 0) { + if (level == 0) { + return x; + } else { + // Switch to next list + level--; + } + } else { + x = next; + } + } +} + +template +typename SkipList::Node* SkipList::FindLast() + const { + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + Node* next = x->Next(level); + if (next == nullptr) { + if (level == 0) { + return x; + } else { + // Switch to next list + level--; + } + } else { + x = next; + } + } +} + +template +SkipList::SkipList(Comparator cmp, Arena* arena) + : compare_(cmp), + arena_(arena), + head_(NewNode(0 /* any key will do */, kMaxHeight)), + max_height_(1), + rnd_(0xdeadbeef) { + for (int i = 0; i < kMaxHeight; i++) { + head_->SetNext(i, nullptr); + } +} + +template +void SkipList::Insert(const Key& key) { + // TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual() + // here since Insert() is externally synchronized. + Node* prev[kMaxHeight]; + Node* x = FindGreaterOrEqual(key, prev); + + // Our data structure does not allow duplicate insertion + assert(x == nullptr || !Equal(key, x->key)); + + int height = RandomHeight(); + if (height > GetMaxHeight()) { + for (int i = GetMaxHeight(); i < height; i++) { + prev[i] = head_; + } + // It is ok to mutate max_height_ without any synchronization + // with concurrent readers. A concurrent reader that observes + // the new value of max_height_ will see either the old value of + // new level pointers from head_ (nullptr), or a new value set in + // the loop below. In the former case the reader will + // immediately drop to the next level since nullptr sorts after all + // keys. In the latter case the reader will use the new node. + max_height_.store(height, std::memory_order_relaxed); + } + + x = NewNode(key, height); + for (int i = 0; i < height; i++) { + // NoBarrier_SetNext() suffices since we will add a barrier when + // we publish a pointer to "x" in prev[i]. + x->NoBarrier_SetNext(i, prev[i]->NoBarrier_Next(i)); + prev[i]->SetNext(i, x); + } +} + +template +bool SkipList::Contains(const Key& key) const { + Node* x = FindGreaterOrEqual(key, nullptr); + if (x != nullptr && Equal(key, x->key)) { + return true; + } else { + return false; + } +} + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_ diff --git a/leveldb/db/skiplist_test.cc b/leveldb/db/skiplist_test.cc new file mode 100644 index 000000000..1d355cb31 --- /dev/null +++ b/leveldb/db/skiplist_test.cc @@ -0,0 +1,368 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/skiplist.h" + +#include +#include + +#include "gtest/gtest.h" +#include "leveldb/env.h" +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/arena.h" +#include "util/hash.h" +#include "util/random.h" +#include "util/testutil.h" + +namespace leveldb { + +typedef uint64_t Key; + +struct Comparator { + int operator()(const Key& a, const Key& b) const { + if (a < b) { + return -1; + } else if (a > b) { + return +1; + } else { + return 0; + } + } +}; + +TEST(SkipTest, Empty) { + Arena arena; + Comparator cmp; + SkipList list(cmp, &arena); + ASSERT_TRUE(!list.Contains(10)); + + SkipList::Iterator iter(&list); + ASSERT_TRUE(!iter.Valid()); + iter.SeekToFirst(); + ASSERT_TRUE(!iter.Valid()); + iter.Seek(100); + ASSERT_TRUE(!iter.Valid()); + iter.SeekToLast(); + ASSERT_TRUE(!iter.Valid()); +} + +TEST(SkipTest, InsertAndLookup) { + const int N = 2000; + const int R = 5000; + Random rnd(1000); + std::set keys; + Arena arena; + Comparator cmp; + SkipList list(cmp, &arena); + for (int i = 0; i < N; i++) { + Key key = rnd.Next() % R; + if (keys.insert(key).second) { + list.Insert(key); + } + } + + for (int i = 0; i < R; i++) { + if (list.Contains(i)) { + ASSERT_EQ(keys.count(i), 1); + } else { + ASSERT_EQ(keys.count(i), 0); + } + } + + // Simple iterator tests + { + SkipList::Iterator iter(&list); + ASSERT_TRUE(!iter.Valid()); + + iter.Seek(0); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.begin()), iter.key()); + + iter.SeekToFirst(); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.begin()), iter.key()); + + iter.SeekToLast(); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.rbegin()), iter.key()); + } + + // Forward iteration test + for (int i = 0; i < R; i++) { + SkipList::Iterator iter(&list); + iter.Seek(i); + + // Compare against model iterator + std::set::iterator model_iter = keys.lower_bound(i); + for (int j = 0; j < 3; j++) { + if (model_iter == keys.end()) { + ASSERT_TRUE(!iter.Valid()); + break; + } else { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*model_iter, iter.key()); + ++model_iter; + iter.Next(); + } + } + } + + // Backward iteration test + { + SkipList::Iterator iter(&list); + iter.SeekToLast(); + + // Compare against model iterator + for (std::set::reverse_iterator model_iter = keys.rbegin(); + model_iter != keys.rend(); ++model_iter) { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*model_iter, iter.key()); + iter.Prev(); + } + ASSERT_TRUE(!iter.Valid()); + } +} + +// We want to make sure that with a single writer and multiple +// concurrent readers (with no synchronization other than when a +// reader's iterator is created), the reader always observes all the +// data that was present in the skip list when the iterator was +// constructed. Because insertions are happening concurrently, we may +// also observe new values that were inserted since the iterator was +// constructed, but we should never miss any values that were present +// at iterator construction time. +// +// We generate multi-part keys: +// +// where: +// key is in range [0..K-1] +// gen is a generation number for key +// hash is hash(key,gen) +// +// The insertion code picks a random key, sets gen to be 1 + the last +// generation number inserted for that key, and sets hash to Hash(key,gen). +// +// At the beginning of a read, we snapshot the last inserted +// generation number for each key. We then iterate, including random +// calls to Next() and Seek(). For every key we encounter, we +// check that it is either expected given the initial snapshot or has +// been concurrently added since the iterator started. +class ConcurrentTest { + private: + static constexpr uint32_t K = 4; + + static uint64_t key(Key key) { return (key >> 40); } + static uint64_t gen(Key key) { return (key >> 8) & 0xffffffffu; } + static uint64_t hash(Key key) { return key & 0xff; } + + static uint64_t HashNumbers(uint64_t k, uint64_t g) { + uint64_t data[2] = {k, g}; + return Hash(reinterpret_cast(data), sizeof(data), 0); + } + + static Key MakeKey(uint64_t k, uint64_t g) { + static_assert(sizeof(Key) == sizeof(uint64_t), ""); + assert(k <= K); // We sometimes pass K to seek to the end of the skiplist + assert(g <= 0xffffffffu); + return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff)); + } + + static bool IsValidKey(Key k) { + return hash(k) == (HashNumbers(key(k), gen(k)) & 0xff); + } + + static Key RandomTarget(Random* rnd) { + switch (rnd->Next() % 10) { + case 0: + // Seek to beginning + return MakeKey(0, 0); + case 1: + // Seek to end + return MakeKey(K, 0); + default: + // Seek to middle + return MakeKey(rnd->Next() % K, 0); + } + } + + // Per-key generation + struct State { + std::atomic generation[K]; + void Set(int k, int v) { + generation[k].store(v, std::memory_order_release); + } + int Get(int k) { return generation[k].load(std::memory_order_acquire); } + + State() { + for (int k = 0; k < K; k++) { + Set(k, 0); + } + } + }; + + // Current state of the test + State current_; + + Arena arena_; + + // SkipList is not protected by mu_. We just use a single writer + // thread to modify it. + SkipList list_; + + public: + ConcurrentTest() : list_(Comparator(), &arena_) {} + + // REQUIRES: External synchronization + void WriteStep(Random* rnd) { + const uint32_t k = rnd->Next() % K; + const intptr_t g = current_.Get(k) + 1; + const Key key = MakeKey(k, g); + list_.Insert(key); + current_.Set(k, g); + } + + void ReadStep(Random* rnd) { + // Remember the initial committed state of the skiplist. + State initial_state; + for (int k = 0; k < K; k++) { + initial_state.Set(k, current_.Get(k)); + } + + Key pos = RandomTarget(rnd); + SkipList::Iterator iter(&list_); + iter.Seek(pos); + while (true) { + Key current; + if (!iter.Valid()) { + current = MakeKey(K, 0); + } else { + current = iter.key(); + ASSERT_TRUE(IsValidKey(current)) << current; + } + ASSERT_LE(pos, current) << "should not go backwards"; + + // Verify that everything in [pos,current) was not present in + // initial_state. + while (pos < current) { + ASSERT_LT(key(pos), K) << pos; + + // Note that generation 0 is never inserted, so it is ok if + // <*,0,*> is missing. + ASSERT_TRUE((gen(pos) == 0) || + (gen(pos) > static_cast(initial_state.Get(key(pos))))) + << "key: " << key(pos) << "; gen: " << gen(pos) + << "; initgen: " << initial_state.Get(key(pos)); + + // Advance to next key in the valid key space + if (key(pos) < key(current)) { + pos = MakeKey(key(pos) + 1, 0); + } else { + pos = MakeKey(key(pos), gen(pos) + 1); + } + } + + if (!iter.Valid()) { + break; + } + + if (rnd->Next() % 2) { + iter.Next(); + pos = MakeKey(key(pos), gen(pos) + 1); + } else { + Key new_target = RandomTarget(rnd); + if (new_target > pos) { + pos = new_target; + iter.Seek(new_target); + } + } + } + } +}; + +// Needed when building in C++11 mode. +constexpr uint32_t ConcurrentTest::K; + +// Simple test that does single-threaded testing of the ConcurrentTest +// scaffolding. +TEST(SkipTest, ConcurrentWithoutThreads) { + ConcurrentTest test; + Random rnd(test::RandomSeed()); + for (int i = 0; i < 10000; i++) { + test.ReadStep(&rnd); + test.WriteStep(&rnd); + } +} + +class TestState { + public: + ConcurrentTest t_; + int seed_; + std::atomic quit_flag_; + + enum ReaderState { STARTING, RUNNING, DONE }; + + explicit TestState(int s) + : seed_(s), quit_flag_(false), state_(STARTING), state_cv_(&mu_) {} + + void Wait(ReaderState s) LOCKS_EXCLUDED(mu_) { + mu_.Lock(); + while (state_ != s) { + state_cv_.Wait(); + } + mu_.Unlock(); + } + + void Change(ReaderState s) LOCKS_EXCLUDED(mu_) { + mu_.Lock(); + state_ = s; + state_cv_.Signal(); + mu_.Unlock(); + } + + private: + port::Mutex mu_; + ReaderState state_ GUARDED_BY(mu_); + port::CondVar state_cv_ GUARDED_BY(mu_); +}; + +static void ConcurrentReader(void* arg) { + TestState* state = reinterpret_cast(arg); + Random rnd(state->seed_); + int64_t reads = 0; + state->Change(TestState::RUNNING); + while (!state->quit_flag_.load(std::memory_order_acquire)) { + state->t_.ReadStep(&rnd); + ++reads; + } + state->Change(TestState::DONE); +} + +static void RunConcurrent(int run) { + const int seed = test::RandomSeed() + (run * 100); + Random rnd(seed); + const int N = 1000; + const int kSize = 1000; + for (int i = 0; i < N; i++) { + if ((i % 100) == 0) { + std::fprintf(stderr, "Run %d of %d\n", i, N); + } + TestState state(seed + 1); + Env::Default()->Schedule(ConcurrentReader, &state); + state.Wait(TestState::RUNNING); + for (int i = 0; i < kSize; i++) { + state.t_.WriteStep(&rnd); + } + state.quit_flag_.store(true, std::memory_order_release); + state.Wait(TestState::DONE); + } +} + +TEST(SkipTest, Concurrent1) { RunConcurrent(1); } +TEST(SkipTest, Concurrent2) { RunConcurrent(2); } +TEST(SkipTest, Concurrent3) { RunConcurrent(3); } +TEST(SkipTest, Concurrent4) { RunConcurrent(4); } +TEST(SkipTest, Concurrent5) { RunConcurrent(5); } + +} // namespace leveldb diff --git a/leveldb/db/snapshot.h b/leveldb/db/snapshot.h new file mode 100644 index 000000000..817bb7ba1 --- /dev/null +++ b/leveldb/db/snapshot.h @@ -0,0 +1,95 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_ +#define STORAGE_LEVELDB_DB_SNAPSHOT_H_ + +#include "db/dbformat.h" +#include "leveldb/db.h" + +namespace leveldb { + +class SnapshotList; + +// Snapshots are kept in a doubly-linked list in the DB. +// Each SnapshotImpl corresponds to a particular sequence number. +class SnapshotImpl : public Snapshot { + public: + SnapshotImpl(SequenceNumber sequence_number) + : sequence_number_(sequence_number) {} + + SequenceNumber sequence_number() const { return sequence_number_; } + + private: + friend class SnapshotList; + + // SnapshotImpl is kept in a doubly-linked circular list. The SnapshotList + // implementation operates on the next/previous fields directly. + SnapshotImpl* prev_; + SnapshotImpl* next_; + + const SequenceNumber sequence_number_; + +#if !defined(NDEBUG) + SnapshotList* list_ = nullptr; +#endif // !defined(NDEBUG) +}; + +class SnapshotList { + public: + SnapshotList() : head_(0) { + head_.prev_ = &head_; + head_.next_ = &head_; + } + + bool empty() const { return head_.next_ == &head_; } + SnapshotImpl* oldest() const { + assert(!empty()); + return head_.next_; + } + SnapshotImpl* newest() const { + assert(!empty()); + return head_.prev_; + } + + // Creates a SnapshotImpl and appends it to the end of the list. + SnapshotImpl* New(SequenceNumber sequence_number) { + assert(empty() || newest()->sequence_number_ <= sequence_number); + + SnapshotImpl* snapshot = new SnapshotImpl(sequence_number); + +#if !defined(NDEBUG) + snapshot->list_ = this; +#endif // !defined(NDEBUG) + snapshot->next_ = &head_; + snapshot->prev_ = head_.prev_; + snapshot->prev_->next_ = snapshot; + snapshot->next_->prev_ = snapshot; + return snapshot; + } + + // Removes a SnapshotImpl from this list. + // + // The snapshot must have been created by calling New() on this list. + // + // The snapshot pointer should not be const, because its memory is + // deallocated. However, that would force us to change DB::ReleaseSnapshot(), + // which is in the API, and currently takes a const Snapshot. + void Delete(const SnapshotImpl* snapshot) { +#if !defined(NDEBUG) + assert(snapshot->list_ == this); +#endif // !defined(NDEBUG) + snapshot->prev_->next_ = snapshot->next_; + snapshot->next_->prev_ = snapshot->prev_; + delete snapshot; + } + + private: + // Dummy head of doubly-linked list of snapshots + SnapshotImpl head_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_ diff --git a/leveldb/db/table_cache.cc b/leveldb/db/table_cache.cc new file mode 100644 index 000000000..73f05fd7b --- /dev/null +++ b/leveldb/db/table_cache.cc @@ -0,0 +1,120 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/table_cache.h" + +#include "db/filename.h" +#include "leveldb/env.h" +#include "leveldb/table.h" +#include "util/coding.h" + +namespace leveldb { + +struct TableAndFile { + RandomAccessFile* file; + Table* table; +}; + +static void DeleteEntry(const Slice& key, void* value) { + TableAndFile* tf = reinterpret_cast(value); + delete tf->table; + delete tf->file; + delete tf; +} + +static void UnrefEntry(void* arg1, void* arg2) { + Cache* cache = reinterpret_cast(arg1); + Cache::Handle* h = reinterpret_cast(arg2); + cache->Release(h); +} + +TableCache::TableCache(const std::string& dbname, const Options& options, + int entries) + : env_(options.env), + dbname_(dbname), + options_(options), + cache_(NewLRUCache(entries)) {} + +TableCache::~TableCache() { delete cache_; } + +Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, + Cache::Handle** handle) { + Status s; + char buf[sizeof(file_number)]; + EncodeFixed64(buf, file_number); + Slice key(buf, sizeof(buf)); + *handle = cache_->Lookup(key); + if (*handle == nullptr) { + std::string fname = TableFileName(dbname_, file_number); + RandomAccessFile* file = nullptr; + Table* table = nullptr; + s = env_->NewRandomAccessFile(fname, &file); + if (!s.ok()) { + std::string old_fname = SSTTableFileName(dbname_, file_number); + if (env_->NewRandomAccessFile(old_fname, &file).ok()) { + s = Status::OK(); + } + } + if (s.ok()) { + s = Table::Open(options_, file, file_size, &table); + } + + if (!s.ok()) { + assert(table == nullptr); + delete file; + // We do not cache error results so that if the error is transient, + // or somebody repairs the file, we recover automatically. + } else { + TableAndFile* tf = new TableAndFile; + tf->file = file; + tf->table = table; + *handle = cache_->Insert(key, tf, 1, &DeleteEntry); + } + } + return s; +} + +Iterator* TableCache::NewIterator(const ReadOptions& options, + uint64_t file_number, uint64_t file_size, + Table** tableptr) { + if (tableptr != nullptr) { + *tableptr = nullptr; + } + + Cache::Handle* handle = nullptr; + Status s = FindTable(file_number, file_size, &handle); + if (!s.ok()) { + return NewErrorIterator(s); + } + + Table* table = reinterpret_cast(cache_->Value(handle))->table; + Iterator* result = table->NewIterator(options); + result->RegisterCleanup(&UnrefEntry, cache_, handle); + if (tableptr != nullptr) { + *tableptr = table; + } + return result; +} + +Status TableCache::Get(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, const Slice& k, void* arg, + void (*handle_result)(void*, const Slice&, + const Slice&)) { + Cache::Handle* handle = nullptr; + Status s = FindTable(file_number, file_size, &handle); + if (s.ok()) { + Table* t = reinterpret_cast(cache_->Value(handle))->table; + s = t->InternalGet(options, k, arg, handle_result); + cache_->Release(handle); + } + return s; +} + +void TableCache::Evict(uint64_t file_number) { + char buf[sizeof(file_number)]; + EncodeFixed64(buf, file_number); + cache_->Erase(Slice(buf, sizeof(buf))); +} + +} // namespace leveldb diff --git a/leveldb/db/table_cache.h b/leveldb/db/table_cache.h new file mode 100644 index 000000000..db8a1231b --- /dev/null +++ b/leveldb/db/table_cache.h @@ -0,0 +1,61 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Thread-safe (provides internal synchronization) + +#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ +#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ + +#include +#include + +#include "db/dbformat.h" +#include "leveldb/cache.h" +#include "leveldb/table.h" +#include "port/port.h" + +namespace leveldb { + +class Env; + +class TableCache { + public: + TableCache(const std::string& dbname, const Options& options, int entries); + + TableCache(const TableCache&) = delete; + TableCache& operator=(const TableCache&) = delete; + + ~TableCache(); + + // Return an iterator for the specified file number (the corresponding + // file length must be exactly "file_size" bytes). If "tableptr" is + // non-null, also sets "*tableptr" to point to the Table object + // underlying the returned iterator, or to nullptr if no Table object + // underlies the returned iterator. The returned "*tableptr" object is owned + // by the cache and should not be deleted, and is valid for as long as the + // returned iterator is live. + Iterator* NewIterator(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, Table** tableptr = nullptr); + + // If a seek to internal key "k" in specified file finds an entry, + // call (*handle_result)(arg, found_key, found_value). + Status Get(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, const Slice& k, void* arg, + void (*handle_result)(void*, const Slice&, const Slice&)); + + // Evict any entry for the specified file number + void Evict(uint64_t file_number); + + private: + Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); + + Env* const env_; + const std::string dbname_; + const Options& options_; + Cache* cache_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_ diff --git a/leveldb/db/version_edit.cc b/leveldb/db/version_edit.cc new file mode 100644 index 000000000..356ce883e --- /dev/null +++ b/leveldb/db/version_edit.cc @@ -0,0 +1,258 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_edit.h" + +#include "db/version_set.h" +#include "util/coding.h" + +namespace leveldb { + +// Tag numbers for serialized VersionEdit. These numbers are written to +// disk and should not be changed. +enum Tag { + kComparator = 1, + kLogNumber = 2, + kNextFileNumber = 3, + kLastSequence = 4, + kCompactPointer = 5, + kDeletedFile = 6, + kNewFile = 7, + // 8 was used for large value refs + kPrevLogNumber = 9 +}; + +void VersionEdit::Clear() { + comparator_.clear(); + log_number_ = 0; + prev_log_number_ = 0; + last_sequence_ = 0; + next_file_number_ = 0; + has_comparator_ = false; + has_log_number_ = false; + has_prev_log_number_ = false; + has_next_file_number_ = false; + has_last_sequence_ = false; + compact_pointers_.clear(); + deleted_files_.clear(); + new_files_.clear(); +} + +void VersionEdit::EncodeTo(std::string* dst) const { + if (has_comparator_) { + PutVarint32(dst, kComparator); + PutLengthPrefixedSlice(dst, comparator_); + } + if (has_log_number_) { + PutVarint32(dst, kLogNumber); + PutVarint64(dst, log_number_); + } + if (has_prev_log_number_) { + PutVarint32(dst, kPrevLogNumber); + PutVarint64(dst, prev_log_number_); + } + if (has_next_file_number_) { + PutVarint32(dst, kNextFileNumber); + PutVarint64(dst, next_file_number_); + } + if (has_last_sequence_) { + PutVarint32(dst, kLastSequence); + PutVarint64(dst, last_sequence_); + } + + for (size_t i = 0; i < compact_pointers_.size(); i++) { + PutVarint32(dst, kCompactPointer); + PutVarint32(dst, compact_pointers_[i].first); // level + PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode()); + } + + for (const auto& deleted_file_kvp : deleted_files_) { + PutVarint32(dst, kDeletedFile); + PutVarint32(dst, deleted_file_kvp.first); // level + PutVarint64(dst, deleted_file_kvp.second); // file number + } + + for (size_t i = 0; i < new_files_.size(); i++) { + const FileMetaData& f = new_files_[i].second; + PutVarint32(dst, kNewFile); + PutVarint32(dst, new_files_[i].first); // level + PutVarint64(dst, f.number); + PutVarint64(dst, f.file_size); + PutLengthPrefixedSlice(dst, f.smallest.Encode()); + PutLengthPrefixedSlice(dst, f.largest.Encode()); + } +} + +static bool GetInternalKey(Slice* input, InternalKey* dst) { + Slice str; + if (GetLengthPrefixedSlice(input, &str)) { + return dst->DecodeFrom(str); + } else { + return false; + } +} + +static bool GetLevel(Slice* input, int* level) { + uint32_t v; + if (GetVarint32(input, &v) && v < config::kNumLevels) { + *level = v; + return true; + } else { + return false; + } +} + +Status VersionEdit::DecodeFrom(const Slice& src) { + Clear(); + Slice input = src; + const char* msg = nullptr; + uint32_t tag; + + // Temporary storage for parsing + int level; + uint64_t number; + FileMetaData f; + Slice str; + InternalKey key; + + while (msg == nullptr && GetVarint32(&input, &tag)) { + switch (tag) { + case kComparator: + if (GetLengthPrefixedSlice(&input, &str)) { + comparator_ = str.ToString(); + has_comparator_ = true; + } else { + msg = "comparator name"; + } + break; + + case kLogNumber: + if (GetVarint64(&input, &log_number_)) { + has_log_number_ = true; + } else { + msg = "log number"; + } + break; + + case kPrevLogNumber: + if (GetVarint64(&input, &prev_log_number_)) { + has_prev_log_number_ = true; + } else { + msg = "previous log number"; + } + break; + + case kNextFileNumber: + if (GetVarint64(&input, &next_file_number_)) { + has_next_file_number_ = true; + } else { + msg = "next file number"; + } + break; + + case kLastSequence: + if (GetVarint64(&input, &last_sequence_)) { + has_last_sequence_ = true; + } else { + msg = "last sequence number"; + } + break; + + case kCompactPointer: + if (GetLevel(&input, &level) && GetInternalKey(&input, &key)) { + compact_pointers_.push_back(std::make_pair(level, key)); + } else { + msg = "compaction pointer"; + } + break; + + case kDeletedFile: + if (GetLevel(&input, &level) && GetVarint64(&input, &number)) { + deleted_files_.insert(std::make_pair(level, number)); + } else { + msg = "deleted file"; + } + break; + + case kNewFile: + if (GetLevel(&input, &level) && GetVarint64(&input, &f.number) && + GetVarint64(&input, &f.file_size) && + GetInternalKey(&input, &f.smallest) && + GetInternalKey(&input, &f.largest)) { + new_files_.push_back(std::make_pair(level, f)); + } else { + msg = "new-file entry"; + } + break; + + default: + msg = "unknown tag"; + break; + } + } + + if (msg == nullptr && !input.empty()) { + msg = "invalid tag"; + } + + Status result; + if (msg != nullptr) { + result = Status::Corruption("VersionEdit", msg); + } + return result; +} + +std::string VersionEdit::DebugString() const { + std::string r; + r.append("VersionEdit {"); + if (has_comparator_) { + r.append("\n Comparator: "); + r.append(comparator_); + } + if (has_log_number_) { + r.append("\n LogNumber: "); + AppendNumberTo(&r, log_number_); + } + if (has_prev_log_number_) { + r.append("\n PrevLogNumber: "); + AppendNumberTo(&r, prev_log_number_); + } + if (has_next_file_number_) { + r.append("\n NextFile: "); + AppendNumberTo(&r, next_file_number_); + } + if (has_last_sequence_) { + r.append("\n LastSeq: "); + AppendNumberTo(&r, last_sequence_); + } + for (size_t i = 0; i < compact_pointers_.size(); i++) { + r.append("\n CompactPointer: "); + AppendNumberTo(&r, compact_pointers_[i].first); + r.append(" "); + r.append(compact_pointers_[i].second.DebugString()); + } + for (const auto& deleted_files_kvp : deleted_files_) { + r.append("\n RemoveFile: "); + AppendNumberTo(&r, deleted_files_kvp.first); + r.append(" "); + AppendNumberTo(&r, deleted_files_kvp.second); + } + for (size_t i = 0; i < new_files_.size(); i++) { + const FileMetaData& f = new_files_[i].second; + r.append("\n AddFile: "); + AppendNumberTo(&r, new_files_[i].first); + r.append(" "); + AppendNumberTo(&r, f.number); + r.append(" "); + AppendNumberTo(&r, f.file_size); + r.append(" "); + r.append(f.smallest.DebugString()); + r.append(" .. "); + r.append(f.largest.DebugString()); + } + r.append("\n}\n"); + return r; +} + +} // namespace leveldb diff --git a/leveldb/db/version_edit.h b/leveldb/db/version_edit.h new file mode 100644 index 000000000..137b4b10e --- /dev/null +++ b/leveldb/db/version_edit.h @@ -0,0 +1,106 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_ +#define STORAGE_LEVELDB_DB_VERSION_EDIT_H_ + +#include +#include +#include + +#include "db/dbformat.h" + +namespace leveldb { + +class VersionSet; + +struct FileMetaData { + FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) {} + + int refs; + int allowed_seeks; // Seeks allowed until compaction + uint64_t number; + uint64_t file_size; // File size in bytes + InternalKey smallest; // Smallest internal key served by table + InternalKey largest; // Largest internal key served by table +}; + +class VersionEdit { + public: + VersionEdit() { Clear(); } + ~VersionEdit() = default; + + void Clear(); + + void SetComparatorName(const Slice& name) { + has_comparator_ = true; + comparator_ = name.ToString(); + } + void SetLogNumber(uint64_t num) { + has_log_number_ = true; + log_number_ = num; + } + void SetPrevLogNumber(uint64_t num) { + has_prev_log_number_ = true; + prev_log_number_ = num; + } + void SetNextFile(uint64_t num) { + has_next_file_number_ = true; + next_file_number_ = num; + } + void SetLastSequence(SequenceNumber seq) { + has_last_sequence_ = true; + last_sequence_ = seq; + } + void SetCompactPointer(int level, const InternalKey& key) { + compact_pointers_.push_back(std::make_pair(level, key)); + } + + // Add the specified file at the specified number. + // REQUIRES: This version has not been saved (see VersionSet::SaveTo) + // REQUIRES: "smallest" and "largest" are smallest and largest keys in file + void AddFile(int level, uint64_t file, uint64_t file_size, + const InternalKey& smallest, const InternalKey& largest) { + FileMetaData f; + f.number = file; + f.file_size = file_size; + f.smallest = smallest; + f.largest = largest; + new_files_.push_back(std::make_pair(level, f)); + } + + // Delete the specified "file" from the specified "level". + void RemoveFile(int level, uint64_t file) { + deleted_files_.insert(std::make_pair(level, file)); + } + + void EncodeTo(std::string* dst) const; + Status DecodeFrom(const Slice& src); + + std::string DebugString() const; + + private: + friend class VersionSet; + + typedef std::set> DeletedFileSet; + + std::string comparator_; + uint64_t log_number_; + uint64_t prev_log_number_; + uint64_t next_file_number_; + SequenceNumber last_sequence_; + bool has_comparator_; + bool has_log_number_; + bool has_prev_log_number_; + bool has_next_file_number_; + bool has_last_sequence_; + + std::vector> compact_pointers_; + DeletedFileSet deleted_files_; + std::vector> new_files_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_ diff --git a/leveldb/db/version_edit_test.cc b/leveldb/db/version_edit_test.cc new file mode 100644 index 000000000..a108c155b --- /dev/null +++ b/leveldb/db/version_edit_test.cc @@ -0,0 +1,41 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_edit.h" + +#include "gtest/gtest.h" + +namespace leveldb { + +static void TestEncodeDecode(const VersionEdit& edit) { + std::string encoded, encoded2; + edit.EncodeTo(&encoded); + VersionEdit parsed; + Status s = parsed.DecodeFrom(encoded); + ASSERT_TRUE(s.ok()) << s.ToString(); + parsed.EncodeTo(&encoded2); + ASSERT_EQ(encoded, encoded2); +} + +TEST(VersionEditTest, EncodeDecode) { + static const uint64_t kBig = 1ull << 50; + + VersionEdit edit; + for (int i = 0; i < 4; i++) { + TestEncodeDecode(edit); + edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, + InternalKey("foo", kBig + 500 + i, kTypeValue), + InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); + edit.RemoveFile(4, kBig + 700 + i); + edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); + } + + edit.SetComparatorName("foo"); + edit.SetLogNumber(kBig + 100); + edit.SetNextFile(kBig + 200); + edit.SetLastSequence(kBig + 1000); + TestEncodeDecode(edit); +} + +} // namespace leveldb diff --git a/leveldb/db/version_set.cc b/leveldb/db/version_set.cc new file mode 100644 index 000000000..fa7ef9208 --- /dev/null +++ b/leveldb/db/version_set.cc @@ -0,0 +1,1573 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_set.h" + +#include +#include + +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/memtable.h" +#include "db/table_cache.h" +#include "leveldb/env.h" +#include "leveldb/table_builder.h" +#include "table/merger.h" +#include "table/two_level_iterator.h" +#include "util/coding.h" +#include "util/logging.h" + +namespace leveldb { + +static size_t TargetFileSize(const Options* options) { + return options->max_file_size; +} + +// Maximum bytes of overlaps in grandparent (i.e., level+2) before we +// stop building a single file in a level->level+1 compaction. +static int64_t MaxGrandParentOverlapBytes(const Options* options) { + return 10 * TargetFileSize(options); +} + +// Maximum number of bytes in all compacted files. We avoid expanding +// the lower level file set of a compaction if it would make the +// total compaction cover more than this many bytes. +static int64_t ExpandedCompactionByteSizeLimit(const Options* options) { + return 25 * TargetFileSize(options); +} + +static double MaxBytesForLevel(const Options* options, int level) { + // Note: the result for level zero is not really used since we set + // the level-0 compaction threshold based on number of files. + + // Result for both level-0 and level-1 + double result = 10. * 1048576.0; + while (level > 1) { + result *= 10; + level--; + } + return result; +} + +static uint64_t MaxFileSizeForLevel(const Options* options, int level) { + // We could vary per level to reduce number of files? + return TargetFileSize(options); +} + +static int64_t TotalFileSize(const std::vector& files) { + int64_t sum = 0; + for (size_t i = 0; i < files.size(); i++) { + sum += files[i]->file_size; + } + return sum; +} + +Version::~Version() { + assert(refs_ == 0); + + // Remove from linked list + prev_->next_ = next_; + next_->prev_ = prev_; + + // Drop references to files + for (int level = 0; level < config::kNumLevels; level++) { + for (size_t i = 0; i < files_[level].size(); i++) { + FileMetaData* f = files_[level][i]; + assert(f->refs > 0); + f->refs--; + if (f->refs <= 0) { + delete f; + } + } + } +} + +int FindFile(const InternalKeyComparator& icmp, + const std::vector& files, const Slice& key) { + uint32_t left = 0; + uint32_t right = files.size(); + while (left < right) { + uint32_t mid = (left + right) / 2; + const FileMetaData* f = files[mid]; + if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) { + // Key at "mid.largest" is < "target". Therefore all + // files at or before "mid" are uninteresting. + left = mid + 1; + } else { + // Key at "mid.largest" is >= "target". Therefore all files + // after "mid" are uninteresting. + right = mid; + } + } + return right; +} + +static bool AfterFile(const Comparator* ucmp, const Slice* user_key, + const FileMetaData* f) { + // null user_key occurs before all keys and is therefore never after *f + return (user_key != nullptr && + ucmp->Compare(*user_key, f->largest.user_key()) > 0); +} + +static bool BeforeFile(const Comparator* ucmp, const Slice* user_key, + const FileMetaData* f) { + // null user_key occurs after all keys and is therefore never before *f + return (user_key != nullptr && + ucmp->Compare(*user_key, f->smallest.user_key()) < 0); +} + +bool SomeFileOverlapsRange(const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const std::vector& files, + const Slice* smallest_user_key, + const Slice* largest_user_key) { + const Comparator* ucmp = icmp.user_comparator(); + if (!disjoint_sorted_files) { + // Need to check against all files + for (size_t i = 0; i < files.size(); i++) { + const FileMetaData* f = files[i]; + if (AfterFile(ucmp, smallest_user_key, f) || + BeforeFile(ucmp, largest_user_key, f)) { + // No overlap + } else { + return true; // Overlap + } + } + return false; + } + + // Binary search over file list + uint32_t index = 0; + if (smallest_user_key != nullptr) { + // Find the earliest possible internal key for smallest_user_key + InternalKey small_key(*smallest_user_key, kMaxSequenceNumber, + kValueTypeForSeek); + index = FindFile(icmp, files, small_key.Encode()); + } + + if (index >= files.size()) { + // beginning of range is after all files, so no overlap. + return false; + } + + return !BeforeFile(ucmp, largest_user_key, files[index]); +} + +// An internal iterator. For a given version/level pair, yields +// information about the files in the level. For a given entry, key() +// is the largest key that occurs in the file, and value() is an +// 16-byte value containing the file number and file size, both +// encoded using EncodeFixed64. +class Version::LevelFileNumIterator : public Iterator { + public: + LevelFileNumIterator(const InternalKeyComparator& icmp, + const std::vector* flist) + : icmp_(icmp), flist_(flist), index_(flist->size()) { // Marks as invalid + } + bool Valid() const override { return index_ < flist_->size(); } + void Seek(const Slice& target) override { + index_ = FindFile(icmp_, *flist_, target); + } + void SeekToFirst() override { index_ = 0; } + void SeekToLast() override { + index_ = flist_->empty() ? 0 : flist_->size() - 1; + } + void Next() override { + assert(Valid()); + index_++; + } + void Prev() override { + assert(Valid()); + if (index_ == 0) { + index_ = flist_->size(); // Marks as invalid + } else { + index_--; + } + } + Slice key() const override { + assert(Valid()); + return (*flist_)[index_]->largest.Encode(); + } + Slice value() const override { + assert(Valid()); + EncodeFixed64(value_buf_, (*flist_)[index_]->number); + EncodeFixed64(value_buf_ + 8, (*flist_)[index_]->file_size); + return Slice(value_buf_, sizeof(value_buf_)); + } + Status status() const override { return Status::OK(); } + + private: + const InternalKeyComparator icmp_; + const std::vector* const flist_; + uint32_t index_; + + // Backing store for value(). Holds the file number and size. + mutable char value_buf_[16]; +}; + +static Iterator* GetFileIterator(void* arg, const ReadOptions& options, + const Slice& file_value) { + TableCache* cache = reinterpret_cast(arg); + if (file_value.size() != 16) { + return NewErrorIterator( + Status::Corruption("FileReader invoked with unexpected value")); + } else { + return cache->NewIterator(options, DecodeFixed64(file_value.data()), + DecodeFixed64(file_value.data() + 8)); + } +} + +Iterator* Version::NewConcatenatingIterator(const ReadOptions& options, + int level) const { + return NewTwoLevelIterator( + new LevelFileNumIterator(vset_->icmp_, &files_[level]), &GetFileIterator, + vset_->table_cache_, options); +} + +//TODO: add iterators for each layer +void Version::AddIterators(const ReadOptions& options, + std::vector* iters) { + // Merge all level zero files together since they may overlap + //blank3.1 + //TODO: add L0 iters to `iters`. + //Tips: learn the difference between L0 and Ln iters. + //Tips: the number of iterators for l0 refers to the number of L0 level elements in `files_`. + //Tips: L0 iters corresponding to sstables should be kept in the cache (use table_cache_). + + // For levels > 0, we can use a concatenating iterator that sequentially + // walks through the non-overlapping files in the level, opening them + // lazily. + + //blank3.1 + //TODO: add L1~Ln iters to `iters`. + //Tips: be different from the iterator creation method used by the L0 layer. + //Tips: note the presence of files at each level +} + +// Callback from TableCache::Get() +namespace { +enum SaverState { + kNotFound, + kFound, + kDeleted, + kCorrupt, +}; +struct Saver { + SaverState state; + const Comparator* ucmp; + Slice user_key; + std::string* value; +}; +} // namespace +static void SaveValue(void* arg, const Slice& ikey, const Slice& v) { + Saver* s = reinterpret_cast(arg); + ParsedInternalKey parsed_key; + if (!ParseInternalKey(ikey, &parsed_key)) { + s->state = kCorrupt; + } else { + if (s->ucmp->Compare(parsed_key.user_key, s->user_key) == 0) { + s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted; + if (s->state == kFound) { + s->value->assign(v.data(), v.size()); + } + } + } +} + +static bool NewestFirst(FileMetaData* a, FileMetaData* b) { + return a->number > b->number; +} + +void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, + bool (*func)(void*, int, FileMetaData*)) { + const Comparator* ucmp = vset_->icmp_.user_comparator(); + + // Search level-0 in order from newest to oldest. + std::vector tmp; + tmp.reserve(files_[0].size()); + for (uint32_t i = 0; i < files_[0].size(); i++) { + FileMetaData* f = files_[0][i]; + if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 && + ucmp->Compare(user_key, f->largest.user_key()) <= 0) { + tmp.push_back(f); + } + } + if (!tmp.empty()) { + std::sort(tmp.begin(), tmp.end(), NewestFirst); + for (uint32_t i = 0; i < tmp.size(); i++) { + if (!(*func)(arg, 0, tmp[i])) { + return; + } + } + } + + // Search other levels. + for (int level = 1; level < config::kNumLevels; level++) { + size_t num_files = files_[level].size(); + if (num_files == 0) continue; + + // Binary search to find earliest index whose largest key >= internal_key. + uint32_t index = FindFile(vset_->icmp_, files_[level], internal_key); + if (index < num_files) { + + // blank1.1 + // todo: fill blank1.1 to judge if the key can be in this file + // tips1: you should learn how it works in level0 fisrtly + + // add your code here + + // blank1.1 ends + } + } +} + +Status Version::Get(const ReadOptions& options, const LookupKey& k, + std::string* value, GetStats* stats) { + stats->seek_file = nullptr; + stats->seek_file_level = -1; + + struct State { + Saver saver; + GetStats* stats; + const ReadOptions* options; + Slice ikey; + FileMetaData* last_file_read; + int last_file_read_level; + + VersionSet* vset; + Status s; + bool found; + + static bool Match(void* arg, int level, FileMetaData* f) { + State* state = reinterpret_cast(arg); + + if (state->stats->seek_file == nullptr && + state->last_file_read != nullptr) { + // We have had more than one seek for this read. Charge the 1st file. + state->stats->seek_file = state->last_file_read; + state->stats->seek_file_level = state->last_file_read_level; + } + + state->last_file_read = f; + state->last_file_read_level = level; + + state->s = state->vset->table_cache_->Get(*state->options, f->number, + f->file_size, state->ikey, + &state->saver, SaveValue); + if (!state->s.ok()) { + state->found = true; + return false; + } + + // blank1.2 + // todo: fill blank1.2 to return this Match() function + // tips1: Several cases are missing here, please supplement them + // tips2: See the enum 'SaverState' definition + // tips3: Understand what the state->found variable does and update it accordingly + switch (state->saver.state) { + case kNotFound: + return true; // Keep searching in other files + + // add your code here + + case kCorrupt: + state->s = + Status::Corruption("corrupted key for ", state->saver.user_key); + state->found = true; + return false; + } + //blank1.2 end + + // Not reached. Added to avoid false compilation warnings of + // "control reaches end of non-void function". + return false; + } + }; + + State state; + state.found = false; + state.stats = stats; + state.last_file_read = nullptr; + state.last_file_read_level = -1; + + state.options = &options; + state.ikey = k.internal_key(); + state.vset = vset_; + + state.saver.state = kNotFound; + state.saver.ucmp = vset_->icmp_.user_comparator(); + state.saver.user_key = k.user_key(); + state.saver.value = value; + + ForEachOverlapping(state.saver.user_key, state.ikey, &state, &State::Match); + + return state.found ? state.s : Status::NotFound(Slice()); +} + +bool Version::UpdateStats(const GetStats& stats) { + FileMetaData* f = stats.seek_file; + if (f != nullptr) { + f->allowed_seeks--; + if (f->allowed_seeks <= 0 && file_to_compact_ == nullptr) { + file_to_compact_ = f; + file_to_compact_level_ = stats.seek_file_level; + return true; + } + } + return false; +} + +bool Version::RecordReadSample(Slice internal_key) { + ParsedInternalKey ikey; + if (!ParseInternalKey(internal_key, &ikey)) { + return false; + } + + struct State { + GetStats stats; // Holds first matching file + int matches; + + static bool Match(void* arg, int level, FileMetaData* f) { + State* state = reinterpret_cast(arg); + state->matches++; + if (state->matches == 1) { + // Remember first match. + state->stats.seek_file = f; + state->stats.seek_file_level = level; + } + // We can stop iterating once we have a second match. + return state->matches < 2; + } + }; + + State state; + state.matches = 0; + ForEachOverlapping(ikey.user_key, internal_key, &state, &State::Match); + + // Must have at least two matches since we want to merge across + // files. But what if we have a single file that contains many + // overwrites and deletions? Should we have another mechanism for + // finding such files? + if (state.matches >= 2) { + // 1MB cost is about 1 seek (see comment in Builder::Apply). + return UpdateStats(state.stats); + } + return false; +} + +void Version::Ref() { ++refs_; } + +void Version::Unref() { + assert(this != &vset_->dummy_versions_); + assert(refs_ >= 1); + --refs_; + if (refs_ == 0) { + delete this; + } +} + +bool Version::OverlapInLevel(int level, const Slice* smallest_user_key, + const Slice* largest_user_key) { + return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level], + smallest_user_key, largest_user_key); +} + +int Version::PickLevelForMemTableOutput(const Slice& smallest_user_key, + const Slice& largest_user_key) { + int level = 0; + if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) { + // Push to next level if there is no overlap in next level, + // and the #bytes overlapping in the level after that are limited. + InternalKey start(smallest_user_key, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey limit(largest_user_key, 0, static_cast(0)); + std::vector overlaps; + while (level < config::kMaxMemCompactLevel) { + if (OverlapInLevel(level + 1, &smallest_user_key, &largest_user_key)) { + break; + } + if (level + 2 < config::kNumLevels) { + // Check that file does not overlap too many grandparent bytes. + GetOverlappingInputs(level + 2, &start, &limit, &overlaps); + const int64_t sum = TotalFileSize(overlaps); + if (sum > MaxGrandParentOverlapBytes(vset_->options_)) { + break; + } + } + level++; + } + } + return level; +} + +// Store in "*inputs" all files in "level" that overlap [begin,end] +void Version::GetOverlappingInputs(int level, const InternalKey* begin, + const InternalKey* end, + std::vector* inputs) { + assert(level >= 0); + assert(level < config::kNumLevels); + inputs->clear(); + Slice user_begin, user_end; + if (begin != nullptr) { + user_begin = begin->user_key(); + } + if (end != nullptr) { + user_end = end->user_key(); + } + const Comparator* user_cmp = vset_->icmp_.user_comparator(); + for (size_t i = 0; i < files_[level].size();) { + FileMetaData* f = files_[level][i++]; + const Slice file_start = f->smallest.user_key(); + const Slice file_limit = f->largest.user_key(); + if (begin != nullptr && user_cmp->Compare(file_limit, user_begin) < 0) { + // "f" is completely before specified range; skip it + } else if (end != nullptr && user_cmp->Compare(file_start, user_end) > 0) { + // "f" is completely after specified range; skip it + } else { + inputs->push_back(f); + if (level == 0) { + // Level-0 files may overlap each other. So check if the newly + // added file has expanded the range. If so, restart search. + if (begin != nullptr && user_cmp->Compare(file_start, user_begin) < 0) { + user_begin = file_start; + inputs->clear(); + i = 0; + } else if (end != nullptr && + user_cmp->Compare(file_limit, user_end) > 0) { + user_end = file_limit; + inputs->clear(); + i = 0; + } + } + } + } +} + +std::string Version::DebugString() const { + std::string r; + for (int level = 0; level < config::kNumLevels; level++) { + // E.g., + // --- level 1 --- + // 17:123['a' .. 'd'] + // 20:43['e' .. 'g'] + r.append("--- level "); + AppendNumberTo(&r, level); + r.append(" ---\n"); + const std::vector& files = files_[level]; + for (size_t i = 0; i < files.size(); i++) { + r.push_back(' '); + AppendNumberTo(&r, files[i]->number); + r.push_back(':'); + AppendNumberTo(&r, files[i]->file_size); + r.append("["); + r.append(files[i]->smallest.DebugString()); + r.append(" .. "); + r.append(files[i]->largest.DebugString()); + r.append("]\n"); + } + } + return r; +} + +// A helper class so we can efficiently apply a whole sequence +// of edits to a particular state without creating intermediate +// Versions that contain full copies of the intermediate state. +class VersionSet::Builder { + private: + // Helper to sort by v->files_[file_number].smallest + struct BySmallestKey { + const InternalKeyComparator* internal_comparator; + + bool operator()(FileMetaData* f1, FileMetaData* f2) const { + int r = internal_comparator->Compare(f1->smallest, f2->smallest); + if (r != 0) { + return (r < 0); + } else { + // Break ties by file number + return (f1->number < f2->number); + } + } + }; + + typedef std::set FileSet; + struct LevelState { + std::set deleted_files; + FileSet* added_files; + }; + + VersionSet* vset_; + Version* base_; + LevelState levels_[config::kNumLevels]; + + public: + // Initialize a builder with the files from *base and other info from *vset + Builder(VersionSet* vset, Version* base) : vset_(vset), base_(base) { + base_->Ref(); + BySmallestKey cmp; + cmp.internal_comparator = &vset_->icmp_; + for (int level = 0; level < config::kNumLevels; level++) { + levels_[level].added_files = new FileSet(cmp); + } + } + + ~Builder() { + for (int level = 0; level < config::kNumLevels; level++) { + const FileSet* added = levels_[level].added_files; + std::vector to_unref; + to_unref.reserve(added->size()); + for (FileSet::const_iterator it = added->begin(); it != added->end(); + ++it) { + to_unref.push_back(*it); + } + delete added; + for (uint32_t i = 0; i < to_unref.size(); i++) { + FileMetaData* f = to_unref[i]; + f->refs--; + if (f->refs <= 0) { + delete f; + } + } + } + base_->Unref(); + } + + // Apply all of the edits in *edit to the current state. + void Apply(const VersionEdit* edit) { + // Update compaction pointers + for (size_t i = 0; i < edit->compact_pointers_.size(); i++) { + const int level = edit->compact_pointers_[i].first; + vset_->compact_pointer_[level] = + edit->compact_pointers_[i].second.Encode().ToString(); + } + + // Delete files + for (const auto& deleted_file_set_kvp : edit->deleted_files_) { + const int level = deleted_file_set_kvp.first; + const uint64_t number = deleted_file_set_kvp.second; + levels_[level].deleted_files.insert(number); + } + + // Add new files + for (size_t i = 0; i < edit->new_files_.size(); i++) { + const int level = edit->new_files_[i].first; + FileMetaData* f = new FileMetaData(edit->new_files_[i].second); + f->refs = 1; + + // We arrange to automatically compact this file after + // a certain number of seeks. Let's assume: + // (1) One seek costs 10ms + // (2) Writing or reading 1MB costs 10ms (100MB/s) + // (3) A compaction of 1MB does 25MB of IO: + // 1MB read from this level + // 10-12MB read from next level (boundaries may be misaligned) + // 10-12MB written to next level + // This implies that 25 seeks cost the same as the compaction + // of 1MB of data. I.e., one seek costs approximately the + // same as the compaction of 40KB of data. We are a little + // conservative and allow approximately one seek for every 16KB + // of data before triggering a compaction. + f->allowed_seeks = static_cast((f->file_size / 16384U)); + if (f->allowed_seeks < 100) f->allowed_seeks = 100; + + levels_[level].deleted_files.erase(f->number); + levels_[level].added_files->insert(f); + } + } + + // Save the current state in *v. + void SaveTo(Version* v) { + BySmallestKey cmp; + cmp.internal_comparator = &vset_->icmp_; + for (int level = 0; level < config::kNumLevels; level++) { + // Merge the set of added files with the set of pre-existing files. + // Drop any deleted files. Store the result in *v. + const std::vector& base_files = base_->files_[level]; + std::vector::const_iterator base_iter = base_files.begin(); + std::vector::const_iterator base_end = base_files.end(); + const FileSet* added_files = levels_[level].added_files; + v->files_[level].reserve(base_files.size() + added_files->size()); + for (const auto& added_file : *added_files) { + // Add all smaller files listed in base_ + for (std::vector::const_iterator bpos = + std::upper_bound(base_iter, base_end, added_file, cmp); + base_iter != bpos; ++base_iter) { + MaybeAddFile(v, level, *base_iter); + } + + MaybeAddFile(v, level, added_file); + } + + // Add remaining base files + for (; base_iter != base_end; ++base_iter) { + MaybeAddFile(v, level, *base_iter); + } + +#ifndef NDEBUG + // Make sure there is no overlap in levels > 0 + if (level > 0) { + for (uint32_t i = 1; i < v->files_[level].size(); i++) { + const InternalKey& prev_end = v->files_[level][i - 1]->largest; + const InternalKey& this_begin = v->files_[level][i]->smallest; + if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) { + std::fprintf(stderr, "overlapping ranges in same level %s vs. %s\n", + prev_end.DebugString().c_str(), + this_begin.DebugString().c_str()); + std::abort(); + } + } + } +#endif + } + } + + void MaybeAddFile(Version* v, int level, FileMetaData* f) { + if (levels_[level].deleted_files.count(f->number) > 0) { + // File is deleted: do nothing + } else { + std::vector* files = &v->files_[level]; + if (level > 0 && !files->empty()) { + // Must not overlap + assert(vset_->icmp_.Compare((*files)[files->size() - 1]->largest, + f->smallest) < 0); + } + f->refs++; + files->push_back(f); + } + } +}; + +VersionSet::VersionSet(const std::string& dbname, const Options* options, + TableCache* table_cache, + const InternalKeyComparator* cmp) + : env_(options->env), + dbname_(dbname), + options_(options), + table_cache_(table_cache), + icmp_(*cmp), + next_file_number_(2), + manifest_file_number_(0), // Filled by Recover() + last_sequence_(0), + log_number_(0), + prev_log_number_(0), + descriptor_file_(nullptr), + descriptor_log_(nullptr), + dummy_versions_(this), + current_(nullptr) { + AppendVersion(new Version(this)); +} + +VersionSet::~VersionSet() { + current_->Unref(); + assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty + delete descriptor_log_; + delete descriptor_file_; +} + +void VersionSet::AppendVersion(Version* v) { + // Make "v" current + assert(v->refs_ == 0); + assert(v != current_); + if (current_ != nullptr) { + current_->Unref(); + } + current_ = v; + v->Ref(); + + // Append to linked list + v->prev_ = dummy_versions_.prev_; + v->next_ = &dummy_versions_; + v->prev_->next_ = v; + v->next_->prev_ = v; +} + +Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { + if (edit->has_log_number_) { + assert(edit->log_number_ >= log_number_); + assert(edit->log_number_ < next_file_number_); + } else { + edit->SetLogNumber(log_number_); + } + + if (!edit->has_prev_log_number_) { + edit->SetPrevLogNumber(prev_log_number_); + } + + edit->SetNextFile(next_file_number_); + edit->SetLastSequence(last_sequence_); + + Version* v = new Version(this); + { + Builder builder(this, current_); + builder.Apply(edit); + builder.SaveTo(v); + } + Finalize(v); + + // Initialize new descriptor log file if necessary by creating + // a temporary file that contains a snapshot of the current version. + std::string new_manifest_file; + Status s; + if (descriptor_log_ == nullptr) { + // No reason to unlock *mu here since we only hit this path in the + // first call to LogAndApply (when opening the database). + assert(descriptor_file_ == nullptr); + new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_); + s = env_->NewWritableFile(new_manifest_file, &descriptor_file_); + if (s.ok()) { + descriptor_log_ = new log::Writer(descriptor_file_); + s = WriteSnapshot(descriptor_log_); + } + } + + // Unlock during expensive MANIFEST log write + { + mu->Unlock(); + + // Write new record to MANIFEST log + if (s.ok()) { + std::string record; + edit->EncodeTo(&record); + s = descriptor_log_->AddRecord(record); + if (s.ok()) { + s = descriptor_file_->Sync(); + } + if (!s.ok()) { + Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str()); + } + } + + // If we just created a new descriptor file, install it by writing a + // new CURRENT file that points to it. + if (s.ok() && !new_manifest_file.empty()) { + s = SetCurrentFile(env_, dbname_, manifest_file_number_); + } + + mu->Lock(); + } + + // Install the new version + if (s.ok()) { + AppendVersion(v); + log_number_ = edit->log_number_; + prev_log_number_ = edit->prev_log_number_; + } else { + delete v; + if (!new_manifest_file.empty()) { + delete descriptor_log_; + delete descriptor_file_; + descriptor_log_ = nullptr; + descriptor_file_ = nullptr; + env_->RemoveFile(new_manifest_file); + } + } + + return s; +} + +Status VersionSet::Recover(bool* save_manifest) { + struct LogReporter : public log::Reader::Reporter { + Status* status; + void Corruption(size_t bytes, const Status& s) override { + if (this->status->ok()) *this->status = s; + } + }; + + // Read "CURRENT" file, which contains a pointer to the current manifest file + std::string current; + Status s = ReadFileToString(env_, CurrentFileName(dbname_), ¤t); + if (!s.ok()) { + return s; + } + if (current.empty() || current[current.size() - 1] != '\n') { + return Status::Corruption("CURRENT file does not end with newline"); + } + current.resize(current.size() - 1); + + std::string dscname = dbname_ + "/" + current; + SequentialFile* file; + s = env_->NewSequentialFile(dscname, &file); + if (!s.ok()) { + if (s.IsNotFound()) { + return Status::Corruption("CURRENT points to a non-existent file", + s.ToString()); + } + return s; + } + + bool have_log_number = false; + bool have_prev_log_number = false; + bool have_next_file = false; + bool have_last_sequence = false; + uint64_t next_file = 0; + uint64_t last_sequence = 0; + uint64_t log_number = 0; + uint64_t prev_log_number = 0; + Builder builder(this, current_); + int read_records = 0; + + { + LogReporter reporter; + reporter.status = &s; + log::Reader reader(file, &reporter, true /*checksum*/, + 0 /*initial_offset*/); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch) && s.ok()) { + ++read_records; + VersionEdit edit; + s = edit.DecodeFrom(record); + if (s.ok()) { + if (edit.has_comparator_ && + edit.comparator_ != icmp_.user_comparator()->Name()) { + s = Status::InvalidArgument( + edit.comparator_ + " does not match existing comparator ", + icmp_.user_comparator()->Name()); + } + } + + if (s.ok()) { + builder.Apply(&edit); + } + + if (edit.has_log_number_) { + log_number = edit.log_number_; + have_log_number = true; + } + + if (edit.has_prev_log_number_) { + prev_log_number = edit.prev_log_number_; + have_prev_log_number = true; + } + + if (edit.has_next_file_number_) { + next_file = edit.next_file_number_; + have_next_file = true; + } + + if (edit.has_last_sequence_) { + last_sequence = edit.last_sequence_; + have_last_sequence = true; + } + } + } + delete file; + file = nullptr; + + if (s.ok()) { + if (!have_next_file) { + s = Status::Corruption("no meta-nextfile entry in descriptor"); + } else if (!have_log_number) { + s = Status::Corruption("no meta-lognumber entry in descriptor"); + } else if (!have_last_sequence) { + s = Status::Corruption("no last-sequence-number entry in descriptor"); + } + + if (!have_prev_log_number) { + prev_log_number = 0; + } + + MarkFileNumberUsed(prev_log_number); + MarkFileNumberUsed(log_number); + } + + if (s.ok()) { + Version* v = new Version(this); + builder.SaveTo(v); + // Install recovered version + Finalize(v); + AppendVersion(v); + manifest_file_number_ = next_file; + next_file_number_ = next_file + 1; + last_sequence_ = last_sequence; + log_number_ = log_number; + prev_log_number_ = prev_log_number; + + // See if we can reuse the existing MANIFEST file. + if (ReuseManifest(dscname, current)) { + // No need to save new manifest + } else { + *save_manifest = true; + } + } else { + std::string error = s.ToString(); + Log(options_->info_log, "Error recovering version set with %d records: %s", + read_records, error.c_str()); + } + + return s; +} + +bool VersionSet::ReuseManifest(const std::string& dscname, + const std::string& dscbase) { + if (!options_->reuse_logs) { + return false; + } + FileType manifest_type; + uint64_t manifest_number; + uint64_t manifest_size; + if (!ParseFileName(dscbase, &manifest_number, &manifest_type) || + manifest_type != kDescriptorFile || + !env_->GetFileSize(dscname, &manifest_size).ok() || + // Make new compacted MANIFEST if old one is too big + manifest_size >= TargetFileSize(options_)) { + return false; + } + + assert(descriptor_file_ == nullptr); + assert(descriptor_log_ == nullptr); + Status r = env_->NewAppendableFile(dscname, &descriptor_file_); + if (!r.ok()) { + Log(options_->info_log, "Reuse MANIFEST: %s\n", r.ToString().c_str()); + assert(descriptor_file_ == nullptr); + return false; + } + + Log(options_->info_log, "Reusing MANIFEST %s\n", dscname.c_str()); + descriptor_log_ = new log::Writer(descriptor_file_, manifest_size); + manifest_file_number_ = manifest_number; + return true; +} + +void VersionSet::MarkFileNumberUsed(uint64_t number) { + if (next_file_number_ <= number) { + next_file_number_ = number + 1; + } +} + +void VersionSet::Finalize(Version* v) { + // Precomputed best level for next compaction + int best_level = -1; + double best_score = -1; + + for (int level = 0; level < config::kNumLevels - 1; level++) { + double score; + //TODO: Calculate the score of each level. Find the best_level and best_score + //Tips1: We treat level-0 specially by bounding the number of files + // instead of number of bytes for two reasons: + // + // (1) With larger write-buffer sizes, it is nice not to do too + // many level-0 compactions. + // + // (2) The files in level-0 are merged on every read and + // therefore we wish to avoid too many files when the individual + // file size is small (perhaps because of a small write-buffer + // setting, or very high compression ratios, or lots of + // overwrites/deletions). + // The score of level-0 is equal to the ratio of level-0 file number to "static_cast(config::kL0_CompactionTrigger);" + // "kL0_CompactionTrigger" is defined in db/dbformat.h. It means level-0 compaction is started when we hit this many files. + //Tips2: "v->files_" is the list of files per level. + //Tips3: The score of other level is equal to the ratio of total file size to max bytes limit + // To calculate total file size, use the function "TotalFileSize()". It is defined in db/version_set.cc + // To calculate max bytes limit, use the function "MaxBytesForLevel()". It is defined in db/version_set.cc + } + + v->compaction_level_ = best_level; + v->compaction_score_ = best_score; +} + +Status VersionSet::WriteSnapshot(log::Writer* log) { + // TODO: Break up into multiple records to reduce memory usage on recovery? + + // Save metadata + VersionEdit edit; + edit.SetComparatorName(icmp_.user_comparator()->Name()); + + // Save compaction pointers + for (int level = 0; level < config::kNumLevels; level++) { + if (!compact_pointer_[level].empty()) { + InternalKey key; + key.DecodeFrom(compact_pointer_[level]); + edit.SetCompactPointer(level, key); + } + } + + // Save files + for (int level = 0; level < config::kNumLevels; level++) { + const std::vector& files = current_->files_[level]; + for (size_t i = 0; i < files.size(); i++) { + const FileMetaData* f = files[i]; + edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest); + } + } + + std::string record; + edit.EncodeTo(&record); + return log->AddRecord(record); +} + +int VersionSet::NumLevelFiles(int level) const { + assert(level >= 0); + assert(level < config::kNumLevels); + return current_->files_[level].size(); +} + +const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const { + // Update code if kNumLevels changes + static_assert(config::kNumLevels == 7, ""); + std::snprintf( + scratch->buffer, sizeof(scratch->buffer), "files[ %d %d %d %d %d %d %d ]", + int(current_->files_[0].size()), int(current_->files_[1].size()), + int(current_->files_[2].size()), int(current_->files_[3].size()), + int(current_->files_[4].size()), int(current_->files_[5].size()), + int(current_->files_[6].size())); + return scratch->buffer; +} + +uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { + uint64_t result = 0; + for (int level = 0; level < config::kNumLevels; level++) { + const std::vector& files = v->files_[level]; + for (size_t i = 0; i < files.size(); i++) { + if (icmp_.Compare(files[i]->largest, ikey) <= 0) { + // Entire file is before "ikey", so just add the file size + result += files[i]->file_size; + } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) { + // Entire file is after "ikey", so ignore + if (level > 0) { + // Files other than level 0 are sorted by meta->smallest, so + // no further files in this level will contain data for + // "ikey". + break; + } + } else { + // "ikey" falls in the range for this table. Add the + // approximate offset of "ikey" within the table. + Table* tableptr; + Iterator* iter = table_cache_->NewIterator( + ReadOptions(), files[i]->number, files[i]->file_size, &tableptr); + if (tableptr != nullptr) { + result += tableptr->ApproximateOffsetOf(ikey.Encode()); + } + delete iter; + } + } + } + return result; +} + +void VersionSet::AddLiveFiles(std::set* live) { + for (Version* v = dummy_versions_.next_; v != &dummy_versions_; + v = v->next_) { + for (int level = 0; level < config::kNumLevels; level++) { + const std::vector& files = v->files_[level]; + for (size_t i = 0; i < files.size(); i++) { + live->insert(files[i]->number); + } + } + } +} + +int64_t VersionSet::NumLevelBytes(int level) const { + assert(level >= 0); + assert(level < config::kNumLevels); + return TotalFileSize(current_->files_[level]); +} + +int64_t VersionSet::MaxNextLevelOverlappingBytes() { + int64_t result = 0; + std::vector overlaps; + for (int level = 1; level < config::kNumLevels - 1; level++) { + for (size_t i = 0; i < current_->files_[level].size(); i++) { + const FileMetaData* f = current_->files_[level][i]; + current_->GetOverlappingInputs(level + 1, &f->smallest, &f->largest, + &overlaps); + const int64_t sum = TotalFileSize(overlaps); + if (sum > result) { + result = sum; + } + } + } + return result; +} + +// Stores the minimal range that covers all entries in inputs in +// *smallest, *largest. +// REQUIRES: inputs is not empty +void VersionSet::GetRange(const std::vector& inputs, + InternalKey* smallest, InternalKey* largest) { + assert(!inputs.empty()); + smallest->Clear(); + largest->Clear(); + for (size_t i = 0; i < inputs.size(); i++) { + FileMetaData* f = inputs[i]; + if (i == 0) { + *smallest = f->smallest; + *largest = f->largest; + } else { + if (icmp_.Compare(f->smallest, *smallest) < 0) { + *smallest = f->smallest; + } + if (icmp_.Compare(f->largest, *largest) > 0) { + *largest = f->largest; + } + } + } +} + +// Stores the minimal range that covers all entries in inputs1 and inputs2 +// in *smallest, *largest. +// REQUIRES: inputs is not empty +void VersionSet::GetRange2(const std::vector& inputs1, + const std::vector& inputs2, + InternalKey* smallest, InternalKey* largest) { + std::vector all = inputs1; + all.insert(all.end(), inputs2.begin(), inputs2.end()); + GetRange(all, smallest, largest); +} + +Iterator* VersionSet::MakeInputIterator(Compaction* c) { + ReadOptions options; + options.verify_checksums = options_->paranoid_checks; + options.fill_cache = false; + + // Level-0 files have to be merged together. For other levels, + // we will make a concatenating iterator per level. + // TODO(opt): use concatenating iterator for level-0 if there is no overlap + const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2); + Iterator** list = new Iterator*[space]; + int num = 0; + for (int which = 0; which < 2; which++) { + if (!c->inputs_[which].empty()) { + if (c->level() + which == 0) { + const std::vector& files = c->inputs_[which]; + for (size_t i = 0; i < files.size(); i++) { + list[num++] = table_cache_->NewIterator(options, files[i]->number, + files[i]->file_size); + } + } else { + // Create concatenating iterator for the files from this level + list[num++] = NewTwoLevelIterator( + new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]), + &GetFileIterator, table_cache_, options); + } + } + } + assert(num <= space); + Iterator* result = NewMergingIterator(&icmp_, list, num); + delete[] list; + return result; +} + +/// @brief +/// @return +Compaction* VersionSet::PickCompaction() { + Compaction* c; + int level; + + // We prefer compactions triggered by too much data in a level over + // the compactions triggered by seeks. + const bool size_compaction = (current_->compaction_score_ >= 1); + const bool seek_compaction = (current_->file_to_compact_ != nullptr); + if (size_compaction) { + level = current_->compaction_level_; + assert(level >= 0); + assert(level + 1 < config::kNumLevels); + c = new Compaction(options_, level); + + // Pick the first file that comes after compact_pointer_[level] + for (size_t i = 0; i < current_->files_[level].size(); i++) { + FileMetaData* f = current_->files_[level][i]; + //TODO: Pick the first file for compaction + //Tips1: Compact_pointer_ is defined in db/versionset.h, it points per-level key at which the next compaction at that level should start. + //Tips2: You should find the first file whose largest key is bigger than compact_pointer_[level]. + //Tips3: To compare two keys, you should use the function "icmp_.Compare()". It is defined in db/format.cc + //Tips4: "f->largest" is an InternalKey, you should use "f->largest.Encode()" to transfer it to a Slice. + //Tips5: If compact_pointer_[level] is empty, you should pick current_->files[level][0] directly. + //Tips6: Once you find the file, put it to "c->inputs_[0]" + } + if (c->inputs_[0].empty()) { + // Wrap-around to the beginning of the key space + c->inputs_[0].push_back(current_->files_[level][0]); + } + } else if (seek_compaction) { + level = current_->file_to_compact_level_; + c = new Compaction(options_, level); + c->inputs_[0].push_back(current_->file_to_compact_); + } else { + return nullptr; + } + + c->input_version_ = current_; + c->input_version_->Ref(); + + // Files in level 0 may overlap each other, so pick up all overlapping ones + if (level == 0) { + InternalKey smallest, largest; + GetRange(c->inputs_[0], &smallest, &largest); + // Note that the next call will discard the file we placed in + // c->inputs_[0] earlier and replace it with an overlapping set + // which will include the picked file. + current_->GetOverlappingInputs(0, &smallest, &largest, &c->inputs_[0]); + assert(!c->inputs_[0].empty()); + } + + SetupOtherInputs(c); + + return c; +} + +// Finds the largest key in a vector of files. Returns true if files is not +// empty. +bool FindLargestKey(const InternalKeyComparator& icmp, + const std::vector& files, + InternalKey* largest_key) { + if (files.empty()) { + return false; + } + *largest_key = files[0]->largest; + for (size_t i = 1; i < files.size(); ++i) { + FileMetaData* f = files[i]; + if (icmp.Compare(f->largest, *largest_key) > 0) { + *largest_key = f->largest; + } + } + return true; +} + +// Finds minimum file b2=(l2, u2) in level file for which l2 > u1 and +// user_key(l2) = user_key(u1) +FileMetaData* FindSmallestBoundaryFile( + const InternalKeyComparator& icmp, + const std::vector& level_files, + const InternalKey& largest_key) { + const Comparator* user_cmp = icmp.user_comparator(); + FileMetaData* smallest_boundary_file = nullptr; + for (size_t i = 0; i < level_files.size(); ++i) { + FileMetaData* f = level_files[i]; + if (icmp.Compare(f->smallest, largest_key) > 0 && + user_cmp->Compare(f->smallest.user_key(), largest_key.user_key()) == + 0) { + if (smallest_boundary_file == nullptr || + icmp.Compare(f->smallest, smallest_boundary_file->smallest) < 0) { + smallest_boundary_file = f; + } + } + } + return smallest_boundary_file; +} + +// Extracts the largest file b1 from |compaction_files| and then searches for a +// b2 in |level_files| for which user_key(u1) = user_key(l2). If it finds such a +// file b2 (known as a boundary file) it adds it to |compaction_files| and then +// searches again using this new upper bound. +// +// If there are two blocks, b1=(l1, u1) and b2=(l2, u2) and +// user_key(u1) = user_key(l2), and if we compact b1 but not b2 then a +// subsequent get operation will yield an incorrect result because it will +// return the record from b2 in level i rather than from b1 because it searches +// level by level for records matching the supplied user key. +// +// parameters: +// in level_files: List of files to search for boundary files. +// in/out compaction_files: List of files to extend by adding boundary files. +void AddBoundaryInputs(const InternalKeyComparator& icmp, + const std::vector& level_files, + std::vector* compaction_files) { + InternalKey largest_key; + + // Quick return if compaction_files is empty. + if (!FindLargestKey(icmp, *compaction_files, &largest_key)) { + return; + } + + bool continue_searching = true; + while (continue_searching) { + FileMetaData* smallest_boundary_file = + FindSmallestBoundaryFile(icmp, level_files, largest_key); + + // If a boundary file was found advance largest_key, otherwise we're done. + if (smallest_boundary_file != NULL) { + compaction_files->push_back(smallest_boundary_file); + largest_key = smallest_boundary_file->largest; + } else { + continue_searching = false; + } + } +} + +void VersionSet::SetupOtherInputs(Compaction* c) { + const int level = c->level(); + InternalKey smallest, largest; + + AddBoundaryInputs(icmp_, current_->files_[level], &c->inputs_[0]); + GetRange(c->inputs_[0], &smallest, &largest); + + current_->GetOverlappingInputs(level + 1, &smallest, &largest, + &c->inputs_[1]); + AddBoundaryInputs(icmp_, current_->files_[level + 1], &c->inputs_[1]); + + // Get entire range covered by compaction + InternalKey all_start, all_limit; + GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit); + + // See if we can grow the number of inputs in "level" without + // changing the number of "level+1" files we pick up. + if (!c->inputs_[1].empty()) { + std::vector expanded0; + current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0); + AddBoundaryInputs(icmp_, current_->files_[level], &expanded0); + const int64_t inputs0_size = TotalFileSize(c->inputs_[0]); + const int64_t inputs1_size = TotalFileSize(c->inputs_[1]); + const int64_t expanded0_size = TotalFileSize(expanded0); + if (expanded0.size() > c->inputs_[0].size() && + inputs1_size + expanded0_size < + ExpandedCompactionByteSizeLimit(options_)) { + InternalKey new_start, new_limit; + GetRange(expanded0, &new_start, &new_limit); + std::vector expanded1; + current_->GetOverlappingInputs(level + 1, &new_start, &new_limit, + &expanded1); + AddBoundaryInputs(icmp_, current_->files_[level + 1], &expanded1); + if (expanded1.size() == c->inputs_[1].size()) { + Log(options_->info_log, + "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n", + level, int(c->inputs_[0].size()), int(c->inputs_[1].size()), + long(inputs0_size), long(inputs1_size), int(expanded0.size()), + int(expanded1.size()), long(expanded0_size), long(inputs1_size)); + smallest = new_start; + largest = new_limit; + c->inputs_[0] = expanded0; + c->inputs_[1] = expanded1; + GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit); + } + } + } + + // Compute the set of grandparent files that overlap this compaction + // (parent == level+1; grandparent == level+2) + if (level + 2 < config::kNumLevels) { + current_->GetOverlappingInputs(level + 2, &all_start, &all_limit, + &c->grandparents_); + } + + // Update the place where we will do the next compaction for this level. + // We update this immediately instead of waiting for the VersionEdit + // to be applied so that if the compaction fails, we will try a different + // key range next time. + compact_pointer_[level] = largest.Encode().ToString(); + c->edit_.SetCompactPointer(level, largest); +} + +Compaction* VersionSet::CompactRange(int level, const InternalKey* begin, + const InternalKey* end) { + std::vector inputs; + current_->GetOverlappingInputs(level, begin, end, &inputs); + if (inputs.empty()) { + return nullptr; + } + + // Avoid compacting too much in one shot in case the range is large. + // But we cannot do this for level-0 since level-0 files can overlap + // and we must not pick one file and drop another older file if the + // two files overlap. + if (level > 0) { + const uint64_t limit = MaxFileSizeForLevel(options_, level); + uint64_t total = 0; + for (size_t i = 0; i < inputs.size(); i++) { + uint64_t s = inputs[i]->file_size; + total += s; + if (total >= limit) { + inputs.resize(i + 1); + break; + } + } + } + + Compaction* c = new Compaction(options_, level); + c->input_version_ = current_; + c->input_version_->Ref(); + c->inputs_[0] = inputs; + SetupOtherInputs(c); + return c; +} + +Compaction::Compaction(const Options* options, int level) + : level_(level), + max_output_file_size_(MaxFileSizeForLevel(options, level)), + input_version_(nullptr), + grandparent_index_(0), + seen_key_(false), + overlapped_bytes_(0) { + for (int i = 0; i < config::kNumLevels; i++) { + level_ptrs_[i] = 0; + } +} + +Compaction::~Compaction() { + if (input_version_ != nullptr) { + input_version_->Unref(); + } +} + +bool Compaction::IsTrivialMove() const { + const VersionSet* vset = input_version_->vset_; + // Avoid a move if there is lots of overlapping grandparent data. + // Otherwise, the move could create a parent file that will require + // a very expensive merge later on. + return (num_input_files(0) == 1 && num_input_files(1) == 0 && + TotalFileSize(grandparents_) <= + MaxGrandParentOverlapBytes(vset->options_)); +} + +void Compaction::AddInputDeletions(VersionEdit* edit) { + for (int which = 0; which < 2; which++) { + for (size_t i = 0; i < inputs_[which].size(); i++) { + edit->RemoveFile(level_ + which, inputs_[which][i]->number); + } + } +} + +bool Compaction::IsBaseLevelForKey(const Slice& user_key) { + // Maybe use binary search to find right entry instead of linear search? + const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator(); + for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) { + const std::vector& files = input_version_->files_[lvl]; + while (level_ptrs_[lvl] < files.size()) { + FileMetaData* f = files[level_ptrs_[lvl]]; + if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { + // We've advanced far enough + if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) { + // Key falls in this file's range, so definitely not base level + return false; + } + break; + } + level_ptrs_[lvl]++; + } + } + return true; +} + +bool Compaction::ShouldStopBefore(const Slice& internal_key) { + const VersionSet* vset = input_version_->vset_; + // Scan to find earliest grandparent file that contains key. + const InternalKeyComparator* icmp = &vset->icmp_; + while (grandparent_index_ < grandparents_.size() && + icmp->Compare(internal_key, + grandparents_[grandparent_index_]->largest.Encode()) > + 0) { + if (seen_key_) { + overlapped_bytes_ += grandparents_[grandparent_index_]->file_size; + } + grandparent_index_++; + } + seen_key_ = true; + + if (overlapped_bytes_ > MaxGrandParentOverlapBytes(vset->options_)) { + // Too much overlap for current output; start new output + overlapped_bytes_ = 0; + return true; + } else { + return false; + } +} + +void Compaction::ReleaseInputs() { + if (input_version_ != nullptr) { + input_version_->Unref(); + input_version_ = nullptr; + } +} + +} // namespace leveldb diff --git a/leveldb/db/version_set.h b/leveldb/db/version_set.h new file mode 100644 index 000000000..ea0c925da --- /dev/null +++ b/leveldb/db/version_set.h @@ -0,0 +1,393 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// The representation of a DBImpl consists of a set of Versions. The +// newest version is called "current". Older versions may be kept +// around to provide a consistent view to live iterators. +// +// Each Version keeps track of a set of Table files per level. The +// entire set of versions is maintained in a VersionSet. +// +// Version,VersionSet are thread-compatible, but require external +// synchronization on all accesses. + +#ifndef STORAGE_LEVELDB_DB_VERSION_SET_H_ +#define STORAGE_LEVELDB_DB_VERSION_SET_H_ + +#include +#include +#include + +#include "db/dbformat.h" +#include "db/version_edit.h" +#include "port/port.h" +#include "port/thread_annotations.h" + +namespace leveldb { + +namespace log { +class Writer; +} + +class Compaction; +class Iterator; +class MemTable; +class TableBuilder; +class TableCache; +class Version; +class VersionSet; +class WritableFile; + +// Return the smallest index i such that files[i]->largest >= key. +// Return files.size() if there is no such file. +// REQUIRES: "files" contains a sorted list of non-overlapping files. +int FindFile(const InternalKeyComparator& icmp, + const std::vector& files, const Slice& key); + +// Returns true iff some file in "files" overlaps the user key range +// [*smallest,*largest]. +// smallest==nullptr represents a key smaller than all keys in the DB. +// largest==nullptr represents a key largest than all keys in the DB. +// REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges +// in sorted order. +bool SomeFileOverlapsRange(const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const std::vector& files, + const Slice* smallest_user_key, + const Slice* largest_user_key); + +class Version { + public: + struct GetStats { + FileMetaData* seek_file; + int seek_file_level; + }; + + // Append to *iters a sequence of iterators that will + // yield the contents of this Version when merged together. + // REQUIRES: This version has been saved (see VersionSet::SaveTo) + void AddIterators(const ReadOptions&, std::vector* iters); + + // Lookup the value for key. If found, store it in *val and + // return OK. Else return a non-OK status. Fills *stats. + // REQUIRES: lock is not held + Status Get(const ReadOptions&, const LookupKey& key, std::string* val, + GetStats* stats); + + // Adds "stats" into the current state. Returns true if a new + // compaction may need to be triggered, false otherwise. + // REQUIRES: lock is held + bool UpdateStats(const GetStats& stats); + + // Record a sample of bytes read at the specified internal key. + // Samples are taken approximately once every config::kReadBytesPeriod + // bytes. Returns true if a new compaction may need to be triggered. + // REQUIRES: lock is held + bool RecordReadSample(Slice key); + + // Reference count management (so Versions do not disappear out from + // under live iterators) + void Ref(); + void Unref(); + + void GetOverlappingInputs( + int level, + const InternalKey* begin, // nullptr means before all keys + const InternalKey* end, // nullptr means after all keys + std::vector* inputs); + + // Returns true iff some file in the specified level overlaps + // some part of [*smallest_user_key,*largest_user_key]. + // smallest_user_key==nullptr represents a key smaller than all the DB's keys. + // largest_user_key==nullptr represents a key largest than all the DB's keys. + bool OverlapInLevel(int level, const Slice* smallest_user_key, + const Slice* largest_user_key); + + // Return the level at which we should place a new memtable compaction + // result that covers the range [smallest_user_key,largest_user_key]. + int PickLevelForMemTableOutput(const Slice& smallest_user_key, + const Slice& largest_user_key); + + int NumFiles(int level) const { return files_[level].size(); } + + // Return a human readable string that describes this version's contents. + std::string DebugString() const; + + private: + friend class Compaction; + friend class VersionSet; + + class LevelFileNumIterator; + + explicit Version(VersionSet* vset) + : vset_(vset), + next_(this), + prev_(this), + refs_(0), + file_to_compact_(nullptr), + file_to_compact_level_(-1), + compaction_score_(-1), + compaction_level_(-1) {} + + Version(const Version&) = delete; + Version& operator=(const Version&) = delete; + + ~Version(); + + Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const; + + // Call func(arg, level, f) for every file that overlaps user_key in + // order from newest to oldest. If an invocation of func returns + // false, makes no more calls. + // + // REQUIRES: user portion of internal_key == user_key. + void ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, + bool (*func)(void*, int, FileMetaData*)); + + VersionSet* vset_; // VersionSet to which this Version belongs + Version* next_; // Next version in linked list + Version* prev_; // Previous version in linked list + int refs_; // Number of live refs to this version + + // List of files per level + std::vector files_[config::kNumLevels]; + + // Next file to compact based on seek stats. + FileMetaData* file_to_compact_; + int file_to_compact_level_; + + // Level that should be compacted next and its compaction score. + // Score < 1 means compaction is not strictly needed. These fields + // are initialized by Finalize(). + double compaction_score_; + int compaction_level_; +}; + +class VersionSet { + public: + VersionSet(const std::string& dbname, const Options* options, + TableCache* table_cache, const InternalKeyComparator*); + VersionSet(const VersionSet&) = delete; + VersionSet& operator=(const VersionSet&) = delete; + + ~VersionSet(); + + // Apply *edit to the current version to form a new descriptor that + // is both saved to persistent state and installed as the new + // current version. Will release *mu while actually writing to the file. + // REQUIRES: *mu is held on entry. + // REQUIRES: no other thread concurrently calls LogAndApply() + Status LogAndApply(VersionEdit* edit, port::Mutex* mu) + EXCLUSIVE_LOCKS_REQUIRED(mu); + + // Recover the last saved descriptor from persistent storage. + Status Recover(bool* save_manifest); + + // Return the current version. + Version* current() const { return current_; } + + // Return the current manifest file number + uint64_t ManifestFileNumber() const { return manifest_file_number_; } + + // Allocate and return a new file number + uint64_t NewFileNumber() { return next_file_number_++; } + + // Arrange to reuse "file_number" unless a newer file number has + // already been allocated. + // REQUIRES: "file_number" was returned by a call to NewFileNumber(). + void ReuseFileNumber(uint64_t file_number) { + if (next_file_number_ == file_number + 1) { + next_file_number_ = file_number; + } + } + + // Return the number of Table files at the specified level. + int NumLevelFiles(int level) const; + + // Return the combined file size of all files at the specified level. + int64_t NumLevelBytes(int level) const; + + // Return the last sequence number. + uint64_t LastSequence() const { return last_sequence_; } + + // Set the last sequence number to s. + void SetLastSequence(uint64_t s) { + assert(s >= last_sequence_); + last_sequence_ = s; + } + + // Mark the specified file number as used. + void MarkFileNumberUsed(uint64_t number); + + // Return the current log file number. + uint64_t LogNumber() const { return log_number_; } + + // Return the log file number for the log file that is currently + // being compacted, or zero if there is no such log file. + uint64_t PrevLogNumber() const { return prev_log_number_; } + + // Pick level and inputs for a new compaction. + // Returns nullptr if there is no compaction to be done. + // Otherwise returns a pointer to a heap-allocated object that + // describes the compaction. Caller should delete the result. + Compaction* PickCompaction(); + + // Return a compaction object for compacting the range [begin,end] in + // the specified level. Returns nullptr if there is nothing in that + // level that overlaps the specified range. Caller should delete + // the result. + Compaction* CompactRange(int level, const InternalKey* begin, + const InternalKey* end); + + // Return the maximum overlapping data (in bytes) at next level for any + // file at a level >= 1. + int64_t MaxNextLevelOverlappingBytes(); + + // Create an iterator that reads over the compaction inputs for "*c". + // The caller should delete the iterator when no longer needed. + Iterator* MakeInputIterator(Compaction* c); + + // Returns true iff some level needs a compaction. + bool NeedsCompaction() const { + Version* v = current_; + return (v->compaction_score_ >= 1) || (v->file_to_compact_ != nullptr); + } + + // Add all files listed in any live version to *live. + // May also mutate some internal state. + void AddLiveFiles(std::set* live); + + // Return the approximate offset in the database of the data for + // "key" as of version "v". + uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key); + + // Return a human-readable short (single-line) summary of the number + // of files per level. Uses *scratch as backing store. + struct LevelSummaryStorage { + char buffer[100]; + }; + const char* LevelSummary(LevelSummaryStorage* scratch) const; + + private: + class Builder; + + friend class Compaction; + friend class Version; + + bool ReuseManifest(const std::string& dscname, const std::string& dscbase); + + void Finalize(Version* v); + + void GetRange(const std::vector& inputs, InternalKey* smallest, + InternalKey* largest); + + void GetRange2(const std::vector& inputs1, + const std::vector& inputs2, + InternalKey* smallest, InternalKey* largest); + + void SetupOtherInputs(Compaction* c); + + // Save current contents to *log + Status WriteSnapshot(log::Writer* log); + + void AppendVersion(Version* v); + + Env* const env_; + const std::string dbname_; + const Options* const options_; + TableCache* const table_cache_; + const InternalKeyComparator icmp_; + uint64_t next_file_number_; + uint64_t manifest_file_number_; + uint64_t last_sequence_; + uint64_t log_number_; + uint64_t prev_log_number_; // 0 or backing store for memtable being compacted + + // Opened lazily + WritableFile* descriptor_file_; + log::Writer* descriptor_log_; + Version dummy_versions_; // Head of circular doubly-linked list of versions. + Version* current_; // == dummy_versions_.prev_ + + // Per-level key at which the next compaction at that level should start. + // Either an empty string, or a valid InternalKey. + std::string compact_pointer_[config::kNumLevels]; +}; + +// A Compaction encapsulates information about a compaction. +class Compaction { + public: + ~Compaction(); + + // Return the level that is being compacted. Inputs from "level" + // and "level+1" will be merged to produce a set of "level+1" files. + int level() const { return level_; } + + // Return the object that holds the edits to the descriptor done + // by this compaction. + VersionEdit* edit() { return &edit_; } + + // "which" must be either 0 or 1 + int num_input_files(int which) const { return inputs_[which].size(); } + + // Return the ith input file at "level()+which" ("which" must be 0 or 1). + FileMetaData* input(int which, int i) const { return inputs_[which][i]; } + + // Maximum size of files to build during this compaction. + uint64_t MaxOutputFileSize() const { return max_output_file_size_; } + + // Is this a trivial compaction that can be implemented by just + // moving a single input file to the next level (no merging or splitting) + bool IsTrivialMove() const; + + // Add all inputs to this compaction as delete operations to *edit. + void AddInputDeletions(VersionEdit* edit); + + // Returns true if the information we have available guarantees that + // the compaction is producing data in "level+1" for which no data exists + // in levels greater than "level+1". + bool IsBaseLevelForKey(const Slice& user_key); + + // Returns true iff we should stop building the current output + // before processing "internal_key". + bool ShouldStopBefore(const Slice& internal_key); + + // Release the input version for the compaction, once the compaction + // is successful. + void ReleaseInputs(); + + private: + friend class Version; + friend class VersionSet; + + Compaction(const Options* options, int level); + + int level_; + uint64_t max_output_file_size_; + Version* input_version_; + VersionEdit edit_; + + // Each compaction reads inputs from "level_" and "level_+1" + std::vector inputs_[2]; // The two sets of inputs + + // State used to check for number of overlapping grandparent files + // (parent == level_ + 1, grandparent == level_ + 2) + std::vector grandparents_; + size_t grandparent_index_; // Index in grandparent_starts_ + bool seen_key_; // Some output key has been seen + int64_t overlapped_bytes_; // Bytes of overlap between current output + // and grandparent files + + // State for implementing IsBaseLevelForKey + + // level_ptrs_ holds indices into input_version_->levels_: our state + // is that we are positioned at one of the file ranges for each + // higher level than the ones involved in this compaction (i.e. for + // all L >= level_ + 2). + size_t level_ptrs_[config::kNumLevels]; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_VERSION_SET_H_ diff --git a/leveldb/db/version_set_test.cc b/leveldb/db/version_set_test.cc new file mode 100644 index 000000000..64bb98321 --- /dev/null +++ b/leveldb/db/version_set_test.cc @@ -0,0 +1,331 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_set.h" + +#include "gtest/gtest.h" +#include "util/logging.h" +#include "util/testutil.h" + +namespace leveldb { + +class FindFileTest : public testing::Test { + public: + FindFileTest() : disjoint_sorted_files_(true) {} + + ~FindFileTest() { + for (int i = 0; i < files_.size(); i++) { + delete files_[i]; + } + } + + void Add(const char* smallest, const char* largest, + SequenceNumber smallest_seq = 100, + SequenceNumber largest_seq = 100) { + FileMetaData* f = new FileMetaData; + f->number = files_.size() + 1; + f->smallest = InternalKey(smallest, smallest_seq, kTypeValue); + f->largest = InternalKey(largest, largest_seq, kTypeValue); + files_.push_back(f); + } + + int Find(const char* key) { + InternalKey target(key, 100, kTypeValue); + InternalKeyComparator cmp(BytewiseComparator()); + return FindFile(cmp, files_, target.Encode()); + } + + bool Overlaps(const char* smallest, const char* largest) { + InternalKeyComparator cmp(BytewiseComparator()); + Slice s(smallest != nullptr ? smallest : ""); + Slice l(largest != nullptr ? largest : ""); + return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, files_, + (smallest != nullptr ? &s : nullptr), + (largest != nullptr ? &l : nullptr)); + } + + bool disjoint_sorted_files_; + + private: + std::vector files_; +}; + +TEST_F(FindFileTest, Empty) { + ASSERT_EQ(0, Find("foo")); + ASSERT_TRUE(!Overlaps("a", "z")); + ASSERT_TRUE(!Overlaps(nullptr, "z")); + ASSERT_TRUE(!Overlaps("a", nullptr)); + ASSERT_TRUE(!Overlaps(nullptr, nullptr)); +} + +TEST_F(FindFileTest, Single) { + Add("p", "q"); + ASSERT_EQ(0, Find("a")); + ASSERT_EQ(0, Find("p")); + ASSERT_EQ(0, Find("p1")); + ASSERT_EQ(0, Find("q")); + ASSERT_EQ(1, Find("q1")); + ASSERT_EQ(1, Find("z")); + + ASSERT_TRUE(!Overlaps("a", "b")); + ASSERT_TRUE(!Overlaps("z1", "z2")); + ASSERT_TRUE(Overlaps("a", "p")); + ASSERT_TRUE(Overlaps("a", "q")); + ASSERT_TRUE(Overlaps("a", "z")); + ASSERT_TRUE(Overlaps("p", "p1")); + ASSERT_TRUE(Overlaps("p", "q")); + ASSERT_TRUE(Overlaps("p", "z")); + ASSERT_TRUE(Overlaps("p1", "p2")); + ASSERT_TRUE(Overlaps("p1", "z")); + ASSERT_TRUE(Overlaps("q", "q")); + ASSERT_TRUE(Overlaps("q", "q1")); + + ASSERT_TRUE(!Overlaps(nullptr, "j")); + ASSERT_TRUE(!Overlaps("r", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, "p")); + ASSERT_TRUE(Overlaps(nullptr, "p1")); + ASSERT_TRUE(Overlaps("q", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, nullptr)); +} + +TEST_F(FindFileTest, Multiple) { + Add("150", "200"); + Add("200", "250"); + Add("300", "350"); + Add("400", "450"); + ASSERT_EQ(0, Find("100")); + ASSERT_EQ(0, Find("150")); + ASSERT_EQ(0, Find("151")); + ASSERT_EQ(0, Find("199")); + ASSERT_EQ(0, Find("200")); + ASSERT_EQ(1, Find("201")); + ASSERT_EQ(1, Find("249")); + ASSERT_EQ(1, Find("250")); + ASSERT_EQ(2, Find("251")); + ASSERT_EQ(2, Find("299")); + ASSERT_EQ(2, Find("300")); + ASSERT_EQ(2, Find("349")); + ASSERT_EQ(2, Find("350")); + ASSERT_EQ(3, Find("351")); + ASSERT_EQ(3, Find("400")); + ASSERT_EQ(3, Find("450")); + ASSERT_EQ(4, Find("451")); + + ASSERT_TRUE(!Overlaps("100", "149")); + ASSERT_TRUE(!Overlaps("251", "299")); + ASSERT_TRUE(!Overlaps("451", "500")); + ASSERT_TRUE(!Overlaps("351", "399")); + + ASSERT_TRUE(Overlaps("100", "150")); + ASSERT_TRUE(Overlaps("100", "200")); + ASSERT_TRUE(Overlaps("100", "300")); + ASSERT_TRUE(Overlaps("100", "400")); + ASSERT_TRUE(Overlaps("100", "500")); + ASSERT_TRUE(Overlaps("375", "400")); + ASSERT_TRUE(Overlaps("450", "450")); + ASSERT_TRUE(Overlaps("450", "500")); +} + +TEST_F(FindFileTest, MultipleNullBoundaries) { + Add("150", "200"); + Add("200", "250"); + Add("300", "350"); + Add("400", "450"); + ASSERT_TRUE(!Overlaps(nullptr, "149")); + ASSERT_TRUE(!Overlaps("451", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, nullptr)); + ASSERT_TRUE(Overlaps(nullptr, "150")); + ASSERT_TRUE(Overlaps(nullptr, "199")); + ASSERT_TRUE(Overlaps(nullptr, "200")); + ASSERT_TRUE(Overlaps(nullptr, "201")); + ASSERT_TRUE(Overlaps(nullptr, "400")); + ASSERT_TRUE(Overlaps(nullptr, "800")); + ASSERT_TRUE(Overlaps("100", nullptr)); + ASSERT_TRUE(Overlaps("200", nullptr)); + ASSERT_TRUE(Overlaps("449", nullptr)); + ASSERT_TRUE(Overlaps("450", nullptr)); +} + +TEST_F(FindFileTest, OverlapSequenceChecks) { + Add("200", "200", 5000, 3000); + ASSERT_TRUE(!Overlaps("199", "199")); + ASSERT_TRUE(!Overlaps("201", "300")); + ASSERT_TRUE(Overlaps("200", "200")); + ASSERT_TRUE(Overlaps("190", "200")); + ASSERT_TRUE(Overlaps("200", "210")); +} + +TEST_F(FindFileTest, OverlappingFiles) { + Add("150", "600"); + Add("400", "500"); + disjoint_sorted_files_ = false; + ASSERT_TRUE(!Overlaps("100", "149")); + ASSERT_TRUE(!Overlaps("601", "700")); + ASSERT_TRUE(Overlaps("100", "150")); + ASSERT_TRUE(Overlaps("100", "200")); + ASSERT_TRUE(Overlaps("100", "300")); + ASSERT_TRUE(Overlaps("100", "400")); + ASSERT_TRUE(Overlaps("100", "500")); + ASSERT_TRUE(Overlaps("375", "400")); + ASSERT_TRUE(Overlaps("450", "450")); + ASSERT_TRUE(Overlaps("450", "500")); + ASSERT_TRUE(Overlaps("450", "700")); + ASSERT_TRUE(Overlaps("600", "700")); +} + +void AddBoundaryInputs(const InternalKeyComparator& icmp, + const std::vector& level_files, + std::vector* compaction_files); + +class AddBoundaryInputsTest : public testing::Test { + public: + std::vector level_files_; + std::vector compaction_files_; + std::vector all_files_; + InternalKeyComparator icmp_; + + AddBoundaryInputsTest() : icmp_(BytewiseComparator()) {} + + ~AddBoundaryInputsTest() { + for (size_t i = 0; i < all_files_.size(); ++i) { + delete all_files_[i]; + } + all_files_.clear(); + } + + FileMetaData* CreateFileMetaData(uint64_t number, InternalKey smallest, + InternalKey largest) { + FileMetaData* f = new FileMetaData(); + f->number = number; + f->smallest = smallest; + f->largest = largest; + all_files_.push_back(f); + return f; + } +}; + +TEST_F(AddBoundaryInputsTest, TestEmptyFileSets) { + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_TRUE(compaction_files_.empty()); + ASSERT_TRUE(level_files_.empty()); +} + +TEST_F(AddBoundaryInputsTest, TestEmptyLevelFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(1, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_TRUE(level_files_.empty()); +} + +TEST_F(AddBoundaryInputsTest, TestEmptyCompactionFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + level_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_TRUE(compaction_files_.empty()); + ASSERT_EQ(1, level_files_.size()); + ASSERT_EQ(f1, level_files_[0]); +} + +TEST_F(AddBoundaryInputsTest, TestNoBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("200", 2, kTypeValue), + InternalKey(InternalKey("200", 1, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("300", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + + level_files_.push_back(f3); + level_files_.push_back(f2); + level_files_.push_back(f1); + compaction_files_.push_back(f2); + compaction_files_.push_back(f3); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(2, compaction_files_.size()); +} + +TEST_F(AddBoundaryInputsTest, TestOneBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 3, kTypeValue), + InternalKey(InternalKey("100", 2, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 1, kTypeValue), + InternalKey(InternalKey("200", 3, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("300", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + + level_files_.push_back(f3); + level_files_.push_back(f2); + level_files_.push_back(f1); + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(2, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f2, compaction_files_[1]); +} + +TEST_F(AddBoundaryInputsTest, TestTwoBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("100", 4, kTypeValue), + InternalKey(InternalKey("100", 3, kTypeValue))); + + level_files_.push_back(f2); + level_files_.push_back(f3); + level_files_.push_back(f1); + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(3, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f3, compaction_files_[1]); + ASSERT_EQ(f2, compaction_files_[2]); +} + +TEST_F(AddBoundaryInputsTest, TestDisjoinFilePointers) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + FileMetaData* f4 = + CreateFileMetaData(1, InternalKey("100", 4, kTypeValue), + InternalKey(InternalKey("100", 3, kTypeValue))); + + level_files_.push_back(f2); + level_files_.push_back(f3); + level_files_.push_back(f4); + + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(3, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f4, compaction_files_[1]); + ASSERT_EQ(f3, compaction_files_[2]); +} + +} // namespace leveldb diff --git a/leveldb/db/write_batch.cc b/leveldb/db/write_batch.cc new file mode 100644 index 000000000..b54313c35 --- /dev/null +++ b/leveldb/db/write_batch.cc @@ -0,0 +1,150 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// WriteBatch::rep_ := +// sequence: fixed64 +// count: fixed32 +// data: record[count] +// record := +// kTypeValue varstring varstring | +// kTypeDeletion varstring +// varstring := +// len: varint32 +// data: uint8[len] + +#include "leveldb/write_batch.h" + +#include "db/dbformat.h" +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "leveldb/db.h" +#include "util/coding.h" + +namespace leveldb { + +// WriteBatch header has an 8-byte sequence number followed by a 4-byte count. +static const size_t kHeader = 12; + +WriteBatch::WriteBatch() { Clear(); } + +WriteBatch::~WriteBatch() = default; + +WriteBatch::Handler::~Handler() = default; + +void WriteBatch::Clear() { + rep_.clear(); + rep_.resize(kHeader); +} + +size_t WriteBatch::ApproximateSize() const { return rep_.size(); } + +Status WriteBatch::Iterate(Handler* handler) const { + Slice input(rep_); + if (input.size() < kHeader) { + return Status::Corruption("malformed WriteBatch (too small)"); + } + + input.remove_prefix(kHeader); + Slice key, value; + int found = 0; + while (!input.empty()) { + found++; + char tag = input[0]; + input.remove_prefix(1); + switch (tag) { + case kTypeValue: + if (GetLengthPrefixedSlice(&input, &key) && + GetLengthPrefixedSlice(&input, &value)) { + handler->Put(key, value); + } else { + return Status::Corruption("bad WriteBatch Put"); + } + break; + case kTypeDeletion: + if (GetLengthPrefixedSlice(&input, &key)) { + handler->Delete(key); + } else { + return Status::Corruption("bad WriteBatch Delete"); + } + break; + default: + return Status::Corruption("unknown WriteBatch tag"); + } + } + if (found != WriteBatchInternal::Count(this)) { + return Status::Corruption("WriteBatch has wrong count"); + } else { + return Status::OK(); + } +} + +int WriteBatchInternal::Count(const WriteBatch* b) { + return DecodeFixed32(b->rep_.data() + 8); +} + +void WriteBatchInternal::SetCount(WriteBatch* b, int n) { + EncodeFixed32(&b->rep_[8], n); +} + +SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) { + return SequenceNumber(DecodeFixed64(b->rep_.data())); +} + +void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) { + EncodeFixed64(&b->rep_[0], seq); +} + +void WriteBatch::Put(const Slice& key, const Slice& value) { + WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); + rep_.push_back(static_cast(kTypeValue)); + PutLengthPrefixedSlice(&rep_, key); + PutLengthPrefixedSlice(&rep_, value); +} + +void WriteBatch::Delete(const Slice& key) { + WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); + rep_.push_back(static_cast(kTypeDeletion)); + PutLengthPrefixedSlice(&rep_, key); +} + +void WriteBatch::Append(const WriteBatch& source) { + WriteBatchInternal::Append(this, &source); +} + +namespace { +class MemTableInserter : public WriteBatch::Handler { + public: + SequenceNumber sequence_; + MemTable* mem_; + + void Put(const Slice& key, const Slice& value) override { + mem_->Add(sequence_, kTypeValue, key, value); + sequence_++; + } + void Delete(const Slice& key) override { + mem_->Add(sequence_, kTypeDeletion, key, Slice()); + sequence_++; + } +}; +} // namespace + +Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* memtable) { + MemTableInserter inserter; + inserter.sequence_ = WriteBatchInternal::Sequence(b); + inserter.mem_ = memtable; + return b->Iterate(&inserter); +} + +void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) { + assert(contents.size() >= kHeader); + b->rep_.assign(contents.data(), contents.size()); +} + +void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) { + SetCount(dst, Count(dst) + Count(src)); + assert(src->rep_.size() >= kHeader); + dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader); +} + +} // namespace leveldb diff --git a/leveldb/db/write_batch_internal.h b/leveldb/db/write_batch_internal.h new file mode 100644 index 000000000..fce86e3f1 --- /dev/null +++ b/leveldb/db/write_batch_internal.h @@ -0,0 +1,45 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ +#define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ + +#include "db/dbformat.h" +#include "leveldb/write_batch.h" + +namespace leveldb { + +class MemTable; + +// WriteBatchInternal provides static methods for manipulating a +// WriteBatch that we don't want in the public WriteBatch interface. +class WriteBatchInternal { + public: + // Return the number of entries in the batch. + static int Count(const WriteBatch* batch); + + // Set the count for the number of entries in the batch. + static void SetCount(WriteBatch* batch, int n); + + // Return the sequence number for the start of this batch. + static SequenceNumber Sequence(const WriteBatch* batch); + + // Store the specified number as the sequence number for the start of + // this batch. + static void SetSequence(WriteBatch* batch, SequenceNumber seq); + + static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); } + + static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); } + + static void SetContents(WriteBatch* batch, const Slice& contents); + + static Status InsertInto(const WriteBatch* batch, MemTable* memtable); + + static void Append(WriteBatch* dst, const WriteBatch* src); +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ diff --git a/leveldb/db/write_batch_test.cc b/leveldb/db/write_batch_test.cc new file mode 100644 index 000000000..1a3ea8fa3 --- /dev/null +++ b/leveldb/db/write_batch_test.cc @@ -0,0 +1,132 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "gtest/gtest.h" +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "util/logging.h" + +namespace leveldb { + +static std::string PrintContents(WriteBatch* b) { + InternalKeyComparator cmp(BytewiseComparator()); + MemTable* mem = new MemTable(cmp); + mem->Ref(); + std::string state; + Status s = WriteBatchInternal::InsertInto(b, mem); + int count = 0; + Iterator* iter = mem->NewIterator(); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey ikey; + EXPECT_TRUE(ParseInternalKey(iter->key(), &ikey)); + switch (ikey.type) { + case kTypeValue: + state.append("Put("); + state.append(ikey.user_key.ToString()); + state.append(", "); + state.append(iter->value().ToString()); + state.append(")"); + count++; + break; + case kTypeDeletion: + state.append("Delete("); + state.append(ikey.user_key.ToString()); + state.append(")"); + count++; + break; + } + state.append("@"); + state.append(NumberToString(ikey.sequence)); + } + delete iter; + if (!s.ok()) { + state.append("ParseError()"); + } else if (count != WriteBatchInternal::Count(b)) { + state.append("CountMismatch()"); + } + mem->Unref(); + return state; +} + +TEST(WriteBatchTest, Empty) { + WriteBatch batch; + ASSERT_EQ("", PrintContents(&batch)); + ASSERT_EQ(0, WriteBatchInternal::Count(&batch)); +} + +TEST(WriteBatchTest, Multiple) { + WriteBatch batch; + batch.Put(Slice("foo"), Slice("bar")); + batch.Delete(Slice("box")); + batch.Put(Slice("baz"), Slice("boo")); + WriteBatchInternal::SetSequence(&batch, 100); + ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); + ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); + ASSERT_EQ( + "Put(baz, boo)@102" + "Delete(box)@101" + "Put(foo, bar)@100", + PrintContents(&batch)); +} + +TEST(WriteBatchTest, Corruption) { + WriteBatch batch; + batch.Put(Slice("foo"), Slice("bar")); + batch.Delete(Slice("box")); + WriteBatchInternal::SetSequence(&batch, 200); + Slice contents = WriteBatchInternal::Contents(&batch); + WriteBatchInternal::SetContents(&batch, + Slice(contents.data(), contents.size() - 1)); + ASSERT_EQ( + "Put(foo, bar)@200" + "ParseError()", + PrintContents(&batch)); +} + +TEST(WriteBatchTest, Append) { + WriteBatch b1, b2; + WriteBatchInternal::SetSequence(&b1, 200); + WriteBatchInternal::SetSequence(&b2, 300); + b1.Append(b2); + ASSERT_EQ("", PrintContents(&b1)); + b2.Put("a", "va"); + b1.Append(b2); + ASSERT_EQ("Put(a, va)@200", PrintContents(&b1)); + b2.Clear(); + b2.Put("b", "vb"); + b1.Append(b2); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@201", + PrintContents(&b1)); + b2.Delete("foo"); + b1.Append(b2); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@202" + "Put(b, vb)@201" + "Delete(foo)@203", + PrintContents(&b1)); +} + +TEST(WriteBatchTest, ApproximateSize) { + WriteBatch batch; + size_t empty_size = batch.ApproximateSize(); + + batch.Put(Slice("foo"), Slice("bar")); + size_t one_key_size = batch.ApproximateSize(); + ASSERT_LT(empty_size, one_key_size); + + batch.Put(Slice("baz"), Slice("boo")); + size_t two_keys_size = batch.ApproximateSize(); + ASSERT_LT(one_key_size, two_keys_size); + + batch.Delete(Slice("box")); + size_t post_delete_size = batch.ApproximateSize(); + ASSERT_LT(two_keys_size, post_delete_size); +} + +} // namespace leveldb diff --git a/leveldb/doc/benchmark.html b/leveldb/doc/benchmark.html new file mode 100644 index 000000000..80e6c2a4a --- /dev/null +++ b/leveldb/doc/benchmark.html @@ -0,0 +1,459 @@ + + + +LevelDB Benchmarks + + + + +

LevelDB Benchmarks

+

Google, July 2011

+
+ +

In order to test LevelDB's performance, we benchmark it against other well-established database implementations. We compare LevelDB (revision 39) against SQLite3 (version 3.7.6.3) and Kyoto Cabinet's (version 1.2.67) TreeDB (a B+Tree based key-value store). We would like to acknowledge Scott Hess and Mikio Hirabayashi for their suggestions and contributions to the SQLite3 and Kyoto Cabinet benchmarks, respectively.

+ +

Benchmarks were all performed on a six-core Intel(R) Xeon(R) CPU X5650 @ 2.67GHz, with 12288 KB of total L3 cache and 12 GB of DDR3 RAM at 1333 MHz. (Note that LevelDB uses at most two CPUs since the benchmarks are single threaded: one to run the benchmark, and one for background compactions.) We ran the benchmarks on two machines (with identical processors), one with an Ext3 file system and one with an Ext4 file system. The machine with the Ext3 file system has a SATA Hitachi HDS721050CLA362 hard drive. The machine with the Ext4 file system has a SATA Samsung HD502HJ hard drive. Both hard drives spin at 7200 RPM and have hard drive write-caching enabled (using `hdparm -W 1 [device]`). The numbers reported below are the median of three measurements.

+ +

Benchmark Source Code

+

We wrote benchmark tools for SQLite and Kyoto TreeDB based on LevelDB's db_bench. The code for each of the benchmarks resides here:

+ + +

Custom Build Specifications

+
    +
  • LevelDB: LevelDB was compiled with the tcmalloc library and the Snappy compression library (revision 33). Assertions were disabled.
  • +
  • TreeDB: TreeDB was compiled using the LZO compression library (version 2.03). Furthermore, we enabled the TSMALL and TLINEAR options when opening the database in order to reduce the footprint of each record.
  • +
  • SQLite: We tuned SQLite's performance, by setting its locking mode to exclusive. We also enabled SQLite's write-ahead logging.
  • +
+ +

1. Baseline Performance

+

This section gives the baseline performance of all the +databases. Following sections show how performance changes as various +parameters are varied. For the baseline:

+
    +
  • Each database is allowed 4 MB of cache memory.
  • +
  • Databases are opened in asynchronous write mode. + (LevelDB's sync option, TreeDB's OAUTOSYNC option, and + SQLite3's synchronous options are all turned off). I.e., + every write is pushed to the operating system, but the + benchmark does not wait for the write to reach the disk.
  • +
  • Keys are 16 bytes each.
  • +
  • Value are 100 bytes each (with enough redundancy so that + a simple compressor shrinks them to 50% of their original + size).
  • +
  • Sequential reads/writes traverse the key space in increasing order.
  • +
  • Random reads/writes traverse the key space in random order.
  • +
+ +

A. Sequential Reads

+ + + + + + + + + + +
LevelDB4,030,000 ops/sec
 
Kyoto TreeDB1,010,000 ops/sec
 
SQLite3383,000 ops/sec
 
+

B. Random Reads

+ + + + + + + + + + +
LevelDB129,000 ops/sec
 
Kyoto TreeDB151,000 ops/sec
 
SQLite3134,000 ops/sec
 
+

C. Sequential Writes

+ + + + + + + + + + +
LevelDB779,000 ops/sec
 
Kyoto TreeDB342,000 ops/sec
 
SQLite348,600 ops/sec
 
+

D. Random Writes

+ + + + + + + + + + +
LevelDB164,000 ops/sec
 
Kyoto TreeDB88,500 ops/sec
 
SQLite39,860 ops/sec
 
+ +

LevelDB outperforms both SQLite3 and TreeDB in sequential and random write operations and sequential read operations. Kyoto Cabinet has the fastest random read operations.

+ +

2. Write Performance under Different Configurations

+

A. Large Values

+

For this benchmark, we start with an empty database, and write 100,000 byte values (~50% compressible). To keep the benchmark running time reasonable, we stop after writing 1000 values.

+

Sequential Writes

+ + + + + + + + + + +
LevelDB1,100 ops/sec
 
Kyoto TreeDB1,000 ops/sec
 
SQLite31,600 ops/sec
 
+

Random Writes

+ + + + + + + + + + +
LevelDB480 ops/sec
 
Kyoto TreeDB1,100 ops/sec
 
SQLite31,600 ops/sec
 
+

LevelDB doesn't perform as well with large values of 100,000 bytes each. This is because LevelDB writes keys and values at least twice: first time to the transaction log, and second time (during a compaction) to a sorted file. +With larger values, LevelDB's per-operation efficiency is swamped by the +cost of extra copies of large values.

+

B. Batch Writes

+

A batch write is a set of writes that are applied atomically to the underlying database. A single batch of N writes may be significantly faster than N individual writes. The following benchmark writes one thousand batches where each batch contains one thousand 100-byte values. TreeDB does not support batch writes and is omitted from this benchmark.

+

Sequential Writes

+ + + + + + + + + +
LevelDB840,000 entries/sec
 
(1.08x baseline)
SQLite3124,000 entries/sec
 
(2.55x baseline)
+

Random Writes

+ + + + + + + + + +
LevelDB221,000 entries/sec
 
(1.35x baseline)
SQLite322,000 entries/sec
 
(2.23x baseline)
+ +

Because of the way LevelDB persistent storage is organized, batches of +random writes are not much slower (only a factor of 4x) than batches +of sequential writes.

+ +

C. Synchronous Writes

+

In the following benchmark, we enable the synchronous writing modes +of all of the databases. Since this change significantly slows down the +benchmark, we stop after 10,000 writes. For synchronous write tests, we've +disabled hard drive write-caching (using `hdparm -W 0 [device]`).

+
    +
  • For LevelDB, we set WriteOptions.sync = true.
  • +
  • In TreeDB, we enabled TreeDB's OAUTOSYNC option.
  • +
  • For SQLite3, we set "PRAGMA synchronous = FULL".
  • +
+

Sequential Writes

+ + + + + + + + + + + + + +
LevelDB100 ops/sec
 
(0.003x baseline)
Kyoto TreeDB7 ops/sec
 
(0.0004x baseline)
SQLite388 ops/sec
 
(0.002x baseline)
+

Random Writes

+ + + + + + + + + + + + + +
LevelDB100 ops/sec
 
(0.015x baseline)
Kyoto TreeDB8 ops/sec
 
(0.001x baseline)
SQLite388 ops/sec
 
(0.009x baseline)
+ +

Also see the ext4 performance numbers below +since synchronous writes behave significantly differently +on ext3 and ext4.

+ +

D. Turning Compression Off

+ +

In the baseline measurements, LevelDB and TreeDB were using +light-weight compression +(Snappy for LevelDB, +and LZO for +TreeDB). SQLite3, by default does not use compression. The +experiments below show what happens when compression is disabled in +all of the databases (the SQLite3 numbers are just a copy of +its baseline measurements):

+ +

Sequential Writes

+ + + + + + + + + + + + + +
LevelDB594,000 ops/sec
 
(0.76x baseline)
Kyoto TreeDB485,000 ops/sec
 
(1.42x baseline)
SQLite348,600 ops/sec
 
(1.00x baseline)
+

Random Writes

+ + + + + + + + + + + + + +
LevelDB135,000 ops/sec
 
(0.82x baseline)
Kyoto TreeDB159,000 ops/sec
 
(1.80x baseline)
SQLite39,860 ops/sec
 
(1.00x baseline)
+ +

LevelDB's write performance is better with compression than without +since compression decreases the amount of data that has to be written +to disk. Therefore LevelDB users can leave compression enabled in +most scenarios without having worry about a tradeoff between space +usage and performance. TreeDB's performance on the other hand is +better without compression than with compression. Presumably this is +because TreeDB's compression library (LZO) is more expensive than +LevelDB's compression library (Snappy).

+ +

E. Using More Memory

+

We increased the overall cache size for each database to 128 MB. For LevelDB, we partitioned 128 MB into a 120 MB write buffer and 8 MB of cache (up from 2 MB of write buffer and 2 MB of cache). For SQLite3, we kept the page size at 1024 bytes, but increased the number of pages to 131,072 (up from 4096). For TreeDB, we also kept the page size at 1024 bytes, but increased the cache size to 128 MB (up from 4 MB).

+

Sequential Writes

+ + + + + + + + + + + + + +
LevelDB812,000 ops/sec
 
(1.04x baseline)
Kyoto TreeDB321,000 ops/sec
 
(0.94x baseline)
SQLite348,500 ops/sec
 
(1.00x baseline)
+

Random Writes

+ + + + + + + + + + + + + +
LevelDB355,000 ops/sec
 
(2.16x baseline)
Kyoto TreeDB284,000 ops/sec
 
(3.21x baseline)
SQLite39,670 ops/sec
 
(0.98x baseline)
+ +

SQLite's performance does not change substantially when compared to +the baseline, but the random write performance for both LevelDB and +TreeDB increases significantly. LevelDB's performance improves +because a larger write buffer reduces the need to merge sorted files +(since it creates a smaller number of larger sorted files). TreeDB's +performance goes up because the entire database is available in memory +for fast in-place updates.

+ +

3. Read Performance under Different Configurations

+

A. Larger Caches

+

We increased the overall memory usage to 128 MB for each database. +For LevelDB, we allocated 8 MB to LevelDB's write buffer and 120 MB +to LevelDB's cache. The other databases don't differentiate between a +write buffer and a cache, so we simply set their cache size to 128 +MB.

+

Sequential Reads

+ + + + + + + + + + + + + +
LevelDB5,210,000 ops/sec
 
(1.29x baseline)
Kyoto TreeDB1,070,000 ops/sec
 
(1.06x baseline)
SQLite3609,000 ops/sec
 
(1.59x baseline)
+ +

Random Reads

+ + + + + + + + + + + + + +
LevelDB190,000 ops/sec
 
(1.47x baseline)
Kyoto TreeDB463,000 ops/sec
 
(3.07x baseline)
SQLite3186,000 ops/sec
 
(1.39x baseline)
+ +

As expected, the read performance of all of the databases increases +when the caches are enlarged. In particular, TreeDB seems to make +very effective use of a cache that is large enough to hold the entire +database.

+ +

B. No Compression Reads

+

For this benchmark, we populated a database with 1 million entries consisting of 16 byte keys and 100 byte values. We compiled LevelDB and Kyoto Cabinet without compression support, so results that are read out from the database are already uncompressed. We've listed the SQLite3 baseline read performance as a point of comparison.

+

Sequential Reads

+ + + + + + + + + + + + + +
LevelDB4,880,000 ops/sec
 
(1.21x baseline)
Kyoto TreeDB1,230,000 ops/sec
 
(3.60x baseline)
SQLite3383,000 ops/sec
 
(1.00x baseline)
+

Random Reads

+ + + + + + + + + + + + + +
LevelDB149,000 ops/sec
 
(1.16x baseline)
Kyoto TreeDB175,000 ops/sec
 
(1.16x baseline)
SQLite3134,000 ops/sec
 
(1.00x baseline)
+ +

Performance of both LevelDB and TreeDB improves a small amount when +compression is disabled. Note however that under different workloads, +performance may very well be better with compression if it allows more +of the working set to fit in memory.

+ +

Note about Ext4 Filesystems

+

The preceding numbers are for an ext3 file system. Synchronous writes are much slower under ext4 (LevelDB drops to ~31 writes / second and TreeDB drops to ~5 writes / second; SQLite3's synchronous writes do not noticeably drop) due to ext4's different handling of fsync / msync calls. Even LevelDB's asynchronous write performance drops somewhat since it spreads its storage across multiple files and issues fsync calls when switching to a new file.

+ +

Acknowledgements

+

Jeff Dean and Sanjay Ghemawat wrote LevelDB. Kevin Tseng wrote and compiled these benchmarks. Mikio Hirabayashi, Scott Hess, and Gabor Cselle provided help and advice.

+ + diff --git a/leveldb/doc/imgs/lsmtree.png b/leveldb/doc/imgs/lsmtree.png new file mode 100644 index 0000000000000000000000000000000000000000..62c444466661b74205f5ba4870563c151d14839b GIT binary patch literal 131549 zcmV*_Kq|k9P)N00E!~1^@s7Y7C{b00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D|D{PpK~#8N?41XE zWmUERr!c)U%nTi=kNT(rirD@jASgbmf`C-1QUw&Lk0K8Q1OX95KoAB{L{LDKrbrQx z4#QBU&txV&H}xi2|NY%3zhvjkxi|MFH_4sh{_aGe=dRaN%NE3eqA zufCd<{`t>;cKGN!3d)S8rl!Qq$7$%$q1N2oYz++!*4oNNNL83*7{@l596Ehztn6Y%}QXf_lGBG;|gTf3b z4s=O~*|TRSWt8C^JY%QpS@g1PXP-4L9ug72(4VKTOPgoZ`$t z%AR}fxsG35pi1_H9+^>FTkAR9n(i)cs-GcvcGg*E+2MyDZbuw(L?Rt}=%MzjU;V1X zM}+wc=D7cDmbR{zfp||l?KB@&CL+8y-+Z&fM;C?;>z*yD3i_j`mZwK%Ao~05x1U{n z(G@mk)O25`>+9>6D_wyIVG37B^;yr8kvWm_mqeMde0Xhbu)4+GdiPz&_T>0_WJYUi zv$dp~91~itskOzLTiw0;4YvznFAM}=TvKz(+S{6K;i3hWNu_3G2Bm`dl5wcTYv!m# zh(MyJQsRo##G(4iJMQs=Uvx|qM+EL6q|sMRA72!1Cg>gR37>F7NHHTB4XJt`c581W zLm4B1+=mG#P@s04NXh8DdGq}5FatF;HU5h2Aja|xzY7;G^ad8K$Ku6{ZPu(=_Rc%+ z`06En@ci@7`|FcWKIy}K_19nT*cjh9|}$-es4 zuXg;(jJ%j}-cQbT|6R#utaZuoEmBLPjU2g3$CH6YwrUn4XJ#}sr0m2Kzwbn`!k+%e z(;ZKHMtOO;PfUAzehe%GkVYhv>O&eJ(vt}}GlR@1Lo`jH{s=);dho#q?Y{f&>nsuK zDdsiL@4owP`{<*O{PX?<>3i+9*X*H(9rh{`eACRIH4m|KcA2G@|6xB#lq86n;p;jzJWo2cj(vKq}Gg?!X_Uc@h z;dIIxoPAMVU1jU7zrJe`K6a)2iA|qA-J084yD8GR2=24bKK@F^)l3Rv1dQ2fr=4v4 z`0;kmIp=t{VjTCPd;ufgd+)uDUtOTpqS_=OBQunMR+q5GmS(&AclY>2LbT~Lp^@Fu zg|f0TpK1KUm|y?;*Zw)b*LRErNJ(fz^aT1DBSwty?vX}%yt$yPY_XL)$$sOlcl_@q z%a(enP?Mg79CQXui5R02%LZ(Tx88co-hTUSpJ{=T;<;>yTmd5~$+}dHSKh-%j<9qN z>1b(6SyN-3x0>lXOq#H|{o#?nT7{GFFkK*swnr$vN;`%k&w2$4) z5Hk)t>@a@?WE@@87bXbkCUisy!bB=FRrid{Xm#b@(wer`_GW8pX|Ss5TG#7yCFWX? z=GF#lbW#J-j>`&s{`~n~HxMZaVeCTS9qxGd!3Q67GUM52pY8OIR8YP8$_PMfrD2AV zBS+e5tF7km8H`S9azWN=0!UbxxG34loTbjwslw2qGZmH$M%SPI^d~WKi=`QFd%HwM0G)b!E}CSVs?+r5aLY91l-W0B_W|W$}iU>@r{G(BLCF!&_Cc^ z7=9s;kr`x&$mT9^;_ZH;c%+3%5Q?+O*&h*-8TI+kf8Jk}=+KRxF}kgLu5l|C;P1#I zkL>u>9r9ZUWMl@BU~GLdZ?03OjqZ1gEA5o8f3Ls$w!Qh@J9g(ix7&n?W1Ri)hW8V< zK=Qb~pN z8L{U%QWHKG)LUwpBDPB@_i;6`CF@EZc_HXr1yGe{I2Hg{)fXP3tr0mvxgeB5!z+0jQI z-I+KN%rM}IC!T1hpMJXi>}Nmo&jCu7X~g4?Ki*GGsyY!LVI6bKG5#I%s>1#E-|wdv zd5%(+v|!HDIuSP#7gnKMP*4Th7bruz=7N0O=7OSOMlqnoi~#-046IP}#U`6<;teDe zAHTQWdTTG@G~IXJd8h5V>#n|j^K9?E_wF1g4ef@3KuIzJ6r35f20dASq2i#^XDpKrQKnBl^;CZql9o7SLq=v0{_C!{_Z*law^zzS93up%i*eus+=PrAx20?+7nGhp-G?|#=Vy6B>gH_?DFqg7aj zSe5F>zzi%;tVT?8y5|DprC6Hyx$LsbIy22e6ykvB;tNb*9d*=E-p{ee9-DD~Bohj* zK$YMm@JA-~DSgRQ>C>Naiqs($Fy z$1_5tnR)8-R$Fc5Cv`5o@IpV+e(-k=wDIG|+Is7)XK%gnrmZ%0HTz)pY+HNnwY(oB zON@Yq4IAd4x46GrS5V_8jkhm-=}W%XJYw`Ht8@Bj#F$}r%yGxsPG8u@uDs%En>KZv zeKh-H?=Ox{W@y@tOmLMZuBl!aEz(+3U8cCP<4Bg-FnNf zY-#y2pFlDvAPscoVaf|Hyx_a?prN78nwq1Adw4YmN&-~Z~4Zqn{Kv&U2@qEeW}uSxaXc|#c&)v3!B$a!6mtmkh$dKFNrdv zdw72Q#jl2^PgFvGFX-=`FoBbjc{buYkTMtmkp6G!AdtDfaXmWikCet&e zC@V9ldyH^(S4h#=Bn+XRQ;j65T;n6-{0{52Bqj-Zu4=^ZB%L6{MPF`1KctlzeTD*> z8XA3`GR2>j8I(Mv!~p_gs1A@(CXR_bloA-E1mQ;S@J9yF_!`ILO8BhyXH0|+A_+$S zAboVkv?TKHfB(A=>#u+Ps}GlzKKNz52Foh3U)YmI8reIGad*>AH~H`wA7%Xr9Z^>O zBq1lAAw(L#&C-50K6g+$aScC^pkQ*Tt~oOUQDZ^dt+(Ei7c(e17Qs<(b1}pF14v`M zN+ls@X0W`GdiUGA@3i`+<@c^;0#$}mLDvj4G0C&Jx`cc7#G-q$RhL9&ple$i>N;CvxOvkrZ}xP<4L8`m z_uk#{raLsEE-2F|$xwr9c1?PuKlBOKZX7UT?AWm#zm`Wv$*5{bTSNIrKF~?4P4&;t zJO6w;=Gdcr{r}ege9MnNh6rc6fF^NhA;Q*%i6#J(nKWXn)s!hyyk4TkL8NmL>ugwO zopn0yxYaZ+WDzzNV&qSu;C+ySZ!mAO`beCe`a(g z7>^No8{Y-Q;Cv>aEb%L&3n=m=Ch$(r@f%O5=0;#hJ{abzH{ofI+V_q;(%yalgWl;3 zYWDFY1#wgDh#O0kLZIN(!X=?cW(ZwO z$qec`8I=hxDFGw`|6JpXNHT>hcUl$+xFT^}i6dSW;>Z+s5_)6?ds}5skV!~}5c||P zD1bf-p!*y`ElM~LMB=nvlgxxA6|V0JSL$#Lr72{x6m21pkr}MJXmBmb4^BVL4*bpm zcITb9`^2u|6cBsVUVio8w&uDsthSDxXq0HJ)DyJKA--3Y31|>L6Z9;;LqI)Vwo$11 zflR9ldxMP3AW|YB-8|BIJ z0-lGf^RNdPmE!rm_ukuConQ?6+*p3iyF!kUh znZvLAlP=jW*)lBm`5>zTr6s{)d+FsD?A6y^^7S4IbpHHki)I|G!wjLOk?Ioy-2zB( zSehZ?C%|gYnhYsH3t{8oWC}MMvPuT)tY{(r^{;=~pC0+6jTkw^)>w0Od+(!JzPU|` zPSpgdka=XoOMQ<>pHvt{vP`x zj98#KWcm{>4}EvD68`3}q6KQ89<-GN^z|`tZa?++u_Qh6zKk$PL7qs)?t3kVP^S z5LZ_%Iwc|MVM6h;##F|_WbYXJ$`};~xWzi;8N?wYN+>1EKrM^kSgi@r`eMqg!O^ea@TruGKeJ*?aH5 zXH_-jR#REw_(38XeTPj6VuG4OM&fl>3tPEz>`SO00C8#TgLh)sqJ|nOqzfPS5d+!M zrG>Rn*o2aD;#PX3qX?7p#n}1u@!U`RS%jI;$&4L$+_B>>Lf2Z0DDX@E_aZ;|J8Aql zOE)+AiQHusOKttN*0IXE8e4nab^M+%68W{Sea&Z*OlhSr#CWBeKb8#avdb=h?<&30 zQ&0VY&nOFFS)y?KX{Xq@@e}R(pWon`XjcFZd=o{CfTAO59K%`~Y}IM2+Ko5=!cQ!- zyx}|FJ=Fet+ih&GuYb$FcjU?TokLx?F3k_z^Wbs?B6Koi=bd-%Cc+w!@dLmujecJm zgF1X6f-%vk8{Td=-Ep(ErW##hR`VylO`JHpWysNI~~{GXnouJ>-$)3W3rjw$qaTRv-bw$ zz@JFSu1^?9W}1O?GDky|ihO5JLmZi6>10L;C4?vmMhS^1TEWonHQE5}appc>qLJpx zMz64U)LYON(lNkDz-r267hMuzXwc3Ua@xF*C|be5l;qsIctQ7uBvkRqo8Ev4V1z-G zROtDT#wCgkpBllyTE${kzlpVH`c6_VP#1!+7WK6XK^#dg;}#9c45ZFyM>|0nmYIky zPwvKX?kH-(U|c7upz|eZ4Emi?-70AHCAszlI>^jys89<{`ued~CsI4C3)YF`Bdo2MrmgB5*fx8bW~|LzgOMS9>{PlA>U z`Yae>BDBP)y1LT$08ooCnZ$Z#yP8OEgH=GeNIz{TLwZ>Mtovk1Qzl|5WD>Dl8~>y& znR;w*D!P_G)50N`^5H&Zf zuF4k$&=}Eq>CE+Ty!hjfJ?)+}+VRK#z!oh0#AlxO46@U8q@8CFoV_pMRB2z(AI8^; z26zyyRc>(nr_!~#`(|6PXpuEKfnkU^bG29=eH4r~C+%q`sa?kgKsEFk7~->^&2rSz*yI z4pb@bx#Blm$blf?kE?)NEzl6uEa|EyjkqAR6#i)1aKkfxGf+j?$_sIG&okO~?h#&T zHrx;vX;K`xAzbxx2pd~Q`BrnPxRpjd6T~;J5b^RG5_pC`(gC;+ue`@E&A8IXJLOAh z%muv`3?fuv6R8*>EkK5J52|Xrmu4#2ZeNbn~dF1gjn zF24cKl&(f{H%g4d6rg<6Sw0}+%KWHNBYkfqpBl2FLqd*#6(ZjS z7Yu}g!a4u^AGrk9x*o`UzX##ZkN>&DMPb1~S;PlJIJ*{$oOYHgg7Rqrnv+Ex(h_~m zd%*>R19i|67^O>>Ms53Ir`BugYi!|?%;N}s0mmsxV~AE}pp_Zkr&4f+#;8%FI**J2 z916m77Pmwzc=AWD1%u5u^gc+WC~D85t)B5dH(NlXcioN2OrQp=z4qE2zq*7M$M6Js z3q})V+16qc#zzL;cH3_4pR?HU-Usiu#Y@ZlbNbGwpLvFF_s=-CUp+8UwY}K zo$k0M#R(L4p|Eh#e4DcBB>UifR?*^Ip!Gu&B6F#K7>cSfBDN_{CQZoV>Lee zLGe$)(k_#hYyy`n$e;4a73f2;@C2bkfa1d2H@IfFrNL4)uF|wbnyQUc`?Y3>OFxe8GT+-53x6<{HO08klF z*_&>w@8*bRmJ@-C-ClsN30fs3&0rxZ?7`XDE1U6N%><|#2 z?3exoO~SXo{cXP~i*ylhNW3RMOBR=To|ly^@fx<@n-Z)7{{^HbhG$`Pi&fP)b=4X% z<)6?0(;j*BkG{un>7^I>t`qd?YcIO)(`fI8G5p~Z#@Hc89Oi2h8HxkLFq`qY-+udb z>t(RU06}0(*|e}1cR?!p%K!qNlCZ2bh_mk?QWY>b!0B7ASndaWbU)iK7KrdCSs$F zHu7sV_uY42zwI-wG3dVX$w3Yiz%72*=EVwe4j^IYQwZXwC&QL6Gy;COLd1g#W-cV( z+ScE|(B~X7CIQhN)`j|z!u9mvKxda2FDO14!4#5Zun!+GXfPlq8 znmGtCtX&i$U&l=X;gLs;eaH{ZSNIj(I2c0-n>hSxz9KHj0_wlm(Cemzk;_;9^O`U4 zk3RX>53U^cy+a%$(suiAZu9+kdL!5exLJMOwe9(rU-ajL*EOW<_kXu! zfJr;Ps)|3{N`nHL2n@h*WfpE|5{W<57!uHJw%NuveYh(o^n>vUgIyuIKUehr&#@<< z?uD5N6b#Ix{iaC~Q5#Y7yblW*@Ehh+FbES(4Of0s*qMM3%Dw!^ZivSLl&&xyG)Xui z7ec~@JMw@XL_WAjXp`a?Z1jU*6b3LK0ie)O#dOcuXY#}oPjuFG9g(SDyc0i$L+UtsD0;GvgbU~c6l`UOp9!MHYgr6jBfuN}Zzzew_DC8{|Bo4+m zIU9rQ&_jT^nzg+1&fnN8ue|Kr=F?6)&DRnR_rS_$QVfJDH5;_n0>+8i#6ypR39=Bl zQ7<7}qapB|%{^fuaDTu72l((Zf%J!ZOpi_S*P0tYL|K$NnntRLK0vs+9F$%fo&4Lr?5D)2){|S9Ox`GfkMGn z$pnu5!HuAFWeOi~zEUUz_>m10_a`mFseReP>?`sX3{))3<6$rf=Ze&Gk2axiPVH=M zMNLMVU_eb{oy}P|*N=FjZ5aKOU}eI2=bhJa*9GFEhBFQjubB*?Gis^dxDA=d{?D)g zcu&&=;)TFHVG$Y3kUFyg62yraqU3!37-{@ zKs=!9+^YUT!EFpO8)pDuCY9$Q^RoeW6aqDtp=^lmC=6jaTax<654DxC2P}Qua%Cox z<|Qd^3WRsq6NE2+6eb0nbHQ_>N2d;#jt+~P->EKHEXcqO9LJcISJBoVSi0Eu`ud)>c*$J*(wBGfQ}9hq2nLME-WLzQ<`4(ZIM_@Y0*1kSzN(+( z9^gj9@&XJ_Dqvyv-FNS1&roO@4b2o!m@a?-X97H9fQz&V_~pdHOyTpMF_cX23xQq= z1|O+y^DD(`>Z+}&Ipq=?smBjLn&pf%)=9Sd=gloCfBX-}<}+x-&NPnhXZ>fZ4<$1g zN@mbVYBdBkGn`NZ+_U&K6T{GM2yp0!8a;jb^iE9{`lT65ntDgL7(|Q5!*6OcUk?q_ zjC+h-!b<|1q(YDmEh>tGX#f@%CBafEY3{e=Pvh8Wiu3C3^XYJdcbYe@YEDns8ZM6n=IFtFbF zPDeNyL@mGy2cd-feP!(_*EBd4*w*Nqh^P=?k}I5c+;*+za&O(QXx$~>1&Ru*i?n;q z&#mN8kff33fT?h5u;w6Wf|vx0($aG{IU{3GGctmqT8vMG~;lur|HFo-Qk7iNt zq_Bw6;Pad@tvw@AFm~N_*N(g8fuR-(46&d4qtm<*dh{`e+EGUx;U}bOYAdY1vDPYT z%Wdi6Mb_9Bsn@dl8e38}$By{k5mr%ML(rHN=HM5!4!{jI!Y3bo zWHoKgR$Wr!?2Y@9gGGTczLWZ4pHX^sZm2&JYjN>i9l(x*Kl zvR62@XvyMe;H}lw(ov&_`ePiM8(VDDn31lTbUbcy_FOu7K~N?vCm3i8Ht1vmi3%6# zVn{yr*ds2XRW^Cb1ka!sUU=RmIOPg6Y76O!j2l1JQmGohB7Ds?r~BPemt1_YRW4a- z?|ksS)wj8xisJ}h|Jm}tx3Yh~_NrgEK^u&wqOe%^5o!R^qE9G=LJoem5HDbgn zonf$GdGpOT^KP&;X1HF}k}{0V=#v;5Z?d8132Qc~@&9@KKi1>~cFNSLHg^0(JMpAb zTt)c68;ml*0R_rh55hyD=tF=d}~4bZJjDMkn#&7QYA%+Jps46v`qeJPVz`k|**^ z-osVy@Cc2CK%pHdKoLy~pVcBYtVw}toOW$)aPhl1qi|e<+tOs^WsavV?q;W17cDFE zJOO-ko|6zs56}1w^qVkwMw7)MHTXxMbH#FkA%l%VKwVLbs4-lrwGo4h3#64Mjx;L0 zsD2o#{0s(&7r`YxqL7cp$V(y7#wg*_jgnB5upCN4QNnU42}KFZp=gByBN3wvV5G@7 z%a;^}u7v>M_7)hQ5QZ?Lse6>`3WP_Uy*!`poUYdQ)k6f7SoHWal`Bo%OP2%)&-l1n=0L}<$ac1u13T#WKE zQd&r%(4rnSZD@}8Kq0@ANOnp0xv9S6QzH}?Uwp9-7e+E<&p~DRFcaZn^puU)nqw&h z>2sb8>$BL0vpX18Cq5aa#G8rw!zTIE2nDN|sEzCm1>7SPxiTMjXbO%okWdstI51i_ z*@TmVqs6}Q8Qz{ix>yWUNPKF9;;O5z>MUH0ScCwerg_#=6N0)1EX7lS7D6bPxB_ZE zPCfNhe+@y}G7$nnz6yy?jiE{Nk*!OrLI$8TPTun#1Thy+zkwhS|}z=n2=b=`4U1A_CAt8b3!vh&CF)J zv37$c_!MY=3k4d0bg{sRGe1aQCPCHzyvdun^xwm>2H z&vIo@$#h9241E;}R(k6k&rG18`MMHm%LYbTHA31~8>7!cK|8HMA*~~^vc-i^gkYU$rg}*@ z6clSj3BR|*9ux+u$Y)q_;@~|?A6r}XA^v2b37`DilYWUHWgtT1q^5DapP`tM?P-55XP4)<(DoY zyB~yt@5QmsleMP^#MRec;{}4c>SFfpazUSk;?z@r;A+nM{(klcAKTpdZ`r$V&$pJg zYOAYFdsWHmX*DNk4;BQ&ob0tXH!#!;Lq6^J{U3gB4?Xmt|CUR;HTmrW%rK`}B<*;e zsX0tSAB2K-IPE1fA`BdUH(GPL&F|r8aw@O6+xw_ZXH<}TH79)mZ7J&b(~g-l?{k9j zfV)e(qG6D(*>6=yHP&0IsnIuw10oc(aV(jSC^}sRqD^jQ8NK`S8;yha0xC3@=A$Qs zLeV?5ymEjZ%FLN;T6W!O*HcN=*4T{c(_N)(wkB5r5r}>l3Kc-K-=j+)pzg*EZD@2% z5lsf~r6Djjd4|@A7ihKu4tp=y6WAv_BOJmE-vbO=!jq-~j9CzOCg5HKX!dXqk$(9l zF7Cth#>3S+(jo9H?k8RA~=C=hmlg@puouYx5^GDOBgNbm$U|G*3_*~32*U0S@N-P?r8AQ10yhZ}4f z!s3|*w#Wm|G<%r@{1F~;NB~K@_TK{W1Q51i%7+O8o;po=RWtjoSTSjD7aAY2}lO3-hStG8#!v2 z)upQKqKhxE+WP*hwHS5t=5f?PG@wSeth{1bw|0AFhw_THfck=h3&nb$S;tx0PkLdv z^wLXg`i#~50M_?UKHe5A%KT~U-h$?e`HzSh7*r9h?7+Ybn+Y@ohBGz=&(V_bgeEj! zJEOohMVmk6hGP>s(S|`S#tD{HL}LXF&3$X@lE5x9Ae%z8GK0b%IdY_*SY*Q#-07tFAj^8RQ*a(X6tvOI_4l(MzE~HFc~| z>L{3q_^of8b_oTy8h*uHu8Cw22~C4=NCT*F8SGIU)DoV-koZU`eqq?2d+ymelDMLR z{K7lNMi`wXzYMB{BehfzP79CGkN4W5r-CLwQ>IMm+?tpx@@ziRR$o_RLq@M^ z^Or2L)yEF;iEGB{cW;+gRoakMTsPEdjrV5Fa@Ul7{K<#*(#tQnTD-ecX`gt@_>n(9 z+T)@R<`S}jvou4daL=iDAS~R_O?)#i!bxpH+p(PhgAL>) zJ$C$f|Iz)l6PB{F#jY^kv(?v_?rQQEJo6ACj`52`DH&*rY>XzO;d>ac&p!M3`?x zYky`nt^!8pduEUa7CvX2@l8TrLP7SrvKZA87|5cUM%QFS+98@`l4|zv9iugI!@;36 z9h^y-ja|%s;unOBrSO;{#`>oA3WUupB!v_fsQhlW*=GKEAq$G8^BdpzhTn>y&r9-* z@X0$Brm$#t0=UR#fAFqfZikRIr&}%M;;e1x+S8)=vE{JMu=i8}azUQW$MT5I<#sBz zBT@RB-Fn}py{IInDxZ{rV ze$P1L48QI|%;~F8;Gg^v7Q{V0fU$h(*$F4TFpO9VZQ-KmgExgilueg zTMRR<{n73d9uzUW-G&YCVkd{!6C3A|bBb_2E1xpM^^|HV?7VZ%vQ&MI4RuC5YhgjP z698hUudlWW*UQ^!m%XgHz1~)vJ}sw87@`0|fr`&X3iD@*n4T+CKuLb`4J%))wbq*c z8m&idwgx_i<9cKq#1408F8hvld;5*o?FT2HXid!xHhSzBd-k8t*rSg<>f&F+rcRz_ ze3a*pk38ZG{77?fo;VewUo}N{=5kFU3WnyRqX}{8;UoLbPIvEe1O8|e(F&oe2G}Pc zZaY9)ekb?%&31$k@6j$?yYkh20ZtOaxY_dx<2gZJ`R^~h{NlfCzyJA`PoF<~JMHW> zM>Jc|SwB3}Dyph&%((GZTVG|XufBTc(t=!(w@{!4qXHn~5&{tg5zSJDBb)3TV~0~) zd!)i4Y>;dTRntKs%a8D_Z+*+FM3^A(UIhii;$dsqTO1gQKw*V(F)E0#WCEw|A$aP! z3iE1@lXE-UFW1XuurNekLP56aezvqV+edTwEK@L|=)YajW*>a; zo=Z@J)zq>_vWuZ?QfX+6dJBy%jIxeTK=@ur0zM>sj<>B0 zi8Dl=1l%(F3h;|?5-!Z#7VyJZ@lp)JNcC3y&ndAFug!G$RN2*6UE>U#ajq7Ycg)f`&GPa4 zw#t~{UN8TtpZ%9r zRxPyKZoShMEc(zkSbrnW)H!oMc75k6n=)gH??#OnI?PY5O`S5uhmpi+&a2D@hQY&( zqT10EN%jQbSvdCrm`%(i6ab+(=%9nVATuh6X36Aspn%cHa6HW$c2_oF>r8dM51N~P z5(?06>Go!4Q#iY#HSNe*yzbiMGyypE48G3WWHr^Ut+kYE8r&~u8T&n@u6e;Zv5R~ zf=YJox#!x}TW{^{DhhtJ)mHO@%V@DSbAugr*un1+K~v~iF33|T&=swId0j{to)EL6 z4ULulgJ-Sn^?nOHL*S7+XN}0cd*%zGwaaRbGoI$*{PWMZZMNCQTlpJqxS?&d(MGn# z7F+nKW0(`Jk08@%kG=NV%Qo0x1Me0;3~w`UfzgO&2m!NhzWHW9y^O7ZAFdFipbUUz zO5b1sgO`m&d>)IY7r{WVIp34ve+)}B2>it;Wx4>tQRT=5c?(4xFiR9ck<&m#zueo7 zJI7!i<^XI21cS*a<}_Gg#Hww6Gwu+#9k^!*9c_Rc+~S87NbJsv_f`aii%J8B`(hJdV&^oH0r6 zf!xWh?pG9uAB9B0g+hreL}IZGRH)Pn2xJRCtQEXh7@6EtyYxN;H7{xT9`1=lLL~5v z8XFHQ9<$eIwk9&lI zRaRPh7YF{BK*N9yX{)Wa>TuB=gv)q^1VHJ+{-B3KcyaI^I|qN*FrxxEciwyz<6XZ3L^)mQW$FIU>HnfN*DJ_%a{2c2opxk?g0&p3mAbtfv^Q? z>glJS?&Fk35lC~={00RKLg50+BLWylXq$v#aSsuHsI5o~0!Ca(0<+(K`*}abguH}; zOi{Q@0D+^9KH7$N%;_^JGiuCeTfBIYU3=}d-s)uuIt;(Xb&F*IxHC$GisTwu_k*F`k&34~`uJZT za~W@OOt{$AU2h%#9F@v^38SFb-Si9JpFZTU?^tE+Qky(=f(x(7#*Q293Wu5$^;iH$ z79}xJKh0Jf#9UswTFe_6W_+dOCu624^P|WxVJtKj!kZZ2F#@{CiA3ddF zYeIU}e5^8Rq%Ete^q+Wswe{oYx$V({56a~RNJ~o7ixedN~K7RZ-Z&Sd?u2b_E&2&8n zKdQ<1xV1Ap`c^p8yPRGdXBx(&Y7hktqam6J;$Q$L+{Zz>Sft7oK}1MGp&=|yfC&f% zaVDv6;$fVF<|PD$7=|m5MhKxGeu1$0@V~+d^N3Bv)bg8z}Qh;jqWqyVUC6i!IgUf#zGj9N>rqZXo3@XI~p z1et()RJ+_m+!J@E!a|b}rs9G4os=%_d5#d1b~Fl28CT+xpJGCRg`$5D&U0|3@G~(} zmOt9`BNb2#uq>hhY&v@j|>0K{$jLj$g<> zdp+aL6u(&mngqfo@9?(2&?}+1%o$58$E$Gajiq5E5XQ2?`}ybp;R>$BFT7<$5NPm* zQOHnevVkrq5cA{*=5x>BRu~S#!Wk4WK8azfc_GjcJg30oARY|EI1oAsW*EN&R|l#B z-sASL!w&OtB*g>Y;$3C(z{siM2|*a_e#>Sj^%Z#w1sNSNdXzPAD!)rKXHZZGOP4OO zgAU%`4*t&mPB0?%i;(afY_P#+?2bEsk2Oqp^M||&4yY;r4tS(jSM>U!7VgPHC|Jum8Z?z@MavRd^Nhq)_LMFrfU3S@} zv&JStJH-Xn`lP}FeB36kb@DkPfh*y&&NHs1LLi(W5DHdMC8djS_)Jw&(^e?tEflnw z<0nk;hm*Yh?mM>m=9_z=Or2#^8*Q|sic4`QuEE_)af-VYr$EpG z#fxk3Qk>#)@_uW5>zuz?WHFh{O!l+)bzhG(mU+-W-P?_GDyYf#6(wfslS2`>B?{N4 zGKuSk!=5P1s0E_P^B!BkNP%-)qs7AO#}9wZCbwX-;1^dNj%gr#@jGJr;lDGlF(h!7 zJl+kBh^<)DF=AYy_C%Oa+>VXUTR~#M-ZBmqBcr1r*uu`8sd2VTN`j6vowQaY^Yd7Z zd@Ed7%{#DxX&ZK9al$h)YYzLt^9Nvl6wc*U6Nm~qs2pA)kw62YqGxN$IwC2c$NUG9 zhZsqm%YvjQ!}+=tbK|9w;GqV9ukc;vm%y;&Nv#5pan=GzhcFVg8a@dzQ^S1%K!K7n z;BV9BQN+bGlx##08cf=(Z~``0XTVmUk+E19Cca2!51%Zh?m5^ZK77K=5eyH$X5;;QBbu z2TB2yJ!WhHzG&G<_}6N$fO9S{k<|uEXoGhjKwULA{pQk@O~o%eK>ALu%Xhx zGBR9JY8TT>jaj2$T?90YOc9m{33cJrzCfKTRZ8y>8=+VRaqN6_jY2LZ6{dIAQ*=0s zP?PtfCcF4Z33jp8YWKD2N8bpv=feowCRaB*HlxoaEE=KoMZWkJDL^jqTdY38l16&R zKu%bGrZ8PAB32_)jkXR1^vS^dg`Xumi-E*O`CG#iZhtO`nNwOug+Yz(5;n*V#06=m zq3w`Q&1h(mJ4C^$nblttqZ5FS|F{zN!X6KblS6Pr@`_*6;RsPD2j87aMZ()Sib4QG z_e0#5xWR#X@4?O(i0>2V=i^w^MwB`pekE}4FziPLvcFhTB80&na+s)2^SN%d{Jh`{TDq={J>KteWl&F~BwIbmaQKl^& z^Kwv;5|S`f+cI_bVpj~&crCTYIG*UKe!t>&phW^cJ3qp^hapl9ez>>vcNk~6XE58z zlBid*Btq}SMaS26?f!ek=;UnQ1~Y#=RQ%xd1~zxN6nttqUE(-V08tClr1fT9d33p&LW%HXUYvX*7b~z;FjArRo`t z6iVX^YxcaFYIUKqua718Y=@?)O0yVH;73KiF@zD2GAqrAMgqCPfYTWZv9o(5W_4v}J!x85y2_ z$;Vr0x05qZhztw|F?6Dc+VUVy95xfdoshw*6jM~Bu?((7J5+n7HbPqff=#IyW_)Ci zl3EeS3pl&D_}5ErJSv&W5{5Q@&%s%@vMca*Jlxi>^2|Sd?&KC#)3j?}_x}d#KrPu* z`OXC2?rpvP(GBkE{T-{@*#OT1~P)fsvr9Qb|{VK(jA_a2qR?;}G- z06hE-sXQ(T)8?#d(}g?@=2Y{CwW0Ue7DR^YMZf=?mG<;w==s?YeEmV^T5RtcEdx7U zwP{4kro?y16t(AHV~mU&-ICtFuXr}xNR_v5F>KkqWA7G8Dq=1U?c}J7kWo54yg8oF z7F!BCu!0W7!nj=o2t{SEYd_6rzEh z0%3#V{0x%6=!?quTo}$P(Q)QrVTaI!z%dCIIs0N&2Z$N){zBGYk>qZf{iTXgdLXhS zrD!I=nEnFAlSrP+a(`FP5{JP<&DdPiqWDPVBSx+!X?6qXh6-F$?lC;-0fgTLg{_cQ z3KEc<2CtV4yj5u6J;q70{+63HkCsBO(pAmKopiSa?4(dFxCswU7Nw4GJZg|4LZ%CX z3%9TVO1GdU3z6dft`P%e$nBczmOBZ&U~8M$1gzO*WMXT)YSK{B;v=tmMG-;i{z_D6 zQ#w8SlZKfPFP(w#IZ2-0^TnDza_VzG&kxcp+eWmx3-{ukzT!pqEu?)~f7wS~^_BL4 zcG-u}$qN|xA<;`O*#|>!D5#GmM75T(DBjXiDp}JW`+wlROU0|FCFY6~9Lv(@zIq>8 z7oMo5S-0P!^<6h}agx;!D^YJPt@+QY+^ZN3&lfcyrki^b6H4vJp@Gd6Pe~uZf(xsi zPBD^&hZgNT!2r#m{cd%@r;b=0Th@?q9IypEbR;I7`t*>&EO<$8$sqqEt%Z8gyTw@v zVZ#91+f$sk>yM@|fV##e0hwIX%U%dwh%Q1SugUmKz-v{%T=D+-cNtbmGw?#@7+Fm% z)#9>yAY@qh59{}Ck80=Be3`C1(9oUzNEjofvakZROV`SyI_jUl9lL#pUklL1PNMBP zhJ;p)PU?2R5JrJtEcV@Mr2LEI`GO8kYAd=+&yy`Fp6^dg>WYh!+%~w;Ef%9&KGjXv zp0q5@&=w`gFjScc5^3KTgQUf+>_qhfm&}p0_vVCb?L=i$%#ntflnH@x+;{+ajZUhTdY$x%D zU}nIynXcr5)JECqWQ+P|wvdiH*=M5qZZV&$4;0PgJlWfVxqa5L*X>c}B#*PBExn+NZ~42smtAMrl;mlr z^KiM4I%M+I=~@jY{+Vv$C5T$>XgV>_HO&dK9rSj7`6qPws-?EyHXwSkI^?b$s4rNB zqzHlCfBcNaO42MDWDAnpvaQLl4;r_VEg_%Peg@_WeyR>wJyLYph+I(tu5FH(F^SG z(b1515olrB9L_=)G=96BzoChSs-hCOteT>;EQ>BA~iqcR_r1q&!-NiDsa$;Jm~d!ApgHwSNDA2k=(^{g6tFI~H@>rraw9M(rh%5HCK-kOXm z)P!+`1cq8HMT%!oDZca6T0YC4egrGCzzr_0oj2Bf>ONge^7{qDn?8g*a`_Bj=te!V zC+s|*!n^?sBMI+Lb5SL!o`wovt!vb&Tw5j%jlNA>bXFAX4DQ(<(#n1-%Rg5&vn=T>4QcjX z;dwyh&Dgi(d+~Ym-H>=EfZLta)49{c_{86fLG15+l9#@A+P}IVREXE+#yvtp{(WCE z=p}$$5>zZ#***8a-(X(bDtPTqB$HY#@V#=qUF<$z&hsnuZ8v`&*RNas&b>j=F=?RA zGBycYw;VdLL`vr-3=B)OB=AUJe~m{yHKVZ*-Bvb^$NMSfs70SxC~Iok`&%@`!LGCu zu9lWa;?V7LzNO4BX*gO(hrm|cV%WtH@G`)f7Z~9qjZ4aD?fVQfP+g~6r`pQ!tcnkI z($y|w{eG{yCqiE*`@N(}E0Ogu+{SzF+b;~>Ynl$)pi-<-auZQgVVwk)WYbVPdjvE& zMVQT|?InL@CCO7XuZrBygS>W%ES_~#R`Pr$3I6OV@#3CaR8(hCCqNBJ*ep)NCdSSL zK2Vz+j}2maIh0d;plh_#?xe@)yF46JQSi2f~W)kum zp6AaZKzAgXi~OGGt^OGoD!Exoh@gn`i~O`~4!zh=@!X<3$9I;mM3_20F&lP+Y68(1 zGg825%`C-Wd=H?KVFn0@CQ@9=p$vN=52Quh8(O{vpKhz_(^SbM*(!lM1f;SJR6yyVia-U z-u@Td)ktvk$oL8RmBFn07nlyEHOoLAijIcH_aGA_+-UO}wne0Y7_l?KY!}ig&=2k@ zOgO|kur3` zP|}12z^>kyOnMnEASJ^+G}d_jDPj7iIc#Dj%^AiWtEbY`OAJF2lVZ0Qc8qXey)ilE zOCCj(`2O9@R)Y*AF_cD*R;rwvXs2{|1vv{fTVvs}{nCfD@km~t`r(AM<}m^-VoJGG z9S4+n-~_-%MhGnV5Qai^G#!a{=RvGqP|F!!DjN zg5i3OFAy71{oIe0{rKW@NmtuBMW0P%(ed6k2pUb`GHaJJhhdhwUf_%*{?)jIUy)OP z1~;XP68XMbvD>ANUl&9nc1q9&sVcj1`tyktpRW#Ix4I4}%Ro0TgC9;7qF zh@W7NB!ha`!8ThcP)9Tx38f?MBgG;3g(|F?#?oa)X6;E5@I{)d{&&SR(Xn78?OA`5 zbU4MT6s9Dd6f}c;qXWVFBh1m-4E)5eE@e+^K0>=dj!z1tBM^n}S!NIU@!uQ69DV`| z{?i0;kD#;HoS{UNau_@f+z~2SNrEQi<54S@_gvG}O0GS5hTr?<;0WW!xu8Z{@(30w z8*z+>!>i9NQ`o@wrJ(pC_wC?p^)3LpFZrk~2C6V3LJ09nJCF$&ZBm(!%(Y^0q@Ej1 zy=LX|iF9j(b~Am86vxBE_-X+`09L6P{JH%~Z{h)z+e>EZm7!RIKz1~_LqR$4^1+-7 zA9lYp>L}&0mzaHHCeoofKf=IIVFThKp{&s7lxY}OFj4rTYu2=InQ|V!-^WMjg!==I zxW&Pn_4z#Uij@J`)UcnjwdxC3)CI*S>L8iW2>q|?LDpAjb6xiXL5gr{lL7VQJ#2`m z$-7CUoQEx#c~hgp@LvCZ>jywp00=UZj*TDF3B1o>8b7J7>8{Yt{p`h>l>_dgs}F&P z(!x`(&_EsOV>c73C6iSVaNZ3F{;cbSYwP2Y`=B)?wU!+{sMo!cU1w3{4NbvX}sHm57xhkh*#k$6~fG4{Z=!XNa*n87K>KZ zT5fq82Ws4-**LaA)>5xObJU9$3?7i3 z{epqn66a96n*;O_Nf5fmD^}S~Z5QzrCX@JrfDJ?0K}NSqJI9X;wGa!~M~eIqsjyr% zi7J^1Rx|4BJ^Ac&Z#lX;5PfE)K>6^lrry3iH~5m`ux7mAWpuaA!mN%&I;a?vi9TVb z{l`B}ZrmL$vq`n7POBCo$lBUUxbT>)D_T(p=wm6?J_7B`9u{Nv9*&V=N`_FX(etJj z>0%sUkJRS+emCS8L_BrnaA@HkGg7oNr|w?+eQNb~mbuWNy7=ck#HtH+)K?oCY*x6i zV_Z7ZQ0jgB^VYH|>cbrT_sallJ!=Rex9>$*^rzJX_qM67n(nNYl_}9DlbWiA=D0uh z>Rp{>Ke2_~q(aF5)*7C#Waw7hONwVEXJzgGV!`tMOd1Mci3766{|JYdM$C0lLo0Nd zk-pdwQGX0NDn`ifcK({3y=Ie#r&ceXIGo^q~pa(ORkbEWlgG*au) z4v7_FuVNqCK0nYEBn?E($qwE=Nj+~)TutWOJ+ga8aJR`E+b*eLaX{}jLzc{I@=CTR zLkzB!f%AgH6zqQg+DZXlAlP?3rvni1A zH92~V!252+L2Th^NXU6BM^EX zdDp}2yiZ@iBoW_7@4}LxVClw~loAtB^Q?>(6`Y}97}$Za`&&2oMhtrs16}(}7Gf&? zPH8)Tj_V=fGnD$3FbRS>NQd>D6y4A~R5-5wVOcduTir`x0Zu-#*y&m8-|M@rPCzL_ zh%)uRh@2YVyvY7gb_~+7uP`|m)BU-~6?|ircEzd-tajd^`^cm)td@hHDCJg!;eh$&xXsHtE89(Zz?FZYwLruj!C@-BX~gb7*#2bVPl`}^Jb zNnaF)GO?CT|C5J1d}qllb<&QA7}t@6qgDdL-AcuOZQu(d=?j{CNa(tzex=6*TMk~#^yJ!4$@pY?|AY6!Lw3Mh4lWS0_-$AQ1 zwI?RSRoWmBH4;X%z3yyqb^o0+q=kdkxs;oOyVuiFTNuI*{y+K3 z`*Pa0HdUAjBy(J1?^|&mnbL<^w6g_-Y*SzE_-qB;{hPj9v7- zRE}#+IuHimf#-|ug&lzy4ZuLsqQUI3xC2l=m%6-E&JL344rYlxh1ZTW+X7Era#@z(O@j74HbhJI4fMi*^$Dr& z2*1%TvN??h2*o9^pZk?X1rpjI9?_u@*T+rjW+NjH1r-mZG~#wd%Y4Mg3>A4!A|=YP zHL8Sch0E0gAKfuWg(dIypSN*fm^=58sUo;8TwCT!6^SmzOgF)qNX0!W80onMliX6@ z2_EJrSf({1?H4o6v|k7U#I* zp$rqgB+w552Oj0vx z5L9jI^Dyjl!>h0i%YBDB1?+K$GS;@9M$;&;sShh94xMHETr3IPFfr+{xf=SuZjM z3FpPqfS`)X5w*p8J|{YM?Zx-{5(HQbdNI5hAicI_ZZuafQ!Jyk>T2}STD`YC(^Qr9 z@6~PNpXD}Rwk4!|nrcfrl35X28UBtez>xrE_bkr@OtdqzBzSp@g)J4tB>k`o-vtHVtKIL2Kt!mZN z{K@_E@&#$Uq^z#&sD`A%MdIaV{`Nm9Ek-059^E(vRHSUsB;EOwuSn&W5y4`JDom8z z@L*Ag*Lv{PX%(h6)|Vn4zTr}isIqBWY-n3`48}rtQlP0D?t~2}gWi#%Vs zD2tsuue3>2_1GNVvD0{$hjX{|2SvD(tri%T73SxMUN|gl?|m=;zk%iuh-YyN2z*-G zy1ipF0V`Ap@T!NUr05mwX0t;=XJ$Nbh4i7cn5aUIbsbW#_q$?)b|&Az1CitD9#+2) z=l3{H4h@ifvP&pwi2a(vNicx>iBjaaU6qu}5!CCks2F})4=qupQnTmd5Pf%ZebZ02 zvX}wVU(UQTi_5;(5G88s;F&~vy3m%((|err>jYCNv@lJICYxXg^k@5#1eop9nAf0~wTedm)1uSb@Vbn~17fsCRkILqTW} z7(b~-af=XHDC#)FQqbP>r3L*&o{?|?y>>*SqIB*Q9FCF1A>0+e0ZYs?k(9%&Tl9J* zGn;ql<&AVrdpT<=TZgX?S5eM;9BMErpxMLCAN1L$#yDVCf8y_#y+gKd@~g4Th1S_- z?wyI`{Hq7tCfjO_OIv}1-61dq`tzkLj^uslR%H94k<)TSU*A@Ft>}s?O+;$8iAQWh zC|klNUq+*Wyvm&nxdipaDcYhojWCE3thDK%cJG^r$fD2Un#UQ-s$xLv8x4^iyP=@N zM%3!V>t~qwaIP|jl>wjmb^ju87@J+Eha0hx$slQ}Ir!|UCG5yZx67Te_H6- zXY`%nZ1ZOJ`sXV1IuVaOwt4_}6U=BnZ07%fr2kXWP5EP*TCU)4SaWQnD$!8r4Zg6- zlRWRV$GPJ;aqJsMLC~#X1BZWvYv3+LFn3ICtgs?GCyo=hB0o{L5HVhEN@FlIT*+XR z(lcR;APn-m0Q*+vVwOUk;~Np!b;uOB71~_nH1LCjLvEa*Ds_y6#d$o-g~Mi47&X8T z9nXj9CP1ikgPVhooxQ;gC^9*ILQyNK>+tXQ^=Yia*>~3Tk;l095!mgrE{9QDUNtaY z&3@Vs>x^wlT{ZqP@0l4<`+8KH>hUy9(XdAD)L=s${G4j)_ggp1@ksRbbVjz%Dp}&z zF5t>lV;M&5?UOkFKeC9x@(-*t7+|xo*CjX={k@1MuWRh{GpUUXb$X&guY=`79u3M8 zpIc@XA3hf#$_Cz^vvN;~Qv!8%;gkU1( zmlLX#GIAMg95`n=lf>=Zq?+^kJ=8A&08aB|s)@R{eyHOxXtr*8OW1?;0(+ak{Cj;u zGE_xLW-%tG;=piqfQ^g}?+rqVU>C7t$!unT6+tHC#nCXHG4yATJzg{_L3 zG)L!)%#I(7eEABA6joTj)Y^OG1wB)FA((KQf zKO_;-vSFt5O(~Qk0kOz1o3N5GeGatb`RZwr!@Q}uES%+h#bR52k}6>@+2z=u5(i}& z(t;Oe=+fhKe~pGxb$BmTmKe{N|9V6|=#j+QbreH(liA%7u9QEg%(vdHxk@#dfS@d- zJ}Fn;LL*_#mkb3|MRrQ(aY(ACeOKcXw+q@Ds}oaK`4T4*HOJyh0?M9V3g3~vyL_yEz(Me(8dp_R)jl}Gh8w2{CGurZpSZ)7Si0G+lL z7m#pVafu6w%;Jt)mxkx{9TT3Yswi-TsknQk>b}l@ckMc63h{rx)+YDmp8}O%X(9RU zE6hLtZ!y*u1O)}V4f6g2kCJ^eU|4S{Bb#G|w%$66LrY!Tgj?tc<-&l_XU$8M9yVl_ zP{*??XwUuECI;SGP9cN4YvRvxh|5TSH z(HYizWL?QqeEOUDK~{U!Q433N*P6(w{Rg8Fpg+N^tOdyS+;7t=YxsadDP<_Xw9!d> zjw@b9b)#U2Q_>vsh3t5q>mxEaOpy=U)7oF<3j+^m4EKzmHzrOWbzXx;tu*y)3i!uXK~pq%R5(S(xh(w$c# z07;e5@By3$Ik%x~)gCeq7Bw~%)_^hs5ow1H%(h}qHDxMzADA7M?s-|(-GDl3rgd93 zCgL>IDHFLh&vu0m2<+xvGaEuiUnSJ00Od}T_l7MK{DQLSyMtx=W-hV{nBQq-qnBf5 zzx{{eUt2nN{tj6wWz}|mBtKa_x~ws@QmM(*(6e-2X~Vk>V4{J36L6y&6}_3d~0tjTGl2W)F!&~{&nd>J=% zOz_lkdft)Z_N5b4Pzqarem+XQjYv;ww^1Q^(%`gs420w7sR{Z)SM|)AmRXZ%1%0513N>pg4S{=B_V#rxNU%1&@Lk|shW!B!8Qe?i8@x90 z&)7r5Y~9pa4!ZJqp3$QHjmYhwR;!ji>0qdiJ&TEMX&}@AaM`vjD++ z7pU-Osa2|n3Hr@xJIR*%y3R+;C)i}g@RK*gplp;JbIzy6%aFCjwcr&gCkLn2_N}1N z5q=83z%S?id=zw@tOAdwhAcby_po%tcjj=z)J8>%l3~4B*U0mqB8>KeIp4o~28u>E zcwwPO9x2Pd7X2$9R!HX$?$4lKi=!(Y?B0+*+ zzY%g#7%lY7fnI3JAnr|qCY60dB1W3)dpu0y46d0(p=#_qp+s#g_-!p4N9W zdTCrsjqNhZIJ$A2JeeF12Uz%&e$Py9?f^I_7=EY`hUZK&w}PkLzCZTOX6Lyi`)tvR z2uZkzCX!>=raB^{mioA=YZ@uMS9q3yeStNIalUJTn~h}7fmGhi$ofqB(bP9IGNd}K2=Jwea@E3j!EKK-eY_pFqh4RaB8rkk+ z+x73VEPrF8wy}ogJfY-a4F|iDOqh_c;!&V7FW{k$l_&DK2gF$~1W!L*_E6hi~6)Tl8c zR0lqw!bSJ;*>Kq4-A#BJi7ZOEO+T4Ha5i`eBAP)?QH(<$If*|tJ%7nh>#@p5MUOlq zf#h?Zv(vMP`DFjOANnEv%naNAqGHqYxuD?KuXt8tJRA(HR-Jum#FMQR+^>zgvLDca z?FI3Bk)z2{R^D*@7)?*#$SLu=-gLW~pvkCNF#qb{)~cReU)Mi*vBSUDChP`Q-KRg| zH-yspX^cKi!gim*ijaE$Jx}ep3+Xg*aY)JNWb%Ud8;7*qiq_Mk%+rc2EF6U7lh-0> zlz$O!LN|(Il0yCVLiJ%gMN^FH9%_%Pl==aAaOv;d>^wRX5CIv&L^!pn* zc$$7<1UYt*IDadTAIK>#M$30yX2)n`*?w<@PI=sr!|$7JZFqowNw{L+PO-M3`RdNz zkaDCj5kAz`nYE!CY^SU9Ti2kP_fPgzj?h-aA7{tCj(m>)lWbQ-aJC0B2NZ78#RG8XP>{nZ!hf;n_1rcOQ_jf zmbX$^?T)msvCJp-6m4hRsZGWCE|XOs*-BIVcG)T)A@%{_6oxLp6e|pCi|e>O7Gd&T z8JDx)Nt9{27zoJMBtw;eOg6LH_p~5lmiHXpfzLs!(!2etIMJN2(xQA*v@9)pR}SWw z(_$)e+wQ)-hb<)^zp8Gm0gDQ|X*mQfZSyQgBLGRoth;k(aP7gnu*B~@NL z9}LyVFL6HH?{=y;Ssawt1sc{BUfO$(N4%q|4YoQK{I53EB*jX~!~b)STpe=I^{&H` zwKFSLXhFz8P4~N9ZH)4K`dm?AV&|{ukk+@baFnm7s4oOmnbbng8WqEKlM!;JIW!Yc zjgZH5*VF|y5IK=TUQEPg^tWWm)nvoQX5Mt00nb_n=>6)_kAuS0SDt6p{&O_uyzT5> z72191`8=wTe+B{;B0?7nmTwdw%=IhTHx7f#09a7@={(AQ_Fep6gQ;Hu(qwo{aa1C} zf)pZ%by0eAF~=jqi=^$2zy3z$FWqYfpu+JI@&kyHwz7=VcU25!TgE%g2T@ha7u?s{_YFotf zgKk#}Ds?ze>>?S%`-8D+2@oS;rUZc^!XqY2J;}$TZ=1@(=RMbf;^=Ra{L?qKoZ89H zlKUCj&%s1c+D`_OD9$&~A3kB86{ zTn^wVqr>e3G%dVvmU-hvV$c+acd0(2r*Dbd72&Cb11dZ;D_Ov*NaVzAP?`S9JTMZg zNsJ!Sa5&tY){dlp3N{!^=7RyADun)-2OG~py74kyjVyBpDYIga$%WQqntzj)B4rQO zf$o}?e~BgfU`U7q0q!W%R?`FkT{q}CXop`ssY7t~!+F${hMrPzkMPlxQ$~m5O7w|o z2?Aszku&iuBuids*@)^ynCR+0DYim(M5B02$xn{Z{?uE%$IXHqh+ML%)*`G|I;CJ- ze0$ePt|Lnu6T{7L6{jm>7RyYMLOeGtP@RE$XTV0=Kr;tOF3T9BchI6r$NfQ|p-4Xs zMdwa1*Zu*le|Tsu*@DeDsEE_hli8g(TjTf`79}iT;fNx+oonAO?sT;&Msrh`n86C8 zrwaNk=-nrsPy$?*$)F%46Oey;=!F(5p2cI437*bBeOVUj`>^Fj>=+fMDmH=-2I8Zf zML^f)pG>dl&|?zFAqsV1<1~R~RS#ck2I2{eTOr06`-$WQPDh$TJ(wu%O1+Gk)9WxM z`Zc)i-YvOVQ?fY|I1u4wgn* zO7M4c5L}l%5}-4hhY1KgX_%-du#MGjpU~ilQXzJssd$znc=pA?eiNLMCjA^^i9%a@ z)1wB>&Q*PVux+iDI%RKP)n0T@v zMeqEui*H7X59y{bEe}F{)1fkgaQq0Ei0q#_hBY*b#6=|Gg4t{F(`FmSfk5Q}O_47+ z^5R)~ALiv)P<*!Btv9RZ^{%I-O@sa$gL$^Sg*|mcYHz4Xx{eRbm)|d$oU%DoU(*kS z5+?zvcs)vSax-(p&b#7+E`GwIU^wb+vD!>)J&_pC^*soEo}m&i>szOpOXh7GxA)cW z+XYwuIH}&TOFb~59m{IEC70M+ZYW^3)=rCGCp4t=;&BOrd5yzTY_)_jqyAv3gQ;H` zb!&y)=UWEXjAr(Y`Jf*=h4@h%hpz@+pU%%K*kIEtDYT3DoS6SWfY;CU;W&Fq>D~6f z`G*HQM=%YT)^h{l%Hx}$&8%wVX!pZHT>c2sTPTXrY z`p*pnf9#`=Px~Ie!^{$IouE~d33De{7%h7%w7i8=GUqxru~1;(_H|LHckau-%V?h! zjfTdn@wlUB^}EkUmMCJ`4o|*k&J_H?8J}8zBDcY!ZjY_Sq#)ov(ZsU^VWP1Yi7;Z^ zHFnN8=F_mw4^<+5a0XJH0n?oU@|R6#9285H9QGqRP%0Wb8aNjCv4^hJgK&$QinS2ndDO_u+R}AEFuAv{K+`9 z_Y*L1Oi_@B1lQeUw+<^xW}&oWEI7f@Lt<GK%Bc%$QN6IwoQzbWD`mkH4XP8eU!(UCKy*LnDz&ULUv;7@VqvV1lc;2 zU;pSYdQ1~SEpkQ#I6`8S_@0|0tTKQ1xrvY@4sYWDC!%h zGGgYtvS;kr2v^dDQdkVi-tv9^*DARKk<2uA4CF`=;ppGFI(feLn1Aq5GV!2+>Qhi_ z6NWV!4!=E#%oJhan{3?@O0K{KO$BfI0#?c3T^N#Zb{!F(!{(04#4d-}Hqp~s`f1I*mXua$wAfq&V*Z9dRhxo387^z|5MzjKgRd-+?P0wRj}gkMZ%arzh7tQiRKH7ba?V1{f)-DKsfG?Qj`S|U z{6ZpK)vss{SQ-{M|8)Xzg4Pjcj+>d*ns(rOZ<5JOSZ3=3I#^-jJs)F7&w3^xwiJKZ zOE*)JF#yqJppO@LHNx7In6FB9ZN8GHETT5E1T4ymaS(40ac|ZvVIpGGbTAvqKw7tI z2KQdF0HJaR5phg3w{jc`JG(w^RBaosWuPUcKUIb5OaK%WjS()=bcjcTPJ*45Ur*SE zcR2+lgAW8JE^2Vn5QcBzo4u%eWTRc70Q#b?6*hr=-F zpo&M&yJ;Xng?4e>6^jhQ#=asWss&lW9BdcA@FpFRp9PLc!g*o}*&QqcMBSBRZ(s@c zl*x9b4(0^k$T$Kqhgj`QCX(%fOF$Qqma3NxVRnp+9-H_sEhXVl(7hz~905RX#1}ly zLg1@$7e8Pjoz0d#t`S%ovGK#p+aKVvk~6uCDsGSp8yEnIVj=MV&e}=#um=!O!HCJ9 z0l~B5zklo2%l#3ZLEa{5zn5LcYX_eqF8Jv z1_zXT6I+r}s><%qN!yI&;>Ka5Njq*KSst5GmKJh z>eS;7+3h1mL#MK~-R@}eH8VDE{)k29qLCZ&;Wfy1r02Cnt z>_sdV#8Cqw%yazU8g0!k<>{(z%vzg2Oxw|MV4$W-LI{NR%tTuJB1-vN$L_XBoe)0& zTxTZ{7N_#CeOMCDbrm?K$EqY15sO)H0Jze4?9{6IB2vt|ZK(BFYNF{oxqwTJyX7A~ z9%snl%VjoHe3OEMY#x?jAlPpO8Io#I$_T38H*hA!wSoUuSxYaU+zi`|v(G;6J@H?d zGNb+Tic9zTnZmHcCrb|M_dmA7um5K1o82&nQhVR*+}`kOtn!WqPfG->M8`%2AHy`l zS;os^bK4nX1{C~08TzldDH92-r+;W)FXtt%y)aK(GuB(?FIp~t%A8vHpu7w8|6 z5pBYxuf+7nqzbjHrPP?hqNXP)8=v!@oQV}f@bpLa(-FZl9a(X4J?OqCAe{sLI;6 z`=x6^yEBu3?K>P%CyQ!ViTfh-_|hK_zk>cYXh^D~CjE>Su?!e75j)j5|8e!k)siKU zJMWm^_fwK8Bb@*C(=Bt$C#Ylk-$?y$L=p6(34%WeAd(r27MrE*G3@ask@DLD9$<5f zBIq{RPzDwCKTWwQ63U-hb!9{3MeIZc==rW@%VP?k?TRww#6LYVZS3I%U#2n^*o=D$ zx~bCn{oKp;W#K1Yu>8$-RsiRUN^oZ&c@ge*k5#7#`+8)A5L>!?zCFQ5ZUDNd?Y(|| zmPO^SAZ|$`V2A&hjH>WQg!ghQWY0uh&)Q?p%==TO?t-3?p?9b5g8tKY(m<0e2^ItX zM^Q0!S>M>;la7L#skT=YH%vLZs2FE%2={+PPtbU_{;5j)y2ZOSWral=U;^-w z>+oRdRgS+-^Z$yRMz3s)jL|~Yse|p6MDFm$0-2?h+);b7kran#mlL01b&HXek*=!8 z70nNrG~yjiU`%kZ;C#MY4E4TR4vh*}B^dgQbjepDjurv02*K@x(x{HZ4G-Tr!M%DQ zJ*-+rzZ2z*4}2!ZfHfl(X)Ln9ae6@V>@D0gAe0O#xj#CFFp8i5-K9uCmVe7-O?lg@ zAfu*n?iS5`Y)iL= zFbCBtGCE(C1jcN$`^hV%*l{oz1zrKp)1`Y@%rk_=G+?_fdrC!QplQ|b1zqj^l!Sdp zKkb-iwo^%n1MpA+M>UwJ5XK+VRJbl%nD?-d>X5DF2164JZ;-SoXUn-r^H!L)9Xiet zhi9rN@ew=QElTr&qrh!x(-9O>8Wp-MEp0zCLYp#XV%hQ>Txn5SFfKcMdN${#vq%M< zwjUvvKHK|gzv&xhm*ZfX<(#WYPcTs=W2DTI@^~1}Pl8hIa=AUE_ItDn$(Bo!8o09L z$o>K%HY^7_Wz9f*R&FRsorK?UmB%GjB(N(cqU_#_6C_S2__b+|$|)}33S$~nJFdc1 zX~Ggfh%p{x^A8-|Y&nVWu@7@d%RA-{=DSPzoJ@qk-F#X7Me}AHP+8;=wtle7Og%JM z2C>we3y%T=rN{8N68G+Os-XRKMqCgYRR{sz4^-MA&PDPprTWY5qzl?%1rxBw7u-Hs z^i+-DP>5cax5nm&)}B&&^AzH%q>#BpB#Nk-W=%g$pAMV}`3$U{=;?6;OM-I9e%vrp zYF|npJD(n^jV5H9wpN{6GzZ$iUh{6>(peI$yENZy{2(`f-mg7n6 z-C}66G$xCjje_CtsqXEhP%I<>C)8hnU^s=zCiT$YtzVIcaKuD&;ugGUwUcUjE#K3- zZxg4eFY+|Dg+5>A%IR|Sf3(DRur|tI*MY=Eq&1UTfKLc3!CC+vy*kid`yv^LmA0Q> zktlb({Bvld4jOH+wk7;oYer6hgq>#A+>qyUacCOuNTk>tT1+&O^%V|?CA2loc2lKE zMm+ruUk~$tPryew=?P;X8oMa$T;PC60N2J$@5_S~clg)ir6rX=n5NH!;4es^c4Mwc z8+hRma=jj5yHiIoY6;0s?{n*q^>#^uE9@+n!#8I?(Y;B?m;EObY2J&6Wdy$v^9ujsG971Hx`-!2sOR{j{V!?7emzDYE~%exeXgJG z;`Q%ctte5xPo|4|_y%o&_dl#GbT&kVxz6+sLq zC#?UIhC>=(aa!X<=GGAP6%)aQtemWck}K0)>#`12TZ(onkq^^dDLhod&(2BA9qv-D zKhek-y0a?Qh9PJ5xxiGdDP%~`Jv2V3Afvw2*H?iu&#riD zkCCLGnvg1vHQR$*4>P4Xw8}x@w9QvDCkVC!$M>kmO1cXr;Fsu`rd|WvT<(Nc_M)zg zmz%@v6&2UskA)i#@wd0R=gaYLzT?=!7i+hDZ=b)voD7o$s(c>0OLF79`&a+3cMgUT z91oKI7zZtNad;qQdH^Pbv~?AGxjDdb0&{%sDOksdov7uZ)ExL=MF2CcE;(afpl}Sa z3S=cS+L}c?gD1LfMp-+I??D!=2}_b1H){^!VCF0_iZ8=DB-6D+Ld^`#)0G~U(PZdM zmW_*I-AOSAh|_V@QgYx0YZDI*1ZhZXdK92&_ZocKV~y3{gO@BZfIF5Z_6InGxlJrB zvm_G{z0`AT2Y4A^Rd@<58@v9nD_rx_AMI)V-svysdIq2n_3Dr!c3r5{%!ll}5As}P zN+NS2Iv&1t-`9Ey>85t!{Fiw4!0a=5ch^2z_)sBLf^!F32E|B`IbCxKHem^xWPf@0 zeFer=Dn@-TKTe+apxM)L1J`$N?*qVvt(FHNXP$-E(+n0$K;r0icQb_fSgoY+&1=f} z%!toBu?2UCoe^y-wiM>E3!=5lT=7a{b%!Jip0jU#FPD1J z%OZ350ZCB|U8N#;AHP1-crn)v8#PcX&^-kOde{=Ten^1Qj}!e?^hJ~E;gK5|GsY&Y znWusyZ^z-h=P4}6r()VL=o^34yFuIO*M4?%?1TLo(5_6Z3IZbC-63@!e*gOcPvEuf6ldRaeXi?$p;>k7 zjsVG*PdaHR3wcd?4VawV*G$I!Spqu=S0}Mp4?xQ1*aVN9YbbAoIl4hGI+xvX(+y#9+c?a+-oX<%H{cG5|Ks@W4w)%#8jJ7uy!i(}uX+x^bL!!8Yn#AHwxoj=h1WL8C|@>+5** zY~)P2(s=|!B|->Vjc~l~+1Rth+MJ&5CV@z-XcW}GwgvNOc5dx{LQVaiVvq}>Ztz8B zX5kN_53j6;kKi%4YD5LMD+zV0!sL|QveFpoDJWws;+^p ziv=Q*EzlA(3{#X|ng^vwA9l@vpzU9$)aIb??O{#W_`R=0WSd&`F9o0=a5P+*KUf<| zk`5ke$R9Pw8?#(P3eVSEt^kX-Q#cz1B))=NKVbClnu}?qWKc4;MkaXJtW%c*mYK!E z=tKAR?9bEZ1>7~Ap>C{!7xmfK4Z9BVk(-*v{Vb{=7;N&hyxvh94;O5>i^GIF-<$*s z&i8yho+qbEU^A(W5DQF;6Q*LX!XI3!kU|U&0xQfVM>ff$!Nc2mt`19yPpR#rgS{`- zBSn?$B5rY_yan7+oQO{*GpdYhb4n1>*jicUph|qu^H<48`99pJO~cZ&2KS~B%~A4*mW$eP<2kU!y%XBQs%Z0sd zBJ7mMb@DCi1#PQRFJbr*oDwAgbLtNv-USW$d*8V~rVy-bh=AiZPqKkoE|fK<4kjaq zf~k((BjA?1K+jwu0H}@fw;|e=cxiQmy|65qdg981emfs8vc!WA5)cWgW(K$AbB?y) zkjb~%6zW51f}*U%a)F=FrGAO1AT8)lT31f(O(*i}ut#`O$IM%PSf=PbBX*J?AwShC zvTPfZS9^K39V2|Xn7*-pKE0P`F2MbFG*db{yZdPIek-TO+;$`^GQoXJ^@RcqBK$x_ zbnbR=j-c%Il$(L!BElk=D)kp=014^T9$cA zXHh{f)kU=9y?lh*H$JCK>A_L+p_p=5-_Xk0^^$$RRL;(wTMpTE^WMB*Hg=H3454FT zpIm5Q^h{r`cGlXjGiF;ILVSO=OxLpq{>Hi|C%{+%=X@U_+L56!tg7zHeAo=iqTRy_ z;BM`KFuY=9<2FZ0T%@ag5u)bvCUjsA!yXMtZuJYBy8JMgr_KzgBw=KkO3?20WE?Wfq_J|4~~5WNlJ*qc};***dxb<1^`mL#(p}&viK+(k%Sqw{rY@> zZE|mtb@ZF2WO(eecDPCS51WmE59{k{!2~qs`2!WePkh4!iR_I{YvB6O^xQ97e2xn1 zVy@2=JC2Cl@9gX7HM!8}q6uW`Xe~p$5HjE@-|dcUNOSwHovf!}s?auf=N4*4)q!-$ z5OB?NJTb|b$*W_$KvF!^x@x@8^N-e0I1pJ>!D!HJc=zsygs~NxU+^~=R*F;!ZQ2}e zl+A~+3Q+G>CmSF0kpL{BCPZx6{|e6*g>)A)XX|*3S3Pq=%*4tnJdm@~tbFtC7yr!Z z*c%(k{@A)_#oR1A?<&5|hEM!5Ct*^lnN{2n5>*bqdCG}NVW&!8NUasVZWSY-n(FtG zYCWhfC;?Tj^VP)F@#@_NtdzI&lNcVQRp)FGlpi7sGnIH78?RW!jP(s?W!koCG|@n? zKty;-hWGP(sV$_5+bf^A1>u+Ood&mcC?`BNqYj-FU4e+|!l}}62lEdCEZ%@rr#QVn7uZ}Q4PX= z(sYzZ^QS+W4q89Bm~z z{V|kQoj2dnJXZ*mU64%+FYD)&g|?p;M{=AZ+Nx?7l;5oK+^ko2zK=~OIr~s-`OZkE zr1pn#;0^nq_Es8qp9aX67bzKxe9lCndv-aJX(vnL8{@o8 zM7%LECi>tfdd^hNhQp%iU9Oc-VKP_%lll;EP;``LY-z;+HSBo?vI^_5$tNHBoUUA; z)NJ!5Ovx*9`zH{)_^ooN?)2J1jr?brP4=B6nSQIt$<}wzol>ShA>pT)pwdg|5ztP0 z9#8^NYCrkvGR>~3tvM%KFWQhuGVteG$MBs^mT%mmobGB?b?wHpi{q6oZ&7QeD^C_* zUwtYQs}aUvo?37CVM|_bEhMK&eNd{&3jQ>!Z+#XKd3cek!(P8r_WtIlWZ;q2{h&o) zBf*>N!GM5O{ELj%gCnOvpCQgWFDeW<$@8!Pd{AGOo zW|r^559_%}j!~TC{+5mEg{QyRFS-s7zSb*?3@!NkYB%)eNV=XEby!QrBSA9bA^QOF zmP+LWSYNd9Ps=B9)en~C=8nE>_`w{~$LF@;&*$I&&T~n8vC-#L^aA9w?;(1sBm#KC zGkyPMvU3VY!|>vAPhw*|#AxbkBXLIeGNyR=3~N z+yM2gvxJ+~(_h6h9BZ2&E|_Qr`$Tt{X9&Oj+bQ=gB+51pL(%yvp!pAANl)qc9jqeh zqEtzB$q8}&ReNVE!1ESxjZ}B0-^RX<)iWp?g0y#!Y%BtT@E)X6YW}IX4VTR*)1w{m z;9p|<^@jau9mO{C#rmz^v>t!p<*Lpa7GeCwTHAGh2H%37d9pJXP1m*ik0tR_HONJDTa;`$Q79gF)_1wNrbxTbzhmJ@UojF;|Iy41C@%`%OC-vaqlFIVNWEH`Q#zixOMe}YkNPlib zz`!Ay&*7cdZ9NbX^S(^0L?G6UP4#m1G&HP|i2x-V136R6$B;e`JFn z3HC~`lR~ZP%$LT1Xt4u8*=V_$+F&^+)-jXy)J)-{5ORpDJ{zF)P0^q%ztW~qG(X## zXzXPUdizDfvVOt0e!&aW0_WwK5;_%Fhw$e8bn^B0C%D;pxPgC55J>rCRMK$oPK0Dd z?%&PQJQ`Z}@>Sk6ItSRB7}&IXd`V~}^_X$7(J$Wb#(G$~2CC=?1R2jhzJ1?TNNERY z>L=16ZLeh$LWJ9+Z)uB-R-IWGRFP~~RV4}TdiHD)arA57{Nukk!&jQEV>C`}{Of6- z-3V0N{pRyPP(Nj2531TulP9T|Jx)CsQf2Zw^yeCSBD~`A$Gn`qJwqOtyJwj!vbLCb zWF}{9KC;hke@>G-^H!C6t|i*ce7h8$57#<`>|@_d?RPdrJe=7u9VYL8ws(%XeM`NI zd>@DJaU9@jpyRy3R)-t-KI(2kPvC1T{sMj8JU9wi2h1gP@Z7y7rIH`aRYIU_s0DKc z>M0L_rG{TZR46CZ9eP}h@QZo4#J6R(!1CTw?i#w&MnF26Wb7zfc(hh`Lmwpt_}$_YD9QMo^J8#kc^cvgw;|K^8tJn(va0(tJS%eZ>2;&)=_v|8lQzBE)dYv!GRdb&ng?$4Pz^|SX zDK2f-)y|F*9DOLS=s8W;%=sGI;amT0kGx#1Q)E4?YLp1tb=~lcEOiy`Yitqw*5eH& zk|6qt=|>AKTN%7lgB)p$XrprMCFX>4v1{o%!rTgnT3qMrzjcF9BvJTBXowXg&PdM3 zGH?55v9JA6c9rlCQ;(Eyku&wbW)^JO<7n0uN%fH)(kv{o9S4hx?J-C%lucNf#? zhPU*Kd(?SuOHr?dwow`L^!hNM?Wg{qd%ac*w~|LpsRu%AF)`UnJeyeD8iyS#E%U=p zX+ZgbSLcmUXY!N0Z19B2Dss~3U+0-w`i^9j<-0pVV!p^yUb{joub63J^ay*H5`^06 zobutS+!Z4fMWMDv1sxZ&P&QESB%{o^2J^CNqLybadu1slk`=1R&-;hirumnUmByd0 z1ri&5;l9ZkwH&-i^)AAM9@YB-u3D}SAaI&SzOLu+8xrrKrN1u+C|57Nx(p8bO!^JC z7VA^e3ddpQ%*!0%!B1(3v~_qses^vW#iJK`d9n&p`xg8%f3Af?x+6jRhnB)gaa$2( zHAMzRW?jq^I{Mc=;>|n&D{Y~Ww%KMc3XVTU7=T$<%69uvlNrW*_vmJSuWtGS6-63tHK~& zKfw~T=w;;#?NDr7w4pc=7~#enw8Xi~W4pCVOJeGAM)B!hCToO*1nKfqFzDML7rf#p z_W^c&SXN?k-8h02ZTVNd!x2VeXe)7xH^vVZ9Cid@8(H|hR5s~a#&3b_lXDY^C;w^I zu-pRJBXNXUO&!7rAV}h2%y44h7qTgkM`?SUK>_|KZEq1VNOC^2L-(JM`BfWNhZ4OE^;3-PH;4jFz&% zt8h{Y(ZcZ8YmWOVZvKWjly3+n^fIeBKqy{W$2{tb7a0wzq-E%RAQsj?KlRk{Yxh1g zP7^9V+(hekP$lur)kJnhSNqB zX53|*V2%O?^&t<`FEC3%5nu_#w@!J3i?8k3cRVnyFDF_6JrN?Em4hMCRg$8quI~o< zn9?i80_WoRLLull$5Rda#rRao7L6>Up~2Oq27*x_PzyWDsAJ2u0bWNfCOV6WKPQr9_c#ja~}?%(kS*oznzAJ*SF!&Zj2wQ~EZ6Lpc8bf2`@-9GETR z%bTxRAXF9NUI|t(evY}k*oszL2x)$v>U|$c&h6GW;0>Sw4QZ@2J=w^@;_vBL6pt@| zew%hJkMQJzRl%-u2qZ?ASi@8&jaDqgC}nN{cP2#YWv_TzZr3Ye^y-F>bftAqUdXF$ zJ@*KDFAYpEj`r%u{++WNC2uCMc+2I21`#OJVAp)z=%P4zgM^a`K+)vYY8pd5Ab#K=M7|#m zUU!kYs_V;IE9Fpc+^6?X3_1*P`ozK|)d-*5{d*prrwhScW%8?lYqZo7UVBE8MxjD5 zipNeXL|ZU*eUF|QpA|Iy@8~@40%c4)Rk|Ep;xFAamZ}-AE@nyZqhAPM$(se#kjROb zkb_>Y)dI>9?GaiLc!$STJzE;tI4u7lQHkRX zbxIYadVw(!lS?NIk^8^XIsELo)(n6Y0v`a=$lJ5ZQw#-kt|qJkRTLgg!-hcz|InAlXC}$6qZZ-bd@huH9UVES%XvjO`&D{ zaZ|+B&4Lbr%-ae3H%K_kJ6+(;0qL9+TOZX?B~r3h_(g2~HplxPXxIfJ(RNc?#|&Nz z2{t`x1#E*Kq9xG2Uk7nBu!Wb`<}MUe{`Y=Cp0X%uzBbFTDHy%}Z6E>&Grq1M7L|KK zHgzFlhzuY9oFlNobpJl3!fjM=fYU_ATNp!tAXoaRG&1nt&+Im6QLJfbB}|pj;bq`j zqHNo_G{)iEKFlMGA)EzUl|g`4;$?uE!OHU%c#@ewCi?f~JQsWX&cUoAL!cXUF5i-eZyalrf+7~TaySrzj5akEs3HJ$YDzfSA_WS*uG!e}29 zu}CyHsHdjGCcuczG;-~O-p>{<nBKbTOX3!fv z+1n>%1n1B&X+Fl>L2|i2kSpxkEOgo;JXWaoY}&_7y&%QX4Zoq`@o83+jd>u%UNB zt%5+Vq}8(iBJhIl9gPWY1DqU|P??Wr++wUG96IyiIz5|4CTw`OB-E)GLqRX|w|BWC z2fL_#FA%b_Jbf4pcfnnbZ&EXKV2>zXk?gQ_sQA`jBvf7+i{AMaQU7!@?t!`dy;WG+ zMjkCX5+|!ZD+H03b{j(g+edqQb`wQ|AwD}HK!;V3%)LHGK`3Ko)x4=ADOrijq)vXV z{cbYI&m`b_{r(d1aH$a(TOkDa6RJ`o6ouEa&X|D1uc|uyn_~LFc&=k>~|C<<)w-EHagptvNaHE(y!!*BHL46)W16h zVB+VO?Wo}-ENsK}DOAgnFu|;K$*bIyu{$T!=U zui7J7!4F0G#K)pE#}Tq+F)*%^=Wpowa3;C+RKVlvT7G-rV7Of2Sg(!(Q)bAT1=FjKJ#>05b0NO3{*sD`|&mMb5OztP0q!s(Bz!Ld#&bcLB%ICRgwf^BG{I z=JyTFy7r%IB`;rHvhtg0Wx;*9)^i=NIYzPt(;=1(J;_thACFJfKE_LA_GUxu-_F2j z<8^iFPJp_4+F8ruGB6jk_;7!@fcN4+0tI!C9rfStS~`)FJ`<4w`Z_b6%1J9XEweMd zR_1>UD3tZd)ZmLI;BGem z;Msc7QvR2>>^^HQ-xYphWmT17WS0Cwr6zT{q4+~F*OpSm_b=g6L@+Rmmj#wb^%fm( zu^M4qX6K8|>e-hSg(J-#DV}Ch%kG!e2N>|+mjo?5?xI^C$M=~Z*B&?_&uX$DeM!=| zjVHhnh6ciTA|joOOeijA-xg#e6Q zXjfY^Bn%Tb8ujt947νMANh+}P`!cp0)VKeqi2n@G@4w|BegLVzR#GdVg>?WRQ zlp1dLsaIw-Ee!hpOyO#_3Dw<$2INdus2?&?l>*KTU-(vSISTzl+W|-T$ZJBVka;6| z)27Eg;iHZ`W8E}MqV+UkngAB7B#%&%7Lz<9^Tl)D5q37nP96Hn^8zZ8T#nkMsv#jW zjNCwYMi4gIdKbzI*Hb(gd)p?QLPRlSVO!+DD&3NTGM2TF@G+T6uzGtsc1UTdl~o%g zM()`}gSkcZo6BEyo_!i3)uiwKi$sFoY=#b}?$=VZH(pBMN#E%>f!%U@)#sHbL?038 zD>2p4IO{K-^pIFdW(M9)zd8~7tR>@W`GGp54bG&1d)>5Ml;QW9|Ka4tWtf+m?;Epa1ev&a%W9UOg6bQXOn_@-Sdgm^*L!M zj|`Dxad@{m3kL2gZxtaay^>{mJ)w32p*hc3v(7r5Lzi<<;0iu&n%C_MqaYW;H$8L# zebWEj$9(^??h*D%&R6Wywh5Hb?tG}Z(Qwiwknq-|tO^qsLNWVTS$xGr%Tva)k{rPL zx=;@Uj;9s#Zes%eOy^Y)+HIQE&6L_bMAH@DLq-GTUryp=ww)(>a(~U<`TBb$GRD`N z!$)-v13|RhWwTJ9^W*`zB_F*;_oFowi626xZR2u^e0>~q)wH60(9Gh0#Tan>_P!Zw zel@G%RH^Q*;gy9Bx9M*1?UA7mMoSWzXKfPS6xD5HV0cdu3pgxD!jkag*rMw=d{mRx zjRjkOb|PQa*IbYP=V!JQaCjK|&F9X5dI4|K&u1ye@YrcAiMl>486Vqr)l-L3BI7#mvqku35cCmg+w9dYvffrLqUZ|#*87a&&9 z04ZnO704T(7mp>4bDda9ZuOd?qn*I|Z?rh{wVJYDMFbDwll4f0#+77JEj9@Erf1|p z3VW+QDmUuV)Dz*orA0>|tJ-*X@#jWFdQEyMBNw1Rr88+mPS-gh$tg4j7=vi&)pQ-$ z{mS}c3s){@#=iUOtF#?cIwpUC9+0)svzoT-dTGyjSJ?-D^_AI2J6xH?2|MEju>75& z_T5|VOjl@`=^uk{N|=j5F+P(7%V2+pM_O-w3rq8zr&HoGLg@x)lCixB2fh^J>eI<`FD* z^Z~))u6rZFE~mhQg-<8lVr@#vR%Tn@s16I!HUsrBCPuF@)x{nc;%!R-%tg{$Q#*41 zY)+jZh1&aa#v*=}=HW`##0MyQqt50)@Vh2qU63X|(RWy31j-SyGZf%~%Jen{LdY2R zRzQc|-8 zjF~FXJsA`ih97pRDIlnm2c<1B>t51kH&5H(G6|WCI!e%y>yfeVrhU;R z0n&6@N4VqkieL3A&Ov1&x!A!B=!cq^uAO4n>qhFvL01GWk^w#5@_{A2k?}le^NOXY z2&~sVCTVZhjhrGQya?OE5-4&SJ|gvVFg8W(mdh0InwU5WzxYmyXSSiWY0L+?q}&V0 z5!8W_<_SAau!!OV&qrUWf@7LoB5Cde!}$$FAWlz5?L z5*F(^NYLA!C60|?i=w&PlD(%|;y)(h++G4FYInWTVv}2zT;W7Y9phZ7kprAWmZ%eQ zi5?oVIROIs~BWl;%dastB83}Z-y0qI}LsJ_2x($_18^>zQiBV&axJJsjv>@qt_zZT&uYNso$NT z39Bi>=9ZK>ULKz)!CV1!XxVIeQq5CRAxSTCwF%*%m8KJI0pGr;yjo@Xb~~Cnx?m~> zl{TTXc8LwJpBohM>SY z`RDjbL`&|znhDX6SPnKkY~IYer;ZL1*P)HJ<5K$tmzYLFNI)%qB-xtADD;l1r${&S z{A_z~>k47U`6cD0Q8^=aUqHL{K~6kUH|V4MXv$)PUHL4)dGb;wu@|cfLtw$MBDstI z^zpFcqruH<5YiI{5FSKmzM&`;W>Kcz=+or&i~Q43BW1OwooONk*AQ^v zPHFMKn*%sV7^WbsKo}k62i*5wa#TYDgj8PPFjg3v`UTsWR8>TXQ#VCml1Nv=xN4uu z%c!a9IW@1p$l%?j(h1kEC~E9UMc99}c4tx5dRsUpD6Ot$w2Y9h4fU%Pz_k64Qj6tL zI)g?sNwg?D{BF0bo&E>Uka`asw*s@}Gu!7RMsZ2O83hpJYo6(N98&7RZ%pK}=6F@M z^+2ZGvEkTe{#2$DRJBq7koFeeDGm)eoHeNd1GU%AY7k-Dm%3tM&GM{0J2HgK z;1+5j8wR0jh8=Th_84n);FJZKo1Vk$VJNYT+I^<^TdmZ98tOctY1eKPc*i?>#!>&Y zT;=NtoDtxr4mT7IkzOBQNBGBN3sn95eC6%bpxKxftR;pk(I-oH@&okNKFP$<6X1Zf zruwqMC$D;uI4S5k%S(S6ci?$!-u!9c{1PYzBGQdZkF$HiYTL$9&--EMRzcP>i$1;o-^h6ap@t;|!7#;z-{h(wYyRk^~< z`mj!_&dJz_RMa7Ks75yw(#~o@i3~cmg^HhMdZSjUzJGT7;_*S`22PqW@a9aPT~O@)m@O0tyCi7hFoPDqdMl3HUfX>XduA~=d=0Iy$Aeb&7aW8^h`(Uh*8Q@$`3BB!jVACD)X*T!N+(E-H)Jl2vipw+>`AJC zA~nIZTJRk1FD-}D9Q3}hjaOB^j{y2{LDK!L&ww{1H=YwEI zY>Z}${uT)@b>M)%DTpo#SEo81=B*k+t;ELO?B5QNv*lL9VSND;W)Tnoum{55=G`DD zXy=4uP4q)MA!YM=!rxQ9=Ort~*DbVT?933L)bmRwxW1ZH{)3T@9x2{gttaTCc|fku zCm}bE{e|MsJC$YA%w#rd0(`!gec(xz?6b$tEgqA4<%XANwu}hCE{`E=Z z!v@EuS)_ohuAqk-Gz&5LCW0Iu@&iQq?f$_ z!B7bH!_lOx0ZQvg2{}3YsdBRjAw1OlA+4!95AqDCtIXU~ha87U3#h_gM-Xx-DTgzm zCNUP4Y6hV7JRo&C)YT!mqdyk#IKg&z6Zf;*gFJg<#~~NC<9!@QGgfDz^*xAV`mhL}4@Dmi7 z0gs^(ySL?)!wnyEzRjg|VeLj~c+)`sY(8mw!Gk2)Jkir!B1u-vN%@ADi!>3jq@ER(NK z0iobYWCh-?)nWBp2sEBijS;@uy|6I{o<8=z$8!dqAURX-*;u)&e$tWl)=mCPjI z$45Mhcer1kiUv$dWeE9yp5p)IO9JRlMRS09Z1spfjRwXpyA}b3FMNS<*V$=dlv|~{ zJgsA<+mEOrd3HY)An@^R@c@@s{w279lu{}~Gq+yc2e7jL-2-s-r*OI~9cG3W$@c_w z?rp1Cer1Xzvaj2SxzUU##EwTumC1^SnSV+g6Z-U=NOcI3KW)yXyg|F0Id=ig%$ts! zt0~@BKeX9DoX>I^#12IS8rP0%yMjUEV|j=w#VL%>tC$Jp}&WSmtu&ljw#SitblDLitEPP>?H8i?FJ5 zP~I}cCQkVfA0}lj4lvPX1PJNFqEI>YP_9uPU~r&kyZeS|gbq97)zj_I<1L9rsxDS5 zE1U=4!fwT+i~5S~gQUfqzduMEA-MAu6+^rJ;ac~p-nZLzy%!f{0T0Az>yMBcG+CzK zUC##ok;#5&v18W!o_FbZj#I8{AoOp1TcBtN_qP7LlTjmOWkWeK9Y=}clwo4gtF&z_sgbk&^l+M$aJ%Oz}e@7F2s)z-? zUGI3X@WWT4O>WHxl5c6Hj~}UnB{1^82eEl54loIqtyG`_Sqnq-(bO*bY3!2kOeB`Q zai%9b_W0;KfiEf>b5uv@Nm0eMCvJ4nRa^vS`we&v%PM|derC8R2c|6--J?(OMgOgy zM$3q?pMB$gz+i1-4(9suzfOMJ{=ymlp$~lY?vh#p%=RQ=GXM}tf=vO(!M}v zi=lT$ovkA4F|mw%PQM6muJUI)tXePcZ586X$L2O<46OES?HwFzJ4G*JXyQ#(-SXe^ zH3r`9`eX?BBvqeLh7Z&^rS$0A1C3dnjv2SVWky4E*D7O}l?A($!iZT5euV$t`z2Yz zc7losWv&Fgh+lQW-!5}+DFnX|N65lMzypU4;WR~|z9mf3sNTv=O#>lx9I*v%T74UW zJ{zmN6}0R2TG%g;=n{!Fw`tYyYt4@UF5(f*nD{mB+0R_AN;)|#srmHzib&cqT^n7< zQH+9-cE6coxEcgTWv!vQHC#l;|N4;NEF;g>tkaVjLN zJa)cy-Z?rWw%j8AHPE3s$x?Zwm>^)gs4(gjMAkZ}kCjEc?ebo@YS`J8v~TBP1$WG# z*}iLd>|f%>6KZ$WH#wAPv|T+`+x)EjI&#qri-+D<0mmkd6&h;p-RHw1iMWZ$3!~y*n9JfN=$bMx z+f*_x(z+fNFn5R*(H(I~4z@urcI9|McriVHi?-;Ls|#*D1&!ZXZf7yH=L@>lB^Ta? z7_;o9S2%5lO9fhI!%PhSxQCjzyd8x@7fA}^BGU3Z2{Ke%u){6He3BfF)Yz8Q z`*~=?Fa-pmf7#_)u52#_pH0Tz&TNd_4H^gTTLHG^-$ws-E1OHZ4_v1YHa)LHczM1G zDDpev$Gv-&SGShZsC_BxZ|V>yLsVGak8zLUJtz`dFo^1x1Oix;9x?P3&)bNtQ!rz4 zeNn8vVAw6s*Q;(&(ImQmBR0)d4b^DH41;xet{99To(B?QkzyG1Y>xsR#hCX)lX$8p zbb2gXfL1y#Kb&piIcxO@rn%lWVMAx(vQsyUAXHHmt6*LJ-dLO+(DLw^*Cd`8;NMJT z3G8kB+pWITdE+^()8nH;h7xJAqE#Ez(1nHszj0A{W(WOkS?_Y_2TBYyp?)PG0y^=6QnlOsy0x7AfWC@FL2u5V8ZgQ#I4XA|{ zZ?cOLJ;;|uPy$5Ki%m`j4kUK-ju#vj?g9V3B&`^-`Ayx6^>GXcR6P*UL{2Akza|O| zO#9=mb%l2a5Z`vll70}&gY2ZXe(+Z^hGzSH`JCwXW|_d1;8;*%C1>);pK$Vf1^Vgv zQM6w)IPtP0#68RkHu(yh_Gq{z({rvg;3$4^FhXBb$pKTXv*U81gDBTwb2MXf5lWDm zcT@}nCP2SOJnx+B?d@u&Q%flPAMRc`0(bF-B5NB*p?9_()5*RIYnsv?Wpg`U)`O)u z%S&n)70Ir7WpavA#~e?Zm27-C3?}tPbu(A&15?gvfomR(wWXv*NS|NKx^yWqNa`6> zt|lohv6oUy4jsqpr;hFOYz~Y{nrYNOZBN%DN(%gvjKie6f>@-;BBxc1uqRV+AOhQc zz5NxI{lQ#7ev(hcjM5LBefipENH~9~p-DX^&m7H8T~9$i2b+9w9ZBro!jLK4lPi68 z`4os%qYPW_ucu%b$?N6yg?id=35abc6D-z~#0gKb!c*gte*=nN>t6lndx?{Bb!LMN zHnfA^_rpSb0qX9?eLMDk{R9i5M@xr!u3sir9rnWPJjeV4kBXZGt7caU75Dy(nisa| zm`elbQ9^4WP2y1I+p@uif7m?Qpx!KEYiZg8Kqz(Hb{lHhX1@CnpC={1lKqv;YzF=+ zCR>kqezm$2y+~D+nhwE=E+FL%p!40?4-Ntl?2A0EhL>nlK!cp$H%Pr-)NU#_h4$vZ zi($)`9qR+xaQel4ZLQj0lOyLB^$)E*&N{Ze>LtbH<4&E;yZnuJal%(sGSZHS&VWnb zh*>hvzkMFv8Lu7dj`=(%q{;p)FA5JD`>rldk55|X_g9*6-rhJGs5j4i&g?t+(bml8 zaK*)mrJHxBwIOY)X6gj@o)*bB>z7>K;rzG;LVso@URdDLG&w1L`-FLquk>UiDWFfh zP_{eFjO0bH8|?P&P#xf{5w{DM6ZT0HJT-pwZx01p7NK?66Enx~Ew;pEkiQM>JUaZ; zwx&5f?LKM^jn0!3#2BAUsdF0@`pd~-$aQF;6PGyUxB1r#NUa7=ulIYDKGV3*Xclcg zV)9&;-fXQ~OOv(nn7>YNthxWF{IbKWTseE~M0ogxlHQ{AnBLAwx@W4~*Y?Y9w$97# z!cQj1rr8=o*azte;j_-1FldJ;SZ2A5c%4Kxu&`7#$Q%VG*5y3}$VY#vuC1@Bz_0%G zv%y0UO?V$}Ja`slpG&804cm(M<-xk<0q;0ToJhY+CHK5_`(@lWxPbNYx6Ao`Hu(O2 zcLcz*6}jVYedtcVIH>Tt#W@`lxoK!$VYwe%E}zv144f`|M>-R!CRtfm)NLK8yLzI$ zRe2w(Q{8!8WKudz?1VX|)^?|MftCm3X3C_0g5#BIaCeY*wPeG6yWuH2JAX!9)HK8@gfzj7pLF)ttgOs9#@>|*>=5#;^xaCxI?H@)!T z?JJ9G{nX`0mGFe?p{j_ttfcYajX0$$`F;tl-$U&}Vm5u;+W};h$`p z)qXSUDF;UM8RK9Cag~g1L`D0dQo#eQJ^`#D`zGBOp9wjf@`d=E_sVa{0Wb|kluh^M32G+Ep2CYK50i0~qQ!qF zr_JVKN}}DXwkX`Ab5CaC-L11R`rZQ7RE#}gC{WI$RNuhni-(^#@7ME-)$H5exYz$~ z!klnPIcsF8{d*%uCO+T=FyP(=2WUZvuT+~>d&qDkUck|xYd%TKNrzzce+;yajWaB~ zC>O&i7EV=>bjor0nxBjuoDFv`!Tvt`*-|X+6B#$k(?3>K&GR8R7 z^srwfgUKl*2$@!$uU!h$eaIlsfqcEgcKZQpw+4K-%8tg6=)el96W>&!L7GrPLJNa> z>vl57QMc*ocR!1KjoFRser*hYv7qrk$z2rf(d1_J5v(!`*Mx{|&_c8v(4Ph;scyef zP7D!yIWDUu$_p$+?X6I?qobqyv|!-jnyOT6FMWI@FIo4KZn!)f$Xa+B=%#;&30cS|I`UnW2wjSV;M0%K{yLO*d; z-5lORF=Av%_^mV{FjkrVuPFv#HX7Q!2E4K@O=W1sBwdNs5*0FpmnaCFS}Jy|GSl#+ zVNaI}8&`txBUQ*{*nsck_laZ~@gIle#UtpW+5;~)(|)Z^HGshig`DDs*AT~jeip(kYu z#iTPah(`F1RccwaE7UaILIR2N6c8~~GiauqyqxquQ|y0zXj>thVe`D>>RKZGgQSWU z%Jx6%W?O8jAI-_lU3i&F68%Zf$Ul7dRCMXR$31k@-| zV|83|Y6x8>gS#xA)w6#=RSct%h6U#48O;W97ZGb@xqm%TgNUGZsDIQS#3JxbLM-Gh zwo;$7zn$LGe;_mV|KT}f=lg&c-t|uSx96PDZq>HI_J;f1q}^|G4>n&6w&!}<9f0P` z;@BUvZ4A5i2mC)XKxM++v=s}Kombvry|IZ(*^exol; zi^<@t#tUKp<Vm zv$-sA8yDJ;OqQDaT^OD1H>u&asR~X2ZzBp_L2nfFc{HvS*2B|;?|FnhuGrTmvtsMv zChlmEepp~Rd{(PG$ z%tWcd{1M-)a;n5^dR*5LP|1%k1AT7QZZ>fr8pih9l5xL{@GW}U-weG9{1#6(T%W6q zvtAA>Y0%}&%$~>q4FDxP4Mpktq?5s>W@Tk6)jeZcC|mX1jp9~HC$d73*<+|jhq7)OVpd-{m21=i`#4kRQ;$nNQ-JD~`iWuNT`&-s$1w6|{h~{bqiCY3_NDrpr zdE5Awj`VWfYPGxY!k4=^f6q;-JLio=gP)dFQ+%`D3^p%$49Wx@_m!h`%Sl>N^(nX= z(xwUwb@GblcPLoyY@MwzDvjr+4efb4dogj_e1FCYLI0`V%mUVzPnmpkv-k7(xu|Nw9!p_} zRXt&eoV!{L+jB}oHz`0~WafiJJx-0NLu0Ly>Pt2+x6&kwf%Uj}CaGWfV5BQvX zjEf8!acL?p&pnPUxW7Px*NIqD9FP5_2LrL2iz%0RV;^M*xe~=14RkFJ%I7~H(Vv2U z{*SJ+j%(_D-#E<}HM+Y-r=$X-yE}xTlz^ntHAc5Ex=UJ0N~F6>6ht}&>CWFi-~WGq zpKY&m&bIU1&wXFl`#S4tX^KhMSF8?G`ivraGj3m9Ry*+5EQfP-!C~SBZmzEGd#CE} z&5dBU%w`Es``cQ1cvDU1PNk^#5r>xU_!;_}CWqaebNQi?y{&(he@j0eMi2dxvGQBl z;mxw2LpBVZYFlR`;wbG_A0|1_w1o$oAEYV-R?f`w z{%HPN!r5F>JxOl3U{=|@ydMg5sxW=qHyBsHbZ1xPKJ`vt-K7^E-P}|?+m`G4W)S}a zTKn+R&2;|59je(aQho6UX|HqMZ@?72@ALjw$3pmZ=;^8T``I0L^W>6~1WPIM2AJE( z4<IC1G^-Qb6=B5>003eGAi~c zv37@m8#%z9?bKl0G7GZz($Fs&-7z+eEC1(orQm8Ymf=BX^aed+U2; z7G8r~d@9LtGfRPZ&94j7l)1HkbXGTpxb3P1M;OFpM#IrJSG~vEw~f0l8aeUacHZt( zeOySKZ0dU40M1G?^ZY~n`=aalcN$Mk^WUt}tJ9p8N5L>d z#N}kK$xb_0a$B|sz3KV$(fx5-l|C^q{*fu-U?sIJ!!o_M4H{`_81Iz%Cvh}Fvg%!R z%e(Z13jMMh0Uc6ZE~z1TqkW#HH+y;)=hol|)-%B)BECmPcP)Ej9yh)CvH~VS-5OY znp`AXDB`0yb4WQz^*%T z8~aR8^?OFuF(+@ZK;#W1we%*=F!r*$!9`1~>pw~QEei*Hj$wQHOAKr>(&(*SaVr|p zkDp?AU5r1Ykr1bz6M}&qUGznv5{H} zL3iGa7rQF)xsYPrSnIYXr96P>z}n7^=divWiIATFZBLQ@rAP`(b^PSN0<(c>b&8|H+@CSIe!yM zU8j9s;5AJZno$8y&eI`d_*9{d4k=#@R}7n^akAQID|}8o7xBUvRr$?kKCJkY?rDWc z#vK@q>qY-71$9lUyqIA*ZZpz$zQqEsg1EZZptS#-9g&465_J^JuL-WVG2WBsyLZUg zJDuf5I&YF7jk);m>j_L&3%!QUnV4m&hZTsDo9hqe>`51ka5416d z)QZpM0=w1vX2W#2z!*FP)P$sjL13|3cqq?k*4|VDvT~RqNv&Z0i*^k?7<)C^*LHBj z_V>x@mLm`ovk+E@<;c( zJUbYd=pkGO%gdcpo9MlKugrWtrGa9y)P|h6xSVQXWdYdP3`avI6$_3N6|t*!&@IdZ zFvCs7?G3#e3T}=m_3jU6*nm=Jv;wr( zE>=Ecrj{akd}lcM=h-m{&a$6KM%0!A+5c)+GsYB&%RuDhVyu}i98aSt6pOcI{5@bU2^ikSkQL9o5*`+#CN|~%n`Dauv-?H~&1S>1+p=SK z&VAXXat}Yg)KKU|hHa)LQE@jF?L`Ced*7x~uZJ5z+L#?c2st&VT z^&34(=#AML+d#J~7h^YJmp|G*8k~@`<|Z0qGodz^Ov(tA1!ZztFNz_;$$9AC)AkcGQ0BbdWg6>@#tw}jmEPAK zux5w7Ai#6xdTa}c8OcltxX5^^6S_fWqHvd4wv9f~dVr(GA42FXV^0@-LwN?AE#S9z zti#I<7?uHY(X3It&+H8U86ch&ZqEY_yO~(RxGFjc_%GsBK@yPN`acRiczT``27dU* z?T_ug#(O+J>i%UkE4n>=&ONc)Yvo!tr(QL{qWD~Nj&5Jxl-Ftz&QMkJn$Y*I1G)$83*eO^=)jkZzQ&XjH;sm1JZzH-jYFHTBWR z+CnewDYeT+dp)C0P}2zXE8(Ty?%l#?y+5aQB~P^7?a|hU@oNMZG@5~`RLO+R3q!Yv zRQ6KCegj@7of68;BxlqGSltf%q8s}ZIVUi8E*~9u{7m&&37+@R0c7}ORFk%v{fK0^ zO|jh9P?Q}gl`}Rlz6HiNw60Thov4LM1Q^lUWEEKh!pIVNWU$~_qk9wj8eC>+x_}?1qxG~s@ zL+^AqlJq7q+ReZ$H{{HmeBu0PUNn2X6?#JuQx@&?*M1ex@iJ%V+h{j#v#K1vLFq|U zk7Z!60EU5FRI~kOnm>45tG$Lt$;HZxo56v-3+BOrU8d>#nNGqT&+( zVI6v6O3^P{<@}tq_-k-%a?C!{`{;UMpn`ay)EV_5&lc` ztJJUok^C@WZ$V9BL7^`jgfs%xXL75N;prrDLVF2oLrdZf-2kR0U?tc8sEv31+)}iM z8V%x(6h{>zZO3=dC)X62fr~?#V zH?S^6!UCygSOvIt<=ryI{g1njv-tyz^-SNtVNYmbNst~bo63Q%K23`L7n@<;h5GoA zVE%?9u3^alVQ7$cr{7prIhuQWOZe^KY|IP;vlq#=jpWd2vkq9A{O9NgvahDMbK$a* z8`j3izv{ra6%x4B#AFHr=rjektBt>MgdpV~vhtr?P;;|1)IWb3eHOrzr8dP=e8xC+ zAROlB5#$-uVCxBCpWXaj=ex zhxIu+g-n}V(;R#iA@W+|L9`bpuj~!OX{CEjfwjJM)ovAyV5NJxh@PJoo|;{i$i~g_krU-7(0WT5sxz=r2<~vL9rk0rMb^g1^r5t?Qz0Dl+`6wVgX2!63dM z?Yx&W1DX4a03xpavp6L`O3Q8`;O`-q(2whrZ>i5|2S1~+fX{6G7<)R9P@}te&s*s^ zZxb{V(rQCT!b!;k9tPzJQx$9svI(UG%kk6|i<=48piN9}IT3wv6LjY-Zg7`S8zCg9 z-jEL3B$g0}YRLySFagB;xmxb4)YGCAXVN`_4+#A%F>&p~Kioi)pxzgC!rv(yihfuk z?J`9EX;$@+9wVJ#dKE{<`qg#@{4n1MVy86EbfeVY4GqKxVs8rw-<*`MU;QY;H2QZm zLt-_V?-UkE=q7FWd$?Y{rZf3Zotn#c*YS%;d}y0Sc5@|#f(a?DI4K3GfkFi5K99k( zx_Pp?4hm&{xV3>DhE1miq1zDOx`V$dtNU66Q0s|E%O3=c(b#i_Bomadr4$Tf_s6FE zs3Wz7vffCx8|a__@&4WKA1*ep+#TH8YboE^^*$0xTgyihI@L2%|5Mro6#_r0KD%nqp)-O7N*e{);->Zhh@=kv zb0&s0Ef`EyZ$BhhglZ2P%26HgNHCKNgJHf~l+@hsW0XeQ9}&~s;@4!!qW|}m7&)|5 z)Y0Jtk$UkFcub&3lH&r?7gG&C-w_NNmTA>*YKgCWvobQUKBow>eZ$eZqu5yB;85;8 z5wDxTSxit-n@LrK?C>oO1Ati=?vP~u^G0EbQ|GFJ;h({bkmQ<_ z`fnJREboX(NBaWiJ^^ALrg=39Gca&bxS8n`hra*34?@L2w|h>}mWGP0_=J7?a$zSO zuCiT3EX+*wSu*OCQ1-~`8#-~hH3frzQFDVut2tPj`51jp0KyM{Z4W3y3d^GeSOald z4Z~UUm`({!UiXH&jFnsOhxlh1IH+J?exuig^mhjwpBG5Tu*qWlX|(Lh;@fA0geF4wd{*}cU{IMK2|lAh(TKB5LQt}^ z1~IMEcW*tc1e{MiE)#O8wm+diFcW|fZpHDUXLuSpA!kHt6f&|Vc$P&YjW|Fq@d=W$ zz%fhF;7i&T1EDfCC&`bJ*CAfhUENqxhZ&{znX?dj-5yk<-X3g*LriX%sxKAn_=K=< z2nzyu^*}1ug?c;YU(5A|-HR!;&?kVnsmMHwKGVUsTU9s4M zJQICHhBaI}aan6DmpPGlU&yXACQ&|>o~Mi!+zjFjeug0((3U|21BS+y+g}^tCba$O zzyw6tU>WCtn{$TU(Fnz|+Mc`Zrt#HD$K`@~AZ=;eT&Sz6)Hho}!yL{9S+wNqGX}jo z)O|aL)K{$ab+eFDX4edP`OjNOkIlb)d$fIWlY2-C=K91#hF3rmqg*;}%mZXQhD&S$ z)>~wo5M91pGge@E`O&7}1HVSlsYtLHy^Tph87za1q;GoJ+FUXtHr#-AG}g#2D!%!> z%I6%9m}6z=4ny|J?0yV6`>Xv2V~gn}00|fWYVLoOAnfDT=eTG2@qqpSB`0APGNxn~ zht%DER7lT)pMf6+gJWUCK=dx$8(DQMVp3SF-J8~xPv05LYph#K=#JDU{IXBkL3JPC zXO~E32aKUIGh`aZpbXk13&@p6kiEp1%IsFM0Hq!>#mHR5GG;JhFkR#x2MjT^sLUnD zCnw50`8?wA^Iup{q10g&&AGGc&qGBflL8KaPJWpKS8>2vu@CZ=))2j`5u8RCpQ zU=A)onTyI!j9vOvQyH$ArNaGtganI>1e#jqK}0w!7sQkbh_W7MfMGy#0q=5Fqm4rTT($(`PW_+ z%AELh%y0$8o2P^Mwh+z=N@8*Wc0|6W%N!9IGTfpLCf6VG`krqYWaoMUPX*Oje30cLu>wHeMe` zVV{m{yVAlTDcCXh3=u4P=PIXT*!?&yF)TlQ{94cAc4FXRXPm}tX*8XMvzt~XSSD;q zqw>c9ZafZ%-31z?{v}uo2M`@^)C2lF8JQuLn5~- zmlazOGeeIIH7R4mV_`LvKHBD^sl(gQ5x0=`5l%^LnQI~foo8q!NZMIdZUMmjjs4{F8d_UMcYqoLHC_GT7gdm0T63JX9+x?Z?| zlQ!0)dPSxKhB24Va|?E;pJqag;N-C3~L8nJXWkROc8o)T694#T0gDjsGk&;U58M+vfHB%_;3L=|M4HZ5qjKlvX<=(xs8y+E8lIi)UD2oM6mMHxgI9x>!?a!~~Zgzv0X zN~7x(m@WvC=sQa*idNl+UmXq8w6Q%4IldY|MFSLN3ftdt-;#9VLg-V{HAalW$1ri6 zI&In6pe5(#i~&2A|GD_b!jn0*LcF;0i4;UsGc*vjHX8)N!@_5=s;*mPr?KgZfbc}~ zS$Lg>F+I}>i4)AU`GhGT8&{153Np3}iQ(|4X}8cxf`Aa0+Se-w-c2^oIHI%PHz%bY z-S0HEn9${6BYf?9EeVjAe_5-RKMswXq(dCrGk$5*7M@ruE%4N($b zflsg_`VaBGR#0lQme0mAeLV^lQCSOLGhaEycPyUm?k-8*ZIcymeTD3P20W{j0sDmE zoMcHs2>}mSsVrm|_}Znc35#b40md@h$f(_v5Uc+1j_p_b%KjH4r6+I8$`#z$ zSJ`4FRImJ+f)k`1PB#><7R+6jkfHYfFcg!m&~Czjf*J~4l~MKermJZHxY_qf7#Xj}39Ws~OyBH8PE>*_ ze|a71>fT^&>1S4G1oRmNRwsw(_T2#Bp;Bjg0|x!N>uoZ!PUYE&PNmy}|SW zWAhiz(2KWTZHk4x0lVL?53{pfcf=4CuFeh;p6H=H3m*k&$#TAP*E2`pGdSCBk7Mv$ z!-_hJehntjxG)40|1(5ksUJ;c+hhoQ2>sVNR2y_%#ry1pYrL0# zbkyg}Z%sZvlyKM^h=gDG%)QXR&wF5I$e64yGcoVJUN`dD&-i3-LT})uBni&lC37yT#R`98Duz2r8%UxJ*p|Kq#L>u1I+YFhM$*XPqAw%ArhR!Xlsd~8aJX>UenWR9;#F1f>*Ke74@P$eX0jw0@iHHNRIDhJ zR`d>vobLqgw^UTsRCyT5MdQ;iuj0gW+3V_@?sMFXe!V_wktcktNAial4v`wNx|E=g zFi$I_Ao8g^p7urWvGdnHEvY43{_lHYL+jv>gwfLkAaGpgPS=|J#h4uUOaD#N*K_Ew zI5MZ0u%o$16+H69<* z3kp)|r7$p08C`LN8LAlLaIXQthY-Rgk}0xiJ_=G&flM_01Y}=ouAorVrgiyr=@}SutvDv|7Q?_$!(jrjAaTSEb(K zi+)^N4}Q%tvLX+UEv7UW#(Ti>28D0Gmzx&9`YxYByN8V(mmGTvCYFlW?FgR@JduwV4bdp{=n;7<5Y8WwG)Q_Lb@6^qQ|yvztfe3nJv8ebt>6^%5^MSn z4w#p}UgCUriP`=`>a+#b?5em1pe~soIa3DnECEGWv#0CmLz(n8vhW4S_d}8^z!^0f zwv#1oDq_A6zlwg1cW5>HiComaWichAXrRHv_JITwXNBV9*-&wT5)~3TjV~RMv#6*_ zl6yXOEnrC$Th5aE2=Uq4ArDiRuRl7ZHV9LV@}g6%`s1&+DE65wGr8DT3r#>O&aNEl ztmep^;pm8ui%{G#uHWU@0da;Q+ePTf+uEW@N*jZs0brcV0o&2dyqN!rpdz5ytN7`)$s zgK3-HHtFPR#>dvgmb?bzsDj8@vv3cPohL@x-hyY8ozi$MO7CU&ND3j)>ELnF9{(@v zF#HFjj{!p+n4`M383YcFaC(v^@u%bw&Hlt2ZaU_yf|A5GAm~;(s6zD<0esAVL#_iv z?i`fECtdlgqZn_G-=;sTW7~gJj0DV&!FubqlWFhAGzkCd+zWfj{zT*2QNK&!Qk?^c z(Zb5aAyJCkJ4ESHq=y4S(!QLT0FkqYG!IfU0zlLx+BoY(FB&TbCPIR)KAIUd0*mhdGSMD3cP2UOJmhcdcRMOGs*4@7t+1Oz7e;2LqQA$@fi`kzm0Gu znru**Mb7livYZU==Cq_HzTC??z~Rp&k#$U5FME>%*m)OasYm#;zyb0>oODa@3<0K6 zWg13dp3ys+9cU86;_R&cOe}^Xn06LDv_M$C)wP~cNWk4c#I7fxB!ys5f!iZbK~he< zAUE@snt$F73b_%xFg180qetLNuWX+_gz3Sq@3lx{lZ-IWR|-~y61jyW2x~mXu9$gN z#M-Z(2b&8mh`U3;f{ULVZB*f5{{RE1X& za`BjCb@N3tLB4mRJ4W_K&P0%B^|NnkO$A!2eJV~MyNHY`It&LDA1Wxf^pG*BKHHzk zB4T>!-2+RDtr9b0vvdB92n7eZ_a9T z_Cs{^6vK+1vi3^qPpFROk5dwpQ2HZ%MtYfOqr_t!o?}(?mt8xjDEf(lXG<}=M#O^D zzwKrdpk4!{XJrTa^OFoj0oWUb9l&q}X(+?k{Kbeod56OOuPy^Kiggn^0?T4@ATs)hRnoIGqYZy+j2{!(VfRB$~y<`Jb%y1R=P%)x-be${+)$vUjXzn0bus8eY)T zOn^jV5v>jW3rN@2KfCmrU;j1x^xT%~T5)K-rXfKrR2%)sC8Gx>DC!C|m2@Apn(fUj zkz|aDj74XAh?nMO1G_*D{i>y#7rKspYC8nJ@+tVFZsKNp zPM-F`Q@eu*UiBT~uOS>^kK5 zBF?n}ewW~lL`IJgDFN6*136JwS>BS0PJeRfmU;k@ay(!FC~=qvqP+n{ClY9YVAcZJ zJ?;N}q!)`pXt@^v=U;BY?*0)I4qCSuh)fd^zj~t@PP$K%$HL>o-W3W* zYhVCP5H`RVVR=kSu~AdkI%+L%c&TdEdGEV$;2%q22CY)~r&P#9?y&-RE;R#Vn5;34-9_I$t|OS0&oZxZhHuW*6eN3xXe1&1ZWcaHW{fNisT@tCm;ga+aZy^ z%_uTV8_FgCF#`GxwNgg>5<)QteJ!&8v^!Z{lL@T?>+p{W5Be*z0PHA%HjiP@5Hm5B z{`@AIFv;*75~novboaCC&Mn$W4&0Lx9Ehr~k$4XgLtUF%f{)KRt0!|7B~?uGH%3 zdeV```mi<^doO`^kylcg!NB7}OXv1j{K@pV`&we}=MGb0x$VMbA(V%yYIoi)tX&e(?qvjNs3CmBpWbp->HqEvii*s4U-L znLva>d#bv*xL6#mRw)giBHB|84V$wT5fkk6U_L&NAcdoMPJ)&!GF!&Vg+4v44zDwE z)l&G;lcXKeEkB+vNtkz7%l6_>_=ex65cH!5*YizJI%sp*HoKv46pXDpKe3pwTGRnh z5bWs%Yp;;kWtV_k0<#0M#G(6+d4D@xy^XfYtl`>uYv;-d zUiQ+VVhJHixCOj_p}XczAZ$eWo#+oM*i3kLR{eQO(A6OG;6R_3YN&|GkpY-_f5-Lt z1Fvn@bD!qb=lkOiM>L1+Rl)XRmce0`AO3v!xc%biuMa=1j&bkHR#zR6Q6upa*AXVMn)}%p1(Ai2_UW>=CqfxT|3@AM&f<_ zoENxo^!->m|CU9Gpyj`@pxva!++^39#r@BNFH3Cc7R$s+_P9VTd~9OSEdWS~SGI!e z3EmvtsdgJ$S^*s$KKGCsGn&DiJ&nePGeb0cdTrb|R;r zt@@_o)#*+8>t|%70f}z6>x993gy>q>%hQSO$P$sy80$6y^?2G9#->Dl!cVuxKmglm z^1$f`2+gm~*`D6eX}e(e3M*jTgfg=_rd%G2{j&8MC3`oD8;j%wjIe9v>`{>`!Jtq+ z;6Ef};cFnlO|a3tj&q;VR1-DFu>RNB>UzgKd`i(@JClh`*s=xlKaBmE+~;SMThTb{ zb&=W#cim_NX=JV;XCLLxT!M6d1~$EhzZSdi zr}6(e-Z2QF%FJBsUzHAH_OAPj@J!$wZLU#ms|3AzemH-kvgO`Np?{DynZ?QB>l)?| zc|q>T_q@^V3Ln!|rrS=x@ys6~%!2fVlkN*Z8$3L488mbHjbxnKy%|clyr)m97ofoVaHg%@@cFgRJh=hC z8_YU~gePI}{`Rf(;Wwj7FLw#=jW&wi*Y~BQO+4#%) zs_DSdWY+@0;2Yj1wpaa$4>2vLZ5;V^lR=kNRxe~G7y`6AkYpB>)h38)_)}!J(EYB= zvty&(#ONM7!E81lrUOzoeJbxSY#W9Z?`F&nl0eNZ)J}=ZW5u9wu$Nz75yRE(-`y3sqVjkcIn5r z%}##e@ylx&y~6KQ{-MKQ$$(JURLhBbBv$191`Ot(5#!{K5zg4shJkrYA2DNeGJ)jn z{^RCCV@l9~74h!((A|bFCfqUv4ZXVUFO%+i*ycCS+s0j%Q>JxSP@e}v$NzH>=FJL?J8YPHwM~${p|?) z?DKehG{!|E0ga#_jsH59ldtaF9o1v>+Smm%JodXx3VWjG^TEHq#Fek8==Tz3TusQ% zu@4pqOwS=?_Oyh1xrQ!i-Wi_w%^4=wq zrRMCWZu_waj_lE~nOM1;S2~cwli68j|NZc;-x9mI`@w|NBFnaZVTd%I#=9jCWL8@o zB2}Vry&7Q~9_#$julrh7BU^NUY>?S_&v0@I_4OJkGL_{?FM$=j^!i27sA4eCd_Edl zeuAfZCPhPXnh=A#IY*UM`$l_}1uPJ`*hkEm;{n-`afk0O{?&@hOpvJPHxRnoD#`tF z!%{tm3Pl-_;o{59Ea9Yclc#K*J7Jryvu6ikcN+R3n~_7#FX>tb-bp^3^nLdq#p?fq zv!8d2oD;mZ-XSK{*lIr;=87Mp$;kPe+was|4cGGIh?L^IH1U3Vp|A~!+;NqhTN*8>`^KaJ(Z zsor?;v&pLbw^PAEm1N|)0$P<~xY2&fQNZ#Vq;5A138A|E&bjaMQ`i*yRsf}#-ivwg z_cD`%UozGXXbYFpvd`6`-mhF-;9WaVE@waW)y1^@qbm+f10j11^)j@c|67~5<|qRVdb&n^}^YlPi4ZOCEt<13xaex_@*u0nk+i2>YNja5|m z?^ym$2aX&!*q6ioGhu==HSztDl#aSNw?OHD*_3Hb%|uCK&f-xGTizNkP26kjzcfyk zrC%--*eIx}hu?O()N$9k=`DXXAm_3Et`HdO56A5fOkpsEQZ@}6qkOF3Lhy*Qm2Dg{>XrJ-gPuo#n8BcifLDl}lV4jDKA+=dNtY*m%kE7gGV`z)xlJ&>6id?H}#_ z%0DKzT-@;EC9hp7%S?3)e9<1t_Vsj4k4iby)tr%cNxahbc)-6fS`2dqIdK`Wdc^Vm4V73VQ^R`0@_& zQDK>oF@WtJ{2#WT2#pdeJ}Y5g9z-yKl0nj#B~8@C;B`7cVb8ksH|}G>raiPBK(9TMZ&uz?UV*#Snnmz#78*yk#g(Xtuq%4IK&@|co9eqI~I%5bV)e%8`N z@7A8PB2(EQD6|we%pRU-tKggTk-U{idm}eNW$qGLnG)*Bjj=QlVYJDbQLRI!S-OWA z05XOPOv%~+CHO~aw67e+*tju9?4V1G@`9O90y0^jfGB-EvP`yOjp9g!n?&5C3i8xAtd2yvXbTFOfx~Bhw}X4=87!~h z@}GZIm2@K$5Wl5l~&Zr0ZU=~WLEd#&3vcW9JBgE9~CBvUx~XR@H(Q4Rz)j0`gr zWyid?;ANC>kldDG2G%GVec|}|$mhHmdR{PwP%DR>&^UV7>`}i+M6{e-nCxkP_k0W-c{SFSHj0iA$*iaJ@49aFL>7ZVxg^Gx@0- zJJT8PTE~1=Zs}CYQc~gOi^uSgYbnWff28Y(>w_*8H~vfG8_;`yCr=iIFCfxaL^ky0 zQV<^jG15UMx~~-b4>9`61+CZYO)om3ucQ5hk=+%oYA_a{Z$j~(qGZ=bDx;K9^UDJ! z7_%jNZFwAs;{``4C;g*v4 z&FJkHm9sA{K_4li#9VB@e%VAT_LfJJOKqd|3pVqoXaL3}-Okrzczj5kX$v#2vDgxe zqr=u_E)3}+d(xW9aQjyYIluqH5@VcG&+H-Da+~;VL9F|8MbCHlrk6LoGHns_m(y8z zXK4v?V1m4v-?a@%1xa;#kBs=4=lNQY*-Xywoz8P0x;PjH4_zY8nX(ati5(t|SiIP) zOqV&l4BspKteoQ-!cJC!Ekyu_af@E(uUB|DdD$}j3m-6x^HYCOFVr{CLneA7Nh06} zBEl>4?@WLzRu!D^>~@|ys4-23EH?r?er&YgzV7;lwNt@+|v(R)jMD5I{Mh%z4`E#6%jaRXho(fvP4_12Zk^X;M}Y2-*$gBl4P>qw^!uv zkK9O+L^2MpWl}KNSmfcn-r{X!_J7&<%~|hhC;iQquw(BB6(Uen2^9nq}TMc8lr%S&7cIPq0wbDa3}UVc&<2n-*Q{x796u^gyo z#>1L-N`N-5l(M5PCNE%v`(*sB9~;(dN_nY?&{CPd+g?w8u53-7K;56Z-&* zMZeSXhKJW%Vw8)dnj|<6Rr*6_pE5&Cwv*OpEp+9b zytn{SRP;*?>e(%JqLgU2z7l%&$2C67&MV4R=eRR&E{uRpownRjAGQBa;QR23iwUBS=fsDf{YLE6hKtUgnSaF6j%{j)BKNsoSU zW~hV%Oe^nXiXMo3rM-M*6?}+`-mGsa89^Z7k)cpEyMH)~%%$U@Nq0y_U9Q*{VKNO{ zO~f0<68}=epW>+NB7v6Zu|t2WTbsK1&tj_6!A|8TWC*sSZ^K2%kd88Y)94pJewcj- z{Q5%@x;~CsRaM>uysmy3Ua{!#<7;h&LDT%Z<;K*plhiRcQ*4{=4V#_&lxC%14U|)? zE&0N#`3_F^Q`8@m3_9ivIEuH09m&zz_yC==^h<`rLe47G-mEB+Dfck*DBjDrdg|gs zJyHXT+r%r8Q&EK1so5Ah4iXXXi9A}3&-GXTe$Tfs(%U(A3R0{`%o~~m>B~LmlNx}= zos+MbHb)? zZ473`27Qn5dLH_y#WvqlTB%&no`i2{D|fK$hYyCHbLGQg#mu(Df331~c>JoHZNt@T zsykRmIfk4r5r}55b>{mb>q1`6nCgoFpPJ%fM_tD6&`}W7W>d#b_U*=hi0({I5|(e( z&83*<&Qr(74#hbUl)~?;H7#tLCd_X#A8(S-oj3Y%@;C?M8Jw)fGhM1QsE}UWggPpx zFOH9qS1nA%Ig1j`QDi(B$KeX`EF$xc9FOB#QU}ICp;G15F`^<=FP$A?3z9yp@y6}I zux2tDcPI!8Gubt>-hTU6|IL1t(@CSTMJ8ai@~=~FosD+@E)`Un+`}J~2ed~Jc%Tr$rcTs2C=u#GW5L04rQ;dpm!7Tb&_%4$xZ~D*6oIYZg zwJ__Gg(~&tRj%3Z{@XuxU2dWvN0x0?ivOk>-8(*95GnuiO}E;r-c9`>u?^I_JTl+? zuorC4TvZ^OE#LvMlF#@Rfa+;;RFB1EnGMp!pr)=uZjGRPcp3eNaaMWGxa5KBnTKJ7 zE7wz#61&d3*x?Zh{Pm7b8eh4CyxV@1WIcYMGT0QN*a;ialD-m`I=#?$DQsjO<-)Zz zOY@1K6>uo@}l;3N`#v4JMnF9naz;^0Y%*vB!~poX>T1PKuKa?hI8 zNSXBXr-o*zFQBiUS~xPkx|w>S-{M^`@>Q(x*7gJ>k$)IxRwy_tLW<`dLdX^jCB_nj z%`7^l)&-x(+C-t3h8BzUQl_C_SW~&TNklI<81eh#JBs+3a|MQ#03@jdaz-5#uo`Mz z3y0IdtCg{<$Mw!;>k%7nuimg$q%7Y&vls;J9Me9-R3`%2I|kIYCV$m69Pqj+jnJjW zQ9;@Wh9V3N$*%IntQw(x*sTv04t#Iy!wi2eCK5_c2L+z`PG<47s2Q_=MI;Xfw z_54^w{!ym$ONX)tn%=H3A5=O&ShIM7*OBd{=-(f*JC$(Dm_}@3piPTl06` zv`oCOdXy$jHeGtVy6ns%wfoTcI4BlNKG~wq*V@hZZ9|Bgr-ziU0sZ>1@NsLH<*+~h zd7Q87(|6im^tH3*JWEN}gTLRqA8Xf3h*S-myOJxD59TxYbz|XCIk&Vn3v^le>??9a z{dMlom=X2vBq8NnK`LZE&W>>02a-YwX8V8|Y050L!l5xZy3zIQzr-Ve`H2N`7g+;f>>YTDxcai>$? zP~v;RT%6sT&BXI3b)C1wF$?>~-5}^to?8yDMaS@p|MRQ+0jjQ(ZP`voUCQjW>6y3?Tg*dboY~)1 z<2LbC_uX716=W~#=ZmZKNQSEq8-(5~&a1bR1XpY5!Jdv^nw@IDFHb&f`(CYl^IR4~ zT+MK}-rl_S?mIp-?vnid&WSolw!9HN2Sw^JXD&z4+|E-b+oel)f3?77Wwf zRl9ao@BPRng#jYq3TH!IoF~E6OZyi-;bS4+idKr30eA05w@{ytNuUW$Xsx|A6o z0GJrg;vJgh6L&rKo^HL}`;_r?&fit-|Ikk-6XYMO9a^VD`7EwbNjU_B%xu^FZW>?Z zjnL?GZOc_N?MXk^H_p%LEcTD!KR!tIp%{u2tvPtCnbTwowt7zJ{?gj~Id){g?4tQn zUpH~}W;`U#$X?k9V1v5Av;{Bg*1Mk@r|dG&hcIbYxtXQ`;boESMBNX?s@G#t=4NP! z3EiSz(m6k1S?E!zQ5c~rv|~ZWgXTJjNM0G(u#k}$ z;=WU#o+m{P+?SzBjl#9w?~&RAuQAuVH^F|Lr;q7o4<`@eowaHiTm!lGgWvg}u>7hu z+ z!bT)gUY$h?GLm^zq;IpHhM{dL^*b8{*Kr71SFU1<*^0Mqe*a@^T~Qw>jl4XV`Pqs- z%dP#IVs7$d1$SSp__@HsTk`#o?c1&wfY}BnD|*H0Z~h>jYF*cxY!U9cM1OBmp%*T4 zztCmcgvF=7!dpPUlpjWJGb#2`7?a7<`+R$rWZw&G7=eFqn`m3BF8B9nh<*BM6NMOg`AozJC8^@+i|sOv4t`8k&ptZz4W)&$zeg% zCL96Zk%jfA3rk8^zWmfbPIvub+36f>4ralvJG1|NZ6xdQ8J7Sxnzy+6^NYv5^|Refzoe1h&=Wr>g@Rvf%Zjui*K{LbLp9im6G$Bd*+Yj!l`NjY+Xv5C56Pf9rbd z6XLfbMCLZ%3eYm&Un-WXG-}!`Pv>&F-Er7_{bl&l1M)Mca9Panq^?OKAO!nQlXE=W zcHW9QN+$SIq3vLhv=sye1;g&%7Y$Ad%=K^lBrqT6UAP)+kDws)IbM;tf0aoTBRhtN zue*%qv17yB|NeMgBmekx)u7|Cd|`95n^xs@usWqHdFO^_VNhp>LHf&Om?tHOP3fyc znyGWhxZbDGj+Xu;?Uw%CO%c^_Ktid8=1A8{9FHW?Vh$6h;nTHr?Meb)M>%*=4#u& zDewu|P{TaovYN@k#z8(_TfO@yDmtyT4?x8ErM&RYN+wu|0uAxkgS<}PlzDI#<8yvF zoNN_kYO@M^{YVnM5O)S~PJ&k};8(vysQZ}f@v@~n>GQ7n38$%r?>G4LnP;I85K-69 zfj}9L{0!1bY*B} z{`P6s(!S6~E}z3!rI^~TzfdNaMn;N?DikrBih)v2w3(Vg>ILa5A&k@Of_DFVKR=2< zM`mzpcg@iA+^N>}MBC}2#L)0hFY_jP2OeQ)?_KIr!=I|ASkddNZ8q^wf9q$}8q2F& zR>g&A$FAqrbe?-Y*6A>IGX_`C5%H{+M9!A-f>rNi0O=CHLXHsM5K!4nyp z?#)o%ka$zxh9LtRm<#vwc2FrwrrQ+(#Hwh@Tp|+iUtIL|N zOY^hbb7M|#ntSt8-#!<5)xx~(&HTk$62F)`L0jEyLX%^Y_>F@k1B2O~!0=fKP5HFV z8pVre*eqZ5-QJ+C>3D)6lim=KQBcSe`?{o7H^Li>uG4IzW=SAa~Ku}`44D6l-J?ydHaiJ zi>}=6xBrEq203RsG}Qoj=OFHaD2Er&O^s>xH3op`GEw(^92x2y36!M_?9p*`dyH}I z-J85!x0&p?3v%py`q%E7ZDw+CS=FN$$-7`Q_Dg@Cl`#^hSB#Tn2WyzQv&nuM&hvRF`$>9Yy=7Zy<>QT^n`3_%+0jx zS=BkgE1m7npgHr93DA`N^l(J~g+DELG&X9|Yd!z#-+s15XPLS-o=tD}-8-FJmKY*+ z-S@Lr6Tj8fzTWP|I29 za+>LQQdLudtWB-X~opHRT>`Yxy)NMr3zo5IUeDz?;Y6-g2WE~X}0R+ibn)_n`G zj=MZorr&6>03??|rmTU)vdTK7-0_+k8fs>X{MjG)SvqpHG#p}2kKEO`C581*Xko0= z3N(HMKWnWV2UXO?%wwir`dTtL#0!Xd7%8_RzwDD!^$H7NPMf*0&3X-fuS8`8+TuRU z3=|iQfVd%@@}m9s(Oe&?XXx`l@s~w4sC}HszdTlEqhq_P1KhPNvBz*45mz;R_c9{9 ztMLj>%lAIpl?&O zhj!5|jq^{b<)^kLV$~ka6ymZ?uHUMA;yy}>zfinQ^qsBTJDKMwT9FytC)DPAMTrcq zt&qI#oFa3L?S7d`ZNsvvQE2`8%OV|7&gq(r5~kUBEaZ@>6CY2T(7Gx^a1q0 z|Mu@96ew?NoYuE?!))UaQr5E~B;XLk%gS+CZYg_I=aby@X*UjhWkEs-;65%qfNe#X zgh;TEH#f-9L1R5#j(p;FE+xMs^SZJ6Trn+2n0gR2Oo5JGn=Q(TkmE-@k60dBY}<}9 zrG=_5gyR3*q>r0E7R?uy8NO48=b=La9mH=zd%l&7i>+j&1U#CD$MZ2tr0>lp$cX%4 z_#p-88T)h@lk_6|iTXS5Np3)lN(V#u(7rSgBT@vyZ7RfiNu4yEQS1V{5*}5|*~wkR zjbfpvvNqy~djl5Hk2E&lQ(GTAj5^c)HL^?3aur0whI^wc#@#5PsOWr~ z^O!0yxHr;99Q|ndF3bS=5OjreXmCdGS?DFQ zm;kqh+4^3LNG=<@nRaPU(4%vUg>~$$l9AYC@obw{6&6AwM~xD5wpsCdknf$#v&cFR zyht(e+i%0?9O&q`-D(7G&s5#StuI%zaZj#FIX%Ct)lnxR3(?-Ek1aIc(3Izar>yMX zw4Sv7`S)HH7g0-W`8!wzVY;!Xw|fyto0(+(kLR6B-}romDBqfyg^$K{tA{Lfz(5Gn zlT?WCdhMS17?zC-Qnf;qs~hy)GKFM6!J67d80;yv=*5ER7ugnzDXubL#D3VF{e6ANl4S)<`Y$S~1Gm@jSh2q~J?Di=*6x}=a)=yw-aoKYi_U%Ue4 zV}3-tqL&#)<&x}4>bk-6fTD#F8Ny(gq zFOB1AvO~7m3o8AnIQR#UR=?}q>U%fYUoz#gHHC%UPcFTEDWY}q8Bbcqq}TG@YI%Vk z6YmS~b9mV%G1A^Nd$902eg4hO#H?3uUwWJA=99m})NrBQT}$n#F13T7O6oPi4NpY{ zl@34Sea7>bSlkhoIGg9ie;@ZNUMMyzC`d~FKdvw1Y?JnN6H`Qn37;N39SpJUqFh!r z8EhpHv7r6=H7+a|3>F{XaWs6?$`}LZ^enDQ*SAqYqSiGABr@ftVR!@_Z!MoB2z623 z_(ZJ9WX=)Ks{UJOm8vTv`BdhNxzUS5$N4qkhGmk`79EHBJpn-q=-ZaG9^riuy6PI& z^Td6|&L#l7n6J$iy)XT{?%+=b(3xb4 z$S=PN+T@l+ty9=km1v^JQU<)PZVH0w7#N5u5Wc@nf^NYBuask?5Tle}C7Mmi-4fRn zRqNA`R%ZN15JS8hWtSZ?_brU9ZKp^jx{?2l&c+kxEs{|9{J`e9)3BW zi-R;~gb+n@RI9I^Wk!V(E%`q5|7H4zB#S4PGb6yriYKIij&JBU)+yg~0Rr%Misk4p z+~b(bv*1ZeA$GgZQzo*YD=n&N{Ji#}L%AIDz=X8r8I#$DdB;Ha!7?KlztlJOJyrnx z6QVR3UwEOoaup_%iv!~b^&jE(T)u5+X;&8D7Jtp%`9E#ZQ4%9!wB*4OZrDdT`%IXf zYA~5E3~@kT0+ZqA3WtA6(@c2I64~{Dua5T!#s$6VXq(&PBGF}e!7w93a=sN$5VbCS zc07R$RGnP9><(|(8TPsNR0fTUW~4lp%4U`Zz;gl!UI4sHK`ZwP>3#h5IV8%jgEkJz;CJLyPKo9IY7+ialb}nD}l;z;Y!XgunlJ} zAPQh&{_&{vV2psDNe1#J&B!IfQTv<3$fOK>I?&eCp16lzjL^@LU@tLbfBUNp<2(l{ zfDie&dwSivl$?6_Ziuhnt1QXyw^||*_}`j@RAur+UH}IZpoX{AUlJZDV%DHyp;aFT zdG?vbSUDr-AR)e|8^ zCI=zn_u*Vvy1%Ast-)TbCPdw+`G(e7f@CpD()bdh#lvOjm)}e8}WRu$fA!PC) zsJIX!^Bl^0@OM4bs;8XNwUK3dAz^$V>TXx zD?$ZFdcO%ON~O>BtsT!2oc-K>KDbx>H<&=$15D{q9{{g}&4Gw$YR)kI0p668U*dr$ ze1>rbX2AGas{}>aD~>blylmabx_vA57Oqdq8ZrHfvF{swaJ_jwoQH~@tG;78u~xFtntGt5K{yjWuIw%`8yE(ZG8_OFc>z2Ai78N)7rxsF z7On1yTUINg4|X%pV(yru->QB<{44#xQ0k9CH~2Y1JAMU(bG<3bDFAVN|lt~2K}P#OAxwpu?jHx=o#lUeA;C7cVS z+F~qZvH<7|^xHc?#{3Ei zjTAVj!5Ro6XY-_E0qBC%Nu&lLhf4JBHY~Y-5QXj>MSx(UeZlcHGcJA~7jS^q7jnPQVa{}jWu_gI$`@rS#+=$fy44Of^ zJ$npTS|Tro5169&n<|bVf>KeibeR@G|y9PxWvpK zjq$y=FSulr9KjfkmAjB7qYG~F&zA7|eeC-jk)@%y@g(&#RMQ5bqAaN+`*@W82y@=8 zZoWMRrBG>(?ya!UYxja6-+b&cqJ3a5yuO!vp56|Y496=vZUr4(2j$L}M_kB#3;@ix zhoAzmY)h}W{74OqYIh ziu$Fhv~;?vWshV%Qe;2N^|5X$4=$!^Rl2tVwX~qSL<094gJ4neZ>n7XXRq~(k4?j` z8I_QqrPrQkp09edqtjEsgLQg zSB;pq4aS>^TahJWO7TTYoAS!WT0kOUG8T|U5^lQ#wR z9g%A|qvu3Ng?75>sAjjpKum@2J)?4#V_zhRL7j-QZs2*;sa3wTNYaCwOna}7pS%Ei zxp4j?B^xRWB2lFSYze>#Is%R+sjn(2&+*J(Vyz61;F1F}%h4FYr(HGlgi+{8ALaxufgJeJ$u- z=Y#I2*lm%buMuRZQb!CMj7Q9&Zz2<`d@-|`tD1Xr2mfsZ<1Bwr6Wt|501OWM)GHta zTmoC6gQGHq8O)-%YGU_=Pv>#|SC^&`&E}m%Trd*V>N-<${1{#!~VQSCjdQ*uqi`>(3=UxQ+fD71OOmsY;xwyD{ z05o|FOT%5&V}*_w)KL%*XiqrdDL|K_wuKZ@OFOlwQSk|cC>d<$Ho#8k+B}Kl&`ZxRqynduckCokFG=Qd< zxJ>na=R`ERAODZ!U%Li;>Be>>kzBqt11NZ%L!)2fIU$80&A@z`Yd@~S#)h`gNY)k_ zx9%IazNwcNji}$)Jk8xLy3w>%>KSlmV)lpw3cZ^amRRlM$xn9b6JB+r4;Qziv)|L> zSOR}*P*9FM`Qvliu|V#*0bi|fKklc%uyjJ{jU0|aFW|yKq{KvaUR4Rp1wTu}kZ{?$ z6op+uwq8m6$9am(iJzgPZ{X!kzKCwDxD1J>oER*#OpO!c7sXan=TE3BNK0qW&o2l- zl>~EQCZbBdi0GOe-c7YcXi3R9eaR9+n8!Lo_l+^{S%qu8(}zbeO-QRBK5S6WxkbXu z?J+DQ>$E^%mX^ITv$f+AAtWiIzR;P|HnXs0h>F)f<8LQ$RxRMBjLeKdC9xHWi#GPG zE6=}5;y;_dkQ-S4Tl>;BH^86xfH}q?0%ANdr4M|l-v6Y>-paPnM((#<5)cR)@7CVtV?(=>E9lL;R+!odJUpofuEz z7WQ7Ge*+bcnD35WS)aSqeh!GNYIUeg{4*Hl&~mtLb6FJy{aO z3mI)_i}+=z&`)1@_uvNIk#_c15ROdnHA`)amKlw$rF&_`fe;d|m2~F?fcJ1oj91ag za(O8fPomLdoOO9Gjg6W9VWlY+ejywxJqe#2;l5z|sfJnS39bGq^Sj4$1AUT9E$}g( znm*S`1y-9Uyo+GH6Qi#C<&6J4e?ixb97m2Bt8uwiY0_ql`EOOo5vi_s_ramY^Kc0z zRKgjflsAXEGvwR)Pvu^5h0EC0v@t*UYrd;{K|(F2R>h$h()~E?HK1JP@Cw!m6P$i8 z?z*QUzV@DF~($60JhK>d2e4e)A5Bue(OIP#!4f)b?OV!Y8=2Q0HYg%69*ca4;q279DO&zi>OeqN| z2A{824&Tg2hK^Dh6JH}8k%>U|^G-fA>?Bk{2qCx$HD)(I>(o|&L+;+1tQ`Bnht3|6?61h@wwsr$eBJD0*E2XWJi3dbk|v^y z{1Y#YeV$aBZiYPfUr`(0ZWvXM2Sn1X4`1GZ`e?ozXD}K^^h*5{n*;u=ti0;tR^m^G z4+8CKTPA}era-h3EYurG05>_p;hr+PtEI$4vR6cv8D}O%LyUmwfKc*q3$fBD{4%@b zPkuXXbpj4BKwlkei*vm$KFP^i(n#|kmaLIvb84CpUk6NP-xfJMj=#6>xL#;_X@6?u zW+BPqG-stP2r|J~FtWSb&Nglt1u~WdwA<+4pU7!7ya;5d#$z*csIl7EY>v|k$(J|EhsLdN?uh?VnbDv@Z zL~Exto$RrM-rqjymYNLkr$MwZa-?*KC2}81a1itPHrXYHXQ&2M0sJ5l2Nzrt+3k`I z1u36>I>jW8;T&@@+<&CQY8N?*4nIvPO+5!SIR`iK0s458BnZFdY|50p_D|JAc2fQo zAJ0-Bj<{{w)B3=PMW&(Nd8-C^!v zy0L%6QSoyXA#Z|z1B7HEEu2JDR`kucoG3BzM~x33tm{7h{s`Dev!RWG$nN$fa3EYpCapN_T^>*q zuvCH24LxIt-;b!Z`wT_`R?F@$*n>lGf%koaQ~^1JsmpYCLL3YXA3hF_g;U}XCsMEf zZ(stW^UmoCq{j3)fy3@G^=+o0Oib0L-lkP6W@LE9$jI&}*C!RhzW#)6bfVBQ6dYm; zQ`U6HdG`iYdBDeJggiRrzfh^whWV24a1W|$tt-xgpkH1*5y<7Zk?#=ZW${_MMYpU@oX}S7I(c-IQL&dk&Z!)8S)Mu~7kfmxzi`oC*BILMXa|Mh zfhR-8`PWyQS~ZdbbOgF3j{bBIP(l8)7=ch$>&=WDMFFMYD`QHU`9D4{Pq#~ehxazU z)i-(ae`BC8*SxxSxrK{dHV5a5eI!PJyC z>MV|0Jhxf`#@%J90M4Qz0)NO*-z$Rq-BI@Rlr2)oSVcu`FDV!(rh1D|7a$~>_t+Z~ z6CKqIt2F1^(0u)y8=)EdLqOI&W+LxwCmM&yWHiBb>2XMO@`y%>wCdo`Cdiv2-3X!s z2mM6+pRLSUR(|5y!)tXCsI*_HZ^wAq^RN3RC1pZTVkU;07e4=c!Wx&8n^WMWUy&}W zEUhPSWi$!~TO#oAj{PqEpi@(0DgWT3u3>CWsLis@*X8`SmFBZde`&Er5kFI*y*_@$ zmMt&S+^3pHbp`MLay2wgF(bW+zKCvG-MN~|y<~1Dh>0D>piBJYpyshlS~qyW;#rl9 zle0CV4WD{bAa!3wRfNd=COHk9s_4j)%fX7~jocgY6+$>{?KbRp`?N$#_%9~L!w@5;f64*yqZeXOp>dwUQQ9n3;=pZC;+!b}+Yq~(RXrS8MHzZ&+ z!Ug^%T{tNBD0}3l!*g>hiv6>`W+cV&;pCpKbG!``MkfXm>1`zcl%F0{DU-n$dD!Qq zuX*rNm3AZbhobU#H}pvM&(d92##N_;x*AodTc>>-vl^=}0{z=AIE5@%9o>Gb9pTO4 zw_DZ>{G%TNgo*R026{Y_sS2I8PFnEjd&9!C)CiH)@B67dyoqkbNzbv88O8H%g51FD zTa_Z(23s3s=8TRj=$&B!TiJ}y70XG^NoYGLwL#jh=j~KoEt)#^Lc?X^Ih*uya;gQr z1EsB8We*U-yb9Xoc(6E{+I>LR8J6qCL5+l?nHCHtnZTAUipAWZK{co|rj5>x7%Gj5 zjc7T76YrC*7lIA?F9!iro({pB?ptCHOPLV|`rO8+$`is4Jv+2|0!4aaC#`CY9X(Pca~?A zS~HR3hS&~&=ywADi?1Gcm7ZfS9OJNZU#h1&vQ*9h`KwA#g!sm(%B`Za#V|Ue-bXsj z5KwPD{1!2NU^84N{Wz8Bp;9LdY>Lo9a|-VjYzpgXYNj!6-WEI|=)4GYA|zK|vkl_z zW?et;n2F3q&lT9j*(mb?$BCn3`AhY_#1ErX-Sws1;fguah>Fluir`#|O@P{!dj zno~wGa-OV8><@0fFl=+OC;<~m7{Pf-cb>7HjDWTzp$Sb{(F#TpI8xzv_!N&CCh)Q5 zgX87Zokwa2m=SkQbf)z&e0Ee~0X&M5d3TB*6plr4@eI@O9aRmkusVv>ju;lH+&<*bMA9wk!Lxm^I@>tfO4-Q?1OwJZGyy0%H$dQPc>xuLb zw0^}6Fdrguafgcl!6?Q+KDQzVm%w)5=`ax@wHK>p@SS>WVM=`%%#Rn85%QUzV=}0r#oaNH!G^R)$((RZ zH`ql&#Dhkq&GAsqgL5{;?~&khwpD#)7L|m}x5@oC&F}Mh;`isMaxO8&TZZBU&U3Dq z)B-QU1(+(^hvMS5z>aX8!Sa1J<`8Ev%DtjK^@5;|={0^!XclFzv%CrDzSJ>-xqaOA zg6J%>euHRIF?UqYci8nV%PmYq>Un>wk}xI(IHYgDh4h^yW);)`Eu|&4wquJ|-X8sp z%c6bVxP7KW(*h zLzAu^9)iSeKials7}=K}0JP0Xa2b%lOQmW2qhr(`zsmgCp8fsTP#lVrNrntQhp%xs z{h}+;ulV??_KVRsgURY}g;As3Y}xkss&5x3e1JC1O=jI0tIWSM99gwTbI+}T+Y1!B z2yy$y0YN(Oe({;8jon#hoyu?<5aeSI(pg$^cM*8BFU6|aM;`EWDxJGIj`GovP5-3Z zeqI9bDs!Ah6-)8%&(wGYLk0VHg9(2C=?R$buK<^iQLYny_W(h`;guk7o|O9h^x)$! z7elh!MK5*-aLs)@X(D+j+s{BHSsaN9;=3?Y=`ieT>tOM$+1|_P0{I7?1b#e{oj6Y% z|E24cvoq<#O~V{q{>NRi^Zcy)H;LlOyios_hxOYN7u-R!5?h7bQF59nml3%;NwOZ# z#9BJ?WNzKZ7NFnl_q&I6J)Ru*e$4eEmTPFLAX-rk!tQwh^a5=o`0wE;5{scpRB5`a z+u~l*zh%Pl=^v9MhBo@hpl4Xmm=KO`{M#K)F8~m1c*h+OaDfP>c>ER!kMn_zYSTsm zCQXC9HV|HrrA;wR(YU0$m~|QKb0O=6@p2$Is^FgK`_33FMhLJJk__%TW7JF^xjLMnPuQ0~9xbr>GNdqF} zvH@7`%+g86K7G|Wabgfvy#5*aIv&L?T9+a{^o(?A6z}O% zb;qxLv|SZYm>lV|g;BT7PTV6^*T|<7La4LYETOK^CKk&hmD+}7Rbh-b+GM#3U9pTe zGR|!L-%`EO7<6=ucgbh)da04bZjA?QG2KU0wc`+KSu6$PDTrRH}F>!%0x4zTu% zrUajQp`7v4u7%0GMfkOq+lNBcTY{A{dkgnl|cKawC?vkGL zp^xy7vbPM0_tHDB_6s+@nwh&5e`!gfu(tYD{BbhgdyUyd98O0*$p$uv|prBi-O^Q>eA)kA)P+UC9oI=Y7|;x-}&fJ5VJ3>DDG=w#3QEr`0I_`B};Kz@zL7o{BID_psYLNdYF% zBVFg_4;JSQeG#a{J+hV~;s8G)l;M55u3*?k$0#EaYFx7`9ovqzLV$quj+)*Htg;p+ zZ+Fa5jNq?;(J=@4eGzuIg>%}>L>Ayni|vBZd)xj28b5hKPukmlY^MjD@X_NC)bo$@ z^-ACd^QA^uZ<`4>O$ar9rlOTa!-{{L(U5mC5T)1m*_)-3ZhrOYZja`Oy<^uDFj9<5 zPxWN{JeuUIV7E2u7_ct#q4P`?wjN1zOXXQTN<+Eo8M=-a(30mA*cn3bS;YU^5JuV! zZ-E(BVzl&~v_L8dsn`Dk{Gy0cSbypO*+yrtM^LUmsB&$Mjf(A$R9~_ zBGr|GBSHj!7rVbj*YLK_ar;~&C-}^8b;l(a>jK3|Ez2Ee$MIbvH8s{@o+sF5l^Li1 z*8D5g8uOY9ert+am1N7p;+gYo}stq?NlAJr#X>8@~7O^He-qwklG6z z`JqE~vb|n{*DKK;1(J*3K)hqkmwsDIg`d=1c2Es@V;|YeVs;aWlCRNo+Zs0%BLKG9 z!u22WICq>)*c#FX##^{sn=s^)9%H}-^-mP0MyAA3$rQ4t770lHR4+wHsZ*Wg2`7^+ z5^y9?Lz1RIEsWnWImCCw)h%MBB}m02=;nvpB}b=Xh!*eXz8`7h5Y4YeX->++MBBhI zKgWVPsXZ$R><3*Cf!QC`q5BwH^3DI9<3Sb7pK(oh14lIqeG@3|Mz}kXB1tzyondGE zKWRF*%K%AO?R6$hKZ-cDx6K+xLg>9A0D<{mrDu5W9j;CIT|z3yI5l+JB>rWK@SX^7 zogpsyi(eXsFUGp$st00!u4lYXp1?lBwa$5}c?FP%nj@3LsCSw#;&aeKEAPo)(hG_C znVOl_2|m6%6JwF?#i$0Ky57&?*UYHR9jkW3QS}gT@aExlXTc3xrJ#-JG(0jGNGXLl zNAC>RWUl+wbxv~%EbRH*rp*6Z5TfZr1MhnaqW6Ag?sc5i5Q}eXgf8gYs--vE_-38E zZ5Ae}8k{sqcnwCJ;%XiV$vWkW5Nn#ZXK-qViZVPjcTe4BlvIDKn_1WJzw(oRI%MhJ zo?ddscv9YXeDrV(JIXz>$SNtGGv52kk)|w%C@T5$)-1Lk`tWz3ly(@7n8TMD#7mky zbnZKT)!qaY4O+nrxRlD`R(yOSXbU`JKQ>nVFeCpBwtg(=)t??Ow5c@o6W;THy0`ny zH97a9(eGGm!BWV2J@Zro4OAGDr7NK$?|$>a#VjR9Pew1t|fz2-AIDXMlnIR_XwHAw`x|!xwaY9~!z9E6qjh>1y^##pOLq!MwB@^2uv?>rztmqtv^x9e5X`+MJZP6)ck2wt9$1e}HD; z7b#4{U`d9^T}JEbj+-*SyG@jdpvu}fvF9oBmlex3&y52^O!U7yBc<+dRX*tGSWrN( zI-m9{`5)oAf$C60Je$(WaoGihmgpFFQSj?;zws*Oy{fuf~@F?g<;=W6+^(pbIM- z+r=ain}Hc{pfAq$+9Qq?{GbD1dM2Nr7;Qvsf3@(Xm~#=}mohz_4w3uMLA{STpZhgc zLW6pXbXL?qAq^lkS}oxj?p0k`&>jfr_2JyC`}?mK6zcCCa|gl0)z4wmAe5E$Qrf!h z>g|u#zhfrzJ(FDP?k-_riR5daF8l!i0)hJkdBwgp)8x*)g3Arj>GSP3C|9yIYh*B`2gTD>fKUXjKYT+G$`H~+})CqOazy!d=^VCu^*t6`Tw ze}eCZdHi#PEjQD?He$UZ?7|A`83%d>>Z-x&gTPVY5)+mJ%DyC3y%Im==%0=eNjY88 z*>$_$bRPYk;?M)IqTcTy%UatA#6JLA+<)s3e0voL3699=pFtRaEFeRyH2gA9@Q_Ki z%p+J=R<;V1x2|utrLpTTA}t&hFex|}8+_B2yG;BCzE=s2w7DK1MJKvsa^G?TGWNB~ zY%Rm80_OtdB`kGBCf;kIaU>rXj67m;3(-cIl%6l8`M44{TvSC^|7|hDyuDL@n#`+g*2su{h`8 z_w%Lc%MYMj*t*Zl=ig)025{^azs~B`o#V@`6r~v-q}NQ4;sN-<-+WfXl%xVBT^)}{ zW6vw2UzIY29joR6@eD8R3~_&P;3C-G7^fZcN6xnjla9*>W)2g;gEwLwZE&>)@;l92;J|L6eAQ^yz+!CFO<{RzR2|(@R^y3XL7mMz98>j9u`xq@OHJ0 zPKe#wuanj|jYzF4pRWy_u0&sdTy9)({4`)aFu`I@5w$k%DGIOkk%(#{kehG1D- zp^>Jppk~TXEdxoxr9%dQ)6D2!!((cT#V{VR|F{?}VPsYI`y`7XBT0Omsv9BHr|JW3 zyxBnMD>a8?pxUO(4#J}I)&RKV0bD98^I6^F;okSKAfIStsHM}-+?N!>H_icCEvE=! z2R(4dY8oacdu!T_Zk46`1#_a+jtcsKAI(_e%xLQY8MFAF@4R=kb5JaNPY(xb@d5$> z>*i(w6Y-hSBWiwBdL3Bmid=goSv3S?_%;(M2oVI~r=0~eG`vNdYPvh%AirA_@CDj4 zJjX6TXo-33R-j_B5u0Ro0+7X zo%1cL4pfkD^3^{PN+U>vf@!!I4)&A;n~HL3`mdz4>rxET%9<|rM8OjJ+WKRfntM7N z`rML35+QQx0o;=!vlP=EfDmCr{eM#M^W^_`x5r2lEwg%g^y$W*1*LM0Kf53MzU#&6 zPdI-Y2;j9P19DeC<%EQarFWNF&Sbduq?|4mBllKM`Qxtt@W*<#J80l1#g_t*yP$-b zJl?Mz4{g^O234lIJa_Z)kp>M1REN}T@ej54qjNtGMB;6?nfI&xXI=A0!_EuolC?QD#pD$UV+c>b?qnw?lR$|_qNW;f&taVl3{R;pm&7^Lp z5@Mgs;BF|95D*#g2b|5o^HYPr6NBZ8q@m~09B{p1`{?45L7lNb#57DxX;dP=J9k@s z`qJSGfDa$94~uhe^D0X5_2%FKPst4lq3!FMH77)ds^sv z-rcw5w0oJBr?}pP6?%#Q2`NX<;bt$>@t^01Vlbm+IDG_R3v^G2zxcM@ZAEqLPH^@C zews1H(EDBe2((o2dQD`o&d3`>`_0=li4hP}=^%r20YXtlWR#NxoLwRz0-!*Ry?Tfq zA{QEs51Ry>Y#d1Ke*~I-9vms0vQ9JpB?V!g0t8uSq;yTZis-i{XMS-{2=o<)36j4G z3z=&7kAw&OfU}dZ9UTVtylU#iK8ZHPgu-rWWwmeI~ zbe#N#TT{IM^A+Rn{aqoEe?)LJ*|owWAgs%l7Tr4$$a9TDvxbxB-V61Y|lJ>y^B zDv}kvQf58|CgKe)dV~#l^a;>8zvv)&iy9ssq=z(kJ4}B>cMr@!!u@nZ(yU_n$l}1In*_S6COWz=Ei6MQ98pp9tTUc_o!%BGq4#PC| zf4YXEs6M9V!riujQ;D6a|MrJnZBuLz2PNFX6i<;(X@3G6)L8hKXn?ue|3lMR#?$@B zeOR~Y?wA}7rn_@uy1TokJEu+mnKn#!cOQlsrn^no!NL7=->>^Q9{8R3em|e<{k}x@ zNp6Xj23`DU+oR~02KnY#kEw2BUFaJxq_}wB4Pczqks?84L`Fsj^zfa=P8Zu~2Q|(der%*h`-={x$42z0W^Ef{8zd`%McL0|Mb;1O zR~uo=9cipXhh3P~aB;r$eX#d1)|(<2A-FrfoQNhH(4JMjN~`MLOag)qSJLXPS-qSy zzG-wc3cAh%XiY_BbB89FI~~S?&47?uhxvUq((j+R3{mc~D5+w-kM=b-lAN^4@?DSw zlZTeHPSPRx;g~QRT>A8OAqf}6L7FDo@+w@>UG*AptifI}i`c+dk{v+$TXuks8Zpyr z__7cG5ZX%{6-Rtsx+f#R%GOTZ19O3j?Sc-O=7t#BDI^#0!IWYWZ=2%oAO0>r1;%C9 z)0;>9wY4ba>>9n)$28KboLu}YA>$>fjH=GR(GN+-L)SH_{A<;>oIwo7tIg{owziJ$ zBT~&grN{r`mhu;yw8|C%sNx9A-CRbdf*arVR!3VJv1O7unyjXCK^Egcb6h75Gb|01;V!W?h|$~oU`-}$850OSDtmdzygq22Jf`^QDeoy((EI%K>($3%`^YB0 z9uTKq=IE`t>3md4G?Q#47MCy#yqyoS4RJY&wcuvE>)Yh>*RPw{uiHx52uH?h0{FXC zGVvih003|K#J4ubZA<)0Fdjb?4w-pB*3LAEBi^nQ>cunm=Z##SR_ZG+91XiFfoj$) ziLa0M(e+%w7^Qof&BEjIu?Ui}h~YfhfEX~QBiOzxZ<1|ClErM$NCM8i1OVEm^t^dO zK;?pcfb0FtcSK?;ire6JI8*oq_-uLd9+P9*J;{R^7vKH`ZJ5_K_;o^!fxd4uGxL6G zEPq|w+wb;5?)%0|+c;Q}#NWBje$;ZfbnkA<#=~js@ax^L>ekk8%M6uOe5}>3vJx8M z1Z@A=e44kq%Q5+G&HXpaUdO~Yn}C4*y4sEb)qIbO^8DF5GO-|1v#!nJGsWanerr*7 zCpsb9O|SP-Am0Y-|g~ z?^JXCumL>zrh1FfMZuFQCspCgw)-7mhwqwh`PMGnqEeyN7OT0ZKN07++@$sWNg*g# zg^VxoX=US1`Rjnh*SEd;pinTwlfl3S2@nVNjCA zY>7e;3Ck5(w#w=}%CVDIbtWkRNDiz3R&)S5O_nPcj=wilf8x?v{ zQQ$>~0TOn;b@xU-7)t*u)Yq3dVCt@U5@aFr8t{hU5Savxw=CTW@j4Ec>`{cGd#+qO<}n*zTc=|#{c|+|hrxD2Y|+oO?z?rH z1(!w3oSylqyM+NRR!^b z;(;3URp^V)4K2&8+rM5;40ylHO8Xzp>o_u#8MnJsYIyNl5AbiWa$2wI3f(s4uiTJu z2TUXM_LI4E+wCx_G@Uqi^~WkIL51TP^Po9m+`E^(&-0yTXG5XLr{{yCSxihZyk?|; zXS)9O{;e!SMOv^W7?%(y#n2e8g)7#&_{Fb+x6VLZSI| zlH@;9xg;J#z(LIaAqQxvad?LCgsFE^5VwJeP}T5Z?uADwB#qTxaK%u1`iJ|audh0P8cvmXbbF^J-awN;_2j=yL3IZ_%H0P^DM31ubo7mY4vuF5DK0&)V1Gl zjuj_eO}q4*Y6BsDpi$=WqgZ~JLXOoh523}8obgOzd<>_5-S@^V`#1BaK-1R6HM8g= z1U~1ZU6_XX%2LBX={j2hkp+<0y6(oU>O|80x3T6dL?-ejzWeU6w~JH#q+;!4O;k3V<^^>6S>!%x~M2IaXc~| zhE+_AWis_)a~us^Fwl=2bRZG0e8-gHb_Hy`o{Whxs6q;p`m4@Nwz2o|L|DzxHSH#) zAq3y@Y@{H8VR?v<&;{Y^(yRl(TZ7#E&sd>5IFb_oDn5C&ofz)pzQ=|-HsE;U#Y_xbUbd-3N*tc=IUcm zp>)6-!D@s(YdznyEpNhHFqmLaB|SjcFwGJ4`k6q?IneTW*ZNG3s-~%JpFXvzYuGdi zKCy>Bli0k1wR3u+L~d6bTjofRo1iB~S*c|z^*F`F>)|+4$Gc$Y*TTVQU%sU0Q#S*r zg!$8WlkZ2-)Wm|!EY@Y?=E1}yNQf&w=YZM~$h7*DwdHwXEN&)E{m6uS1+mXbaRv?W z1H+?;4Mzn@s!|0d_y5q#O1dWPHEy+6Hswbm>9$GF>knCgR02PfyMCjAQ*(N7V$rv& zQ~QD@;uq7!Fk@VNoUyVZFlM!x{$EX3;8wt|e=N+%#^fFHH1%2XD%_pGYp5J0)PFmn zA%kxHhzFomn?@vyI8yAqi-N00KL6t?( ziE{E$QH_mqww>Q{24ZNpQv5(Duq8h}ZcE;q%U#?`DxmFv#BICPuTs8Pd=)W|wHh2Z zC9MNlB2qV)Sm%pWd6;rT8glqDLQ>|x#1D*ogkB&PrzU4h|5Qa=tPrQlUEQ@Yr%_lL zUYWfiXhDOvA?0`%{BEP=rd(faHTy=pQM+f$LnsbKOkbp%Q}uYxq0@s6I7)$b0cnBGa#23O5zMt1=)OA!^-6{0BsNojf{Nq`i|Fe zCNM}N*(hK9Q&_@$V-?Qj-c)8Q8aVhDq9 z7LrJaZYY>&>N+s}$_^SHEboLb#A~s4Vt-VM>&+Isj#79-&ZMKv)EFK}KmMF#dfy`v zI?G~CW*kotC4m5u0~c|pA#Ettf1M>E7((}ULM7rOMkiLp4oY6|X}490gwnrj!-4IQ zQgCp_+HA$3>_P2!$=i|bWoC?@o_WQFW{AzHcXS@nS~uf&lM@rZiQHU{I6VgmcK++Y z#)b2LzWsYIB}!!I8hS8ZcM#u2ZH#^Yuxmxm67IMW$w6?++B$o zdt%1q{F@SbH3gC&eE!cSwEzYhWFCpAR8@swggrhNvsu5;WvvvJ+<}%>dkoWsOp|k1 z-v*3kyUmnU5qt59i^wFh2XTX3(7fSg0yba#t#b3`L|r_c7q2?7-_HMw#eMoN+gyH znPI8$bwo)<3DZj+o{IXnkV`i|ecOuAQWeNIBqkaC90erA2ZBAlzoD_={o!0TF39?j zCQya0Ch|3(vU!MQjT~qR)Xe1D5!| z5D;qkRa|B`L_?N2HSMU-ED>v^$zpN!O>!TEamHzBnHwpqA9ho4Io!ofY6#?`M%RN* zF#XTJ^PN`nnY0E!UlTmppB?Z{i4g6>X&W0? z&`CV&;iV=u*Sc*^v^S(%7`MCE**Nq8KWOM_?A?<|>cF%hi=V~3wvF=d86$lRfT``1 zyP)g-F$6EVMUD*#@k<~t_^TJdDnNgHbXe-aqMs5hyTjm4M2Jl<8=`i{#bSG8p&0}D z&ln?$P;E+FN9}xd5CNomSjI$|7o>#xHB=g8DsRq(6ZS!~s_ES&@?JFdHBx+>iF~`3 z(Sd|EZ)_H!XevC;vH!pNXetf4vCsAL(N_{BA4jgK=_P!{mq{#!lQ+rVG1SvG*`+kV zg@G0M#lf^>+g3w-5Y@l*Yq1u1N=1h@~=&4>GWu7s0npG5^Qqt2+{VjDD#QAFCWvO9vcKC=F^n_5-ZZ z_k-=If+02BA6_gxdLeVMK-*8pvZ9WPighNwUZ7C|55K?*W9B(=QjTLscIrWjo9<_A zv-##*ZKH$T)?)C!J8NZ)=Uhh_rVWQy+XgVLH5P+$@{8$`A(Ti1w2fe#Aeb>B5~2&N z+g$fiijCjmTOuP-mZoDS-G#>Ye#J#}1tDBG7?M~>Io4*DntvB`3me@-^*eOS;K&>m zLewB26reL_jF<-2Xb%Y-*;Ww2c}t-jmtSHbBPmG~9_{Ska{OcX)8AX)gYPzPa*F3o zWU-5#ohC||CPJEhGfP^A$D>U)20ORlRkzV*@`d3#M+#Bp*KPSdDIA2wC+)4gM(HwQ@XBE0Zx%@S@-Q)X>5() zD_RH7Y0?u;`dkcmgr7#m7r*54)A#uW9$hOQ%u!iZ_Yp)gmJCgq%4f9P zOEtrJeZq8}bnUaCvVf#E*2cXN#ls5gTOv6`O+jN`F={>XMbcPiMd5+$aB@V5RgYAF_6&GZBWq8a2T3GaHwItQ83cF zvuIS7vZT(ry0Hq~Y^klr)A?~;Vv+1hi?FFaxJNJyWPkPZ+~JNIqePvAj$*d~^=1Y! zHD)n{ehiXJ#=u%{or2Z-ZLf2_@m3C&fI9M?6XxR$?3m8>JSBGU6+x(K_MbnJf`^!9 z{1Kk6lqh(}J4c~07V#XO=*`;fy;XPJ%zN7Pfb@44=!=zm8i)R@!SH{L$kw+H8VbUzfb8$o9eQCa)4L zTe3OJ<(#RGyN6#}mX95laybs#b$RuG9pOBmU|y)nDpfay2RYRkL}nxgODgXa6Ic^e z&BY#S&hD|vssDhR-epgHN~d75#NbM!8_4E#P(dAnDIiF^!uVSA;yDCDi;#@g-A3I) zmcnb=3`TOYh}X2liOK?$ zdX9SpDJg1O(2}{f7r!ryFtb}w8=E_DkAv6e@z;fWG z>HQD&^bE|>-f+w!#`)S#l;0XEwdi2mblN?Ae>(ACqOawY9{BiB#v8OvPsVK>i%G$? zFh75Av6LwEB2UI=7U9&p(Kqd6uC*BK)Yxny({g}ycQ^4jn^-qd(qrhe zt62N;@ZfZ0*!eduZ|Y|pGc{d}?(+&=kW1os_gOdIl3+11!MgnK{&yVJb*3rl>4bt7 zU5h{Mi?$f{rB~hlQt+Byz&tIqwtD)WfhW=psyiR=&ve?kwPV3?PWe=47SFk%?n|#m z+nBQ8%)M8o+>|Uuo~8EQp2;Z8Y+C?{>|s4u6xu4GbG)bvx_iA_|0Vjc&3%x%bj;0L zr@r%JMIt)SRw>TG$xI>ztE|AYUaQS@GkVD+;EeIP5uQ8e{VYgws^8-xAKcYC_b^m` zA(NkY#mCZtEJE{+BAP`umwrrrw$LE-_3a|Cq#ll0l$Ma>|3w$Sl&4^O;=L@lBY{^#c6Khu!N z1X7`q=5NRl5r>}E>#5(LJ#2&v3>A$kwJMwR&y2F;aEsxRUt<2t-O@7Bb5(o%u(bl8fmj5SNFc^;C%U=X^aIc_ zX%UJ`$VdXL$hjj^Rn1i|M_3*;3dmB{1kBr&0llWBO|7Z7J#@4p{ zIu=bJbgR!LN`%^v0YX7a;C0(Ewv&C0MJJ4vt9&YvP$G^0&57;>V zs&&Z2(Au6Rf&5=BCV6)rWXIsIW)Xt`Pa*1aZuI0nbuh(bXihK%=_@cDMv_JBhNBye zZ$eFFU*q#X;6~r(&U$5rMJ6ytMj$4#Wph{NPu1vqy=2_?@46i^h}Ks9h>NwBr3EpL zXz__E@KCjZQoyP;*2n@zSkM$apuYSK&`0vVi z%@?uBczu$eA#%tM6g*-N5DSAYpA-&*-d!2n0UUz-;gy?p@jI2P{6-(3KPZdDG)4zRAzPFx65}7oq zlS2+-6uKOLj;?;#w^uhoX>B}_(J9g)Bfxc}MQ!6hdcK-*sWFmBaV?yuwdY}yu68rX z%f%WiBfR*)#YCFJvPNJ&j+g2)=kfcRy7c{f89g-x>IeaFr1$3x6h9oam~))%Y$gO& z5!a-ACi4d@U7+l(g_PiE;zv44xI-Im4E!j}OC>?>R|vrVmixrIz&^w{TPFcXLPf49PLmvI`FhV+N}LkY3@-+j6gC)r_%| z48@0EaV<9Gx)O zfIyABu=f&r^iJ&o`|w5)27xO*7c*orI@E}l6HdI`6=%17o2=!_W%!1Up>~%0CuvuiSWR?2dWi| zB#|{Qlg{IxmEECS-t3+1LN6XyT_sJ6gzW#hoCAYeKKrA>O+&H_^2)NX5jAs7Pi|~6 z8oMY@X%wb%t*;g^A5`6_u1XL6A6Sb+-R6=d!{N|&aMi6tKa3=_T#ShQ^}Cgg#QTzW zSTW|P=_5uSiPznAyB09r3E?w>jrd4F+MlR)a3h{pn-&xUI_rcjGw(gk)C42(RAWgK50NArzW5*4x5kgBgV zWrGwG+_!`HXKy4F^!~}If`2KnF+%>XC`0*e70R;`NAY+M2%5aa(gu$8ktQtYo2Vol z?E|(PAagaA-7R2Mlh#-oP4wilrG<04%XSUYJEQ$ z{N}TZI-TNzR@PeZ_g)jUrI4BzG4M^YvciLHeVHZLmKaIgi!C}vhHWLxD$Z!D=+F*R zuCcjTK?lYv&s~xZH)fK2LBb@&Te)X@?COv#D}p`f)0pI9LEo;N#*sztDCU;NxtlE+cplFB9La)V|q@kBf4C9XSw66;pswSq`}S{~p_qJp zb<0tI5u?)Cfa`_a!<5$aj{NhC!1uorcdUl-HWi3qR!#I-soCp%=G1p2Urjqzw!rsEW5>^aW z3O8|5qvBmgfP{_qeh@!{yn`Jnk&dU+_p-O6DKG63f`=e~9mPq`{SfOecaciijs6GC zOddx^J~^0lVIX1q36LEw1C4BF(_SHmpG5&IH?T4Zq{hYKf5j0oFgg61oyT|>RQ^HB z`2u#ZvT8ZBSAB6zb3Qc>XJ`tpPv7CZkffy&Ix`G@`s%_dmJ(x)+S=A#?u`1-Xlkym>U+SI zcQvdMlsLfh@DG;uqw>f#&zGvCXfFCw1)0(rO&p9BC*GZ@pAk&ih5}*rXI^0U5Zr@^ zA;kgLO=S}F(4Iiay^7iq`sL5L%MB9DT3vkM@5doFu>H-U`peT6UDWc#t55&%{O}(V z871`n6|vXb29qxz&3CbLI*u*eri3$8EIs~VTD-vZd%VVnO@$K0qKbd z+w!D9ZBhw*1X3h~3tCh5cIdhN#Oxw%sy4^JH|=cO{K@OC{~wkkB+IPjz0i+Ra*cMe zG#7C5vezVkl#Ie9tz4Yb?R=TQp6K%aFB9|9J)Wt|@`J(i#eK|JMO7XGm5P9udSHLG zc(-h0@)U5EIi-q3bnG-f>l+la-J;ifZv&U>wNOh@EFJ5iw`HCqYQeMOTI`ietG`Jg zX=nE_F^+l_NZNG?wK!99TQ?bb{&x9Z2hHXNMXeBSJpLfa{7?@d{BqJCKjZH|7e#gHF(>tW5*j5?Z_rKQg=^b$#JYGcwww;y`s5s6a9t8o; zD#~?Nxpef2|5QnC7aX0^c}iGvx73^*s@tP5$w5Ob zEE?KshDR89eiHT+9)vt(_*U9FEn~+lX#eDpp-F(vW@O#xNrbi5ycdPB3mlg#S<|rG z;okWYhPNk`*rjpuy(> z**jlQw5T7SYyzdd#c0CC&C&sto>5>rz^o)W9g}MO+Zv^+p9O#R-^;IEJghHvyc+sy z45La;@k!bOV4q%~mIfo(iW=ehI{g8$ZzRYM~ozVMTk1qS$`QhqCFR|x&XmI|{2xYEr`FtMDIoi*i9H-L)-sw~>YS4-+} z%!&Qp*8Y6o27J7&p;E6UW1NJ>7?Cv?E9gQTym_o8hkUu|yQ4R}CPjy4DCvD^(RxqD4`pN# zvzZ6L4o$OH`{O3f1VQT&8g$w@KO2@jL|b?k^nblLoK!>o3mZB{GCr`mj!0X+Us%Y{ z{p3}D+V=d8H|U=wD|2;Y5tm&0)S;}`@i z7B<7kwOs&RR=!W#r;T1FYve26>3&-Tj zfY*n6GIp|@0CJpvZb)ZL7vb7;A_%Ml<ref z|2+G?^j9he*=KD-NP4<}l<9X8hm{+i1rOPfrEM#ft+_3a6kDeGUCHVt!*(-;(#1wzW5K>0v~aR{_k1k$sN;I2Qo|z(p}5|fbyw-Bk|qbHU|4MpS_G z%JFuVW1B`%rLxgH5*CGXcwd{}b+@XYuawr6w`-aEih5x?fM4<2#w5U8Ol$|AO!Q0JFQc3E;Kwr{;;?BJ{``O-h5W9?MLFNjp}HsP^hL~n zg`jnHvxkfIXAFaN(duZsV(aO`N?X~KoX!&H0}KWk%Sd9FBE3VdaTY)aCyxY#P^tMR zpmn=&NCsye`b~@6LwHVi?Oga+h#w1hd?z_aePsITMgYG3Y^QXd1Vv{%KE%mEAFm2p z@UD;bM%p~<^9H9pjbE!zBHXC!a9e>KXMP?UBwsMp0HQMF!u(J-wTs?EGc_k+IZ~z9 z*i{%0n^02})JQ_uU33&5yB=aJsGd6M6?r!I{+c;U4%SSw3Ab#vv}-mob)%@caGf+~ zQ7zUg6zbe0XhS{M<~^?lepu?W{6)&@9=mpr&)@p~J>uOy;GDYP+&TT};QaAj>;vSF z_nJSEwNwQk2s}~ji$_e52lb^$XYWyC`;Ic@^7orJ6&}@HdPotZou9AcNp@D_WLxvJ z9ITPmBY2-UUcqw)-qlRJ*fW@2*o9hU6&FnnxBdYCG}I@4)UC5E#PlY-OnN6gGl~VY zo$}}qYZcvTXLFiI?Ib8LvE05xa#JM;%n8Z*_OXTml>Rs59@bBi|QBHAg0%I}b- zHXeuHGijR_DoW-m*A>HIRMWJ*D0BTI$N6f6^3q0Ex-U@P3GoscqT~kx!ytRs8!*99 z?ZkM-ybeI^UD{9|-zC2vBRYOx*|1o64}>GHydpcM7ycR^J)4tod!d%rxNPx_qlrIT zIU@;GanWS8i%k0Fhz534*N>0}5ozQ27novSB(@t6KZ(E4xX>4)WctBObDMpKm`a;M zOx3XEzT{fbHt_aIe)$}I5&rnwU`8ncZN^@cU+bU3h)iV)Y+=D4jh_nm%gEuSlh{f? z(=KSssG_rxl$MPw7Q`U2nTfIgUWljuoo;(8J{m(75eeBjAizl9ArPvuPIXrn0^{!s z`ggg%;<9vsGuS^-3uw6K$1e4iC>8kMiz|UTY5M>(sRRkUg zZCpyB6Qf^{HPB2MTrl0DzMqQaY;mzM2MzwlPyiu}5(G>C$3H2gO>4#kZ;9?vTRshm z>t`?C1m40ePAy9#!NCO%tM%@v57N0b$GoZFc5p$m@)?^~ArTn?lLKQ!mmOdNNRqK- zk{*#XVB(V0|7zot26qn{)aK!lT;rIVPJ zF6O4CCc(qqcl$hy*G#~q$98U!i(L#C4!0LGV8b6FLSOC)m#%7&_YjffKVYUCmT@mb zWVCzFR=~J5LgsSQjr#9Y{QYQh%2uIb9y(eMIAI1TeG>%$=Adar2B)AmD!SM1D7DN9 z_9b2nT;SfY^-Wx$Mj>{6KujdxRAXm$ zXqg@DGku5U9wrz+8FDa}3_Cfo$>(evU{)~W5keD^67~TV^)-d9i%ytSg%~^}(d|hr z93f*gT+n}EM$6#l3nU!_o|9oCIAPT}K=Lv9v&0M0d5P%*QYU? zLU=rxgdpMV2y?tnPuWRLlj)ym2%MLm@xKEM+coYJbL;zw&M1_GWrBK9T!0&Ab(t-ZfT*G<0NL z3imOZ7jh*1340z3{23l9+)lA5&x?NKrvt~$n+cPb*`(1ww_iE?!Eh0<;Osgadi$?T z&JW^t1@0Q-c9Ew1rX|#_x(Ps;p*Q3r+bsz2Gd-Ps;0E+>6wj!mEmt@5SbM*+x=}k53j~mn z;?mrD0uznx6Y9Zh#&;AGvbhYIi9W3XTYh+j+4||8tUf9e#IgGIv5^?@W5^`L>;bQG zy3he^x1nx2!-<0JYNi|3c%zszOGOq$G%4xQLi1wH@?G6WfMLkgvGs&E_T5X@L)|0A z-H4Qq%Q*{(M+LM^Vn{I6?igs>;<*Fh|2QB>)NQsnDTD~Hb`p!Of;8PICDBbF?}*YW zn!-8(Sp3hVX}o@D5Tx)Jy?;zT&?2i^Z4o;OaT;-g{1#ee?yz?>QN4nEp-mv~GtF_+nM`*xWtF9@%zO zj6qwB)re_5EGz`%-!{jL6ORgAAe1x_!6oopQmxG0$od1aC-QbEUm-ShJzEBT5+Gf9 z^oIyj=^3w~tTBXHsdj9nYW@LwHe${anUX8iKgC>Q(k0=9RMoOq@*@bs1sMim2o#aW z*79Z|>4k-l9KQG+f4SF{j3X4Oie5@JUbwJlA69UV-x8`SQeVPp?w``JG%>V3%1_@I z5>O5JVX?9+BJYKBMcTs7fKd!6g*7Ur6p4ZdKvFcc$4mns#c{P2(cu)TQ+)nQtMOT; z&f7A68QTDGRWom-Z7ZA~H1T5i99Z&4W6}X~0x4ghv?eHk$R{tVjgD>nF`xF6AM_MGsUiS&;;=I@Oq#VcirWU*s5{{3L7@GA)()>C0xV>DWm@X_ayE` z)L41G7tJ+dKWJJ97PFeoaP1rDFBnSgANnUukA2KVU^jJRE5j4{kJiT?cK zZ%BY<7iwi=8jlFjwgmqgdi7~~r=-oA_EyvlaY)a1@SY-a8%grjvlbE;qj9LRVN-n9~)tF@6`#lPLa`Z9J=Z*#mXGt`PB)p(+#+S8z%a63%52*#G{~I?o)>~0z6_PK; zS0YE@j39{+Lil3PIr^YYP43msoklIgHeeEoh@&kR8XUpAM{5@Es_oaHPAqo@`%u)h ziF7zLFq4)1?Fc2thM$`Po6wC~bck-Uk$;Pw495g4`2f#Eg`YjUfMeTGwvCa-F(g8n z&p%trOds;Q)Ws?!g)bS5(5k2keG0%1!V8i!=6p!9iI6cyizP}s^Iw}tn8gme9}?T` z_TfjxZ6aWeR4+uE`^$t+yrzIAMuhzjg}LvaZ3uR_nx*oLr7!b@)cu=L1Skf6hBcje zQ;FnT(XD#OFTEjK_wJ?wjI8gosF`FK3Nh#*rA#Ymr3TWZmM|cNwC0nN$X`y2z>F_K zIzYyeu?s+tnoEF$<(JC?0#{Q)C1pLMegp*uRmCHRK~O=!DN2?l+JKd78s3v)opc62 zOa78%XY~-5-a8bFaS9Rni+7u(CeObq)e@Pcp~a30N#HD-o3pxcLZ!g1)HiJh$Gz1O z`(<3FYVMFhfH$B zO-Eef!$#yHW!r`xLmgpk|0%7X8APC*iUm$Sug?Pbtu4-)p6c+y|!Wv2%d}nxtD10!>0@I<1$wE9!{bJhWPG&Hoe+dLqu2 zI`nh?d;l$iVmC_ntX>p(YFy8t*)%G;ns|A1QwzGQRs7htxPYdJIs%3O-?h|0eeG8l zqcOImymMRTHp;VjGPR>m?Z=S!DxBD@%a4Los#DlBLFAu*G+pLxlvK<{$_a`)U00M^ zS&}o!qGDbF~{evoS&^)kl8o=`z+=jdON@ja}}g3oZush3#VmxreSuC zs`_9LM_%@kAkR^xVM*(!8<&fy8SWy}TZ9>tT%pR;3a(HBtxT!!W+Os}vN3OTEsqCu zx(WLS0Xke^Y_0LiQZ$V z%?9*)q7dzpx9k#hIpr|rU9}hv281|Q+R>1Q| zg^5R^E{4T6Ir~b79E0PH@lL+e@f$lve1rS8#_mW8zt#LDOb^WSx4s!MQ;wIAf2q!V2-V3gv4_f$Pf?@KV>m?EcTt5Z8=eY}6$P;qnTRA?k5IA))d&W0hzWO@ z)7YTcg_GOHc4nCi`Osa)7lFM3VT=j+hb5edm%0^Zvbz3_lK1SYc=TgLK>DB{Or50M zvLiDD){%Os!o{Jc{YNX*%5amgKy8uMcXPA9;7+P3ag1renHq{4tV;U~h3Jj+=4j<`n>BX{B0{%3+SIhxDXLOj8UNrwYl8=Eh@L%ovYl?;J zN_epE7#Y4x?^B}cc|oC+W7l;m-9j`b)!Tfy#s9@K^M0qYc7M6G_t_**^krA-wW=o1 z`8kb`{a9l%#g>Pie=S{DtLFzio0)#1W7rVnBilU4q=Pg*f$oL`P9oGUSC9%@)hDtN z0~bSZUl6nb16$R!BMZ**-&1Po8b|0!2_8ysh|WUpdza%VEwji}t8DfIJ=2B>y{A<9 zBKiTTsZHJ5U5awcL@v~1myDWXD0AaNxSM_M6GlM1)BK+2S|mE+M#{QIr9(sM0r^gNfL< z##LSM3*%wV;k<05d^)C?d>lUFcG8dMeTE*HnxCPZ;;IO2kUv3g9ekv3#@^R|m5A3) z1G@?>;5p_VouV03;}C;_;_;*3{Lk^+U9&-Z^E#_!QLM3DWKkGELra3n78XE^J*LMQ z&|kn>bOC%dL1=i`fNo!(c;DyjX~B~ZWP+aMM&8f2EuMOzR2}>$e{G)X*n9qc4h}mi z3M+l#iwF(b>YKr|dpb-_a{32x3as<9xvfW9i3#WD)AK+e5EBSQYG??62S^BbaD;S8 z6m-CM7V<0E;c~h-aK}h=YDA?9O^`%a2Jt({Xh_K!Q(@q2$cZ1&0s4rVOZ*?x zc3mqig%#Z8u8?0MlSx$ZA{u%5Y6tF6!nJi)-#9j$uVQv`UVuaoJS6U1fDBQfPne<` zz9ju)DeFlhj2Nf&7~e4(-uLWYsQ|%a@35Y)#$xKJin7DW*jq~m=UqZnc^qr zU99Qc(s!`Uv!I6aUVlI);|++x%O&E6)!5QE8?GT}TU;;+76Ax|f9`BnX}EIxTs!$B zOI;?x;h6sb%Xw=97|lg`j$hyEc3zl`*_|zW+S)&S>oLdW>PXi+{MKovu8`Ms%oN~M zyD%w4Cx;usL7rn0>>}`|Ygp@dqd$5tt{SIcNUXURzB|7{?^*9Q2k%!#avV+n{P#>b2VNe_Ngm`yf1R3j|OY$}W)i4}J(s*(y_ zN-yw6D26jrs6~x>_mHm&88Wd#JiIg~j%NR&jZ`i}eq}6A0lN|1g=wZ);0R)oM~BkA zPvvlp8=$6OGJoeTmEz^)8}U;7(QYd>=>`O#OaY-d(j=ybZW$QYaPk_u)<&4|Xzxb; z6sp~Hl*=*XH^q1#I)$p85gX<<(=wTiNu_(sQy{t(Ajg|A$J5P9>?pe_B^R(A-pa~! zfFsp8O=rYcd>TJF*|`;JE&Tn@L2j9=EW+VP#MK#`>!yC3W>LybDf-`Yh0f%O0OB*S zqw=*dca!)C+b~}j6CZWUr@s-EKyIHu{V+*Fz{IFuj#KD*-sO>z;Cjx{v)jt1*tR%q zwhF1|wV#vU`1Ju+@x)?+t@TEVLO5T*;|H+09*LIfievpg>w2H#*dECTd`7;e2SSH! zc$Y|s51@Ael`8XkUp=lfM=6^XDpl%xUx2(`!-wM$Q3%LJ6XV3irTeLaTPvYk3Hx3M93|UK7^|{frm%Y<~R8;#@O+y%&4-z%FIv;>yz4 zQtW%3LYQ65Nk9t)YGk|NmBD+K1E;MU>#Y|)(Qt(y+OaCQu^&g(-YFp-JxGwGX0$|w zKM?>dA_}0y@|yAZraG-oF4A9aI_SMwCI< zccwA{P2zu8JIk=9-}mp+9iu_IyOoa7-AFf5A`*ghj~+0jrD3GfDUzcR5s+3Iq(ope z6913i-}C#?{cOiL#pC6XR zd5+|s#Cwtwt3q~7Qim%^19{4F`x-To^gZLeQ#Jm zBoJ0r=*ddk-ReJb>n2628KarLKUS2y)@+Uz&F-K|H zEG&U#f6`=s{H3u_FCOdrfE{<^sym@J<~!!_#Z;CQ7*+8k+C}8OOpfpE@8)ko1GzSm zg`DbBCg%W5{Nsxn3~CDP&fnt2y*gG-O{NvBBPpk6#QKF33U&gRSF|2YjkT?p+Tq_q z@M1wiBGiAb&(HL_?NOf;I6U|~(EXjO|8_Ed+Zx_IBa%Q;x-%IP{o~OqFPTPkysl&O zvCF68d8%|sj+n5@x zZdl&!NFb6zoX53%*WDQ6ww3p9a@&=1s3oKqaXCU0+qm>MRb>o9xWGsZl?m_NYqC^- z({e{AF`m6{7Mq(2wUBtyZ_dV5e2NDI>%`tpHiW#odohkFi9zJjy!jZR*g&QUhnGru zCqqa>vL9DCWSTOuM@zEiYA_%7E9W|cSMTuq51+H{-uJW;@msa@^|FRQ@a($s)_kHB z=WRgd)jjhs>F45q`pRn0!|^V)+dk7H?)x(;!^+y52?)tw zl_QH8+(!8gDUcV0-BOdqZeD@tF~P>*TyQ-GSbh45vo4BefyrAOB(9Ru6^>J;lR&Xq zYbSOY^cwv-(*GQ-@fLkZqdtM)3GUe|7;TV={xCsNao>{;#-(9?CtzJ#3Ng5svCm}I#&fgICFoI+P1@iE#E*-PT58tEq|(LU_Q zOK+k>sf93vX5jHZo)A5xV-Wz7*%#tQ0HctmwueY;*I=)g+__EYG+Ks)5&aWqs`|n| z50<}Axm7&A3}@jec1JmM!#;MwEG@c-AUwXFTO7Q|?8*HAT=$3D;YH^T5vQShCT^?` z*bD*1dgwcfQvV!o!3~Bq6wHm16uc}=22y1#hWQ`o>d&wsK`b5En3&V~m~DTZb_M}3 z*}do2%!n!|ND=p4_>KO!L8#-~8h5WSX#g@QN+blzs(hlxhZ)mh`ye|&L*OZtMjjp2 zF%$J76Aw)XjQ|oz`|Ynb*kDRj!T+AOzH1Y+oFK^;4BL<&RTKIWq4Y_e1Z`9Kr#W}O z$zu537UO+B(XY2tBB~v~(79RfIBg*P0iyiB|IvQ`@{4z|PQo@Re$l|MZVPxHPskn( z#2{mGjPRIwzC`tIfg43{n1Mo}%*Cux4|Y-j$RZqjq6esSQu&Y@_F7SG<9F2!~JcohR1l4}3 zZ;e_#L}?Lt2lj_q$soMN$|xIIgvuEH=Cipk+NUY#=J3^Ybp{ogQPGv=V?3F!`9w59 zGI2;kP6f0EL&9))*|1FY1uYK=1-}S~V$3%otVA<@L7qTmZZ$#^E8|UwPt0@EMi;C# zBktZm*kB#O^)I0g6^s2rlZovcHy^G}|A}%UeJx*U_aY*U>Qe&Jc$dPN+o$&g&VCpJya@ zB?c;mARX+9+}2jRx4}YuLPv! z+~G0&wFc+g6!z%I1c|;cw*|yrsne_vRL6Z(`R<561rsVvYwLoWQ6j)Qjbr{?&+2i8mT*&;GX34`dl&c`Cq^WhkPJ0zxrfj5SuKCSf8ay)cO>_PwgpyuG1@LDBC` zs38l_GOR&Nd>~a6<}yh%I6s+s5f|00gIZ(>4oMD<;M)#muE&3!k+t_<#ePf_dG}@R zw6@HMExjPK^z6Q9BHg9*5*P-!C40mt{}|?XsAm5_Fl*?L*KY}E5F^w~$>7>wRyY*s zSJcua_TGTQ=VpZqH?{7H-v8%xk$clW{Pw)W0EM@M)~on)1putR?*rcjxs~$fkL7ZO ziE%032vMP_Xx_A9X3(s`-p(lA=u>i$k<)0=n4|W6;@;-J!AWt3wip;hRbXseM=IpJ?oml#_SL`-`~D`PL^+4bHl&eibuek{?fxbssT;|@_&y6i zobgczDrA+z8-YX=_g{Y*;nIw2xZ@(@Pj@oa8OLza;LR0b=D#8Jj^)B92`iojeh49_ zDCCUE7DO+1(DM6@A{HrLF0y=1=Sp^*#4$mmR(pn>afnmDDu9{W32B3os$fq8tQrCz zm}IU^+#-v{}l8{y966i z{MJ8qTsyTnJQ;~FOEXZer7P{3S;li*Q|&XDA_0z|Br|4V3b=DuR~;j~ zHk35?v6LaH&5+vpTQr0nvaV)UG6;IlMfuem*rAoePxzY4L~g*L2#SRu6h|GEiyQQx z8d~gEm4S015KMXKTDGJv3Ro$CfK?SF3Wyh}l} zB%ENE#rT4V*vdXOLs=EvB6~9^5wRmPY9^< zARyOk;Ckq$@a7pcBefxQEK8URYe2@WVwi#8);>T^lLZP1Zz`17(Rg%m z1R>OH(Mp<}bs<30y%1YZn%%cJsl+M4a!$*;t<$-)6IyjdL5nXHxqqgb+@=V*lo!@NF zb1{U;SdkoiQT8o_8y~2haZ?RsS zql_1|RCP(=9iW}Ed0gify8v8jQeSRG<_dc>5-<~!1c`)UvEns9u%VmnMWT_DxV&jA zSFB}=M#jvI%99vJ#o0&E4ri2CWU<|gA~xM}ZB=%`F&I+1iH1=kf))Opv2!c^;r6Vv zU|Gs#lHcsYG?0VfUelXqB)V`I_Qa#yq}XOCb$Tiu@PRy$gkd<`5Cn8Y5-#)_2GYGN zpQwu4hmN;_y-C!gXGrRIk1&T}2dH62af+G^4*%JSF-WaS^zmb)6muILa2di_f5M@W z7-{OmXoH>@Me)b_5Gy5n%DByPAt|(@BviIha??%nVR7BlYu4GD!l%V8`cI%`T8n3d zV5W4v0@NK-99+jjGBNnmgiW_GWv~+o&$|YwG%Y-#Va(_koa=E^ZzzVhH?v1u*#_bb z$Wh9gC{Bp4m_7YWZn@kM7=SV%s!^F3w2_#xd)kDD3(~t|@MD#$W zhM)-F=-s3n&fD$Wm0TM-dk(on>A;QLP&5PDd;W6-Gn2msA@uFg=(JWB7ZWa!d})rj zoB!h63!BZq`Fqaww9r&uy)G|@{%qz%{A)+bzS9uWkkA0fDAtsL%{Gb2R?XT*Y-oS; zmz0NF)Z`?XVX1eT|B4F^OMiRmXD7;o{Piyap-5V+u8zG2z5ejS@_{v8u=mcVSCVEI z>dIZ7Wtxa6&t8h(LHxb|e`>%Wi%|)j`a44bw&%9RiN5D)89|S8YShlY7g-lZ%3J3v zmGUhD)|bV+s8Cfds;m{mOKMHH8s59$QxlhrRHzyh*95&WKl9%DABZ@-gKqa*{Nfs> z!}oS=40)39%Q$;Ay8!$l=%aSkP+#BT!N)?3Q;I&%9o!E-Eq|-yjl!ABnJ@V+Ry6TpX zh=$lO$1=ysZokQD^Y?zLNg{c?$PZS}ObzB~xnq$7XXZ=nF+83#c$u1vIrM9dj?MikYfErgf+-jFr7}38`8Qcu%5GqA zr2OD>zZj2ClGRxZ4A)U2`#9>-r>3Fy>=;-Kn(0lrC0#7KH^1`kc4!asW)yN8AUVkm zTx40sLQQw6cU}GJ_VUCLy*Km8uk<_D zO?MdG>?@6su`O@O;i$bG@5Hz1UG5vYvgU;BWz$}%PhI>gO_8qJ??9&EEeOrYW0+#q zbCkh|5T&7XOP=D4_6Wqxi>KV=5wtxn+42!E989KlbF;{~g1VDa^g~+6N+#HwMUGT? z9G?lbgQY~Gbi&+Z>`C}r%-J_}%m%UhK$+38|vSd#!;E8977B$<%9bhoc?#p}al-P?pigGv0s?$(5q!dG*as z++4t2U!RBQ{al((E1^e<7Ws@~%#n>}ij>sZ>sQe!a(N=|#xY;0nYkqJ`swU^Y0`eX zJ4k3>^MI5t@-@IJk^XZ^%fQOtGK@JhDvf;Cp;gz4soAa`<{d+H(#6E#*YaY{6I{W!xts+e zTyM1*o~v()`R?h{ioA#?cA2d*cp#qwvfh7pt%j~U*on4=lg}1NSXBese>M}g@DQ2E zkCP6e#PeQ}k{*#@S!2$`MpP!{h|0defyH^ldR}D^U9 ztEPA#rXPE|&mx*X)GR-|c&EY9O=Y;p^t_K0q}&)?z%dhpQvbBtL7~RD+VK246^Q#u z15`tDNjD09%_&i5tX}>0SIV2d`U*n$h9GkJh=RD@fh zaD#<0tO8e5eM+U?4qApat%Dbh=Z#zz=?C{kH@7msNNfdUd~3A=OCZX^Goq#X*F~ku zYlY4als*kFQIMO`G18hZaYlWIJoRhW!9Z%~fzKDW`hF?dWEKi?eWRbl+~vm3lVnDl z<-TFI{RF;<(m#fgY_7Nd!lw7tXR41{Hn+d|N<}W1mwbam6NKt^G=?M(9GKi#oHiJh z3Fgl68xuFCibq!WGUrUh8^nQhzTTFb-J2b&W2T%_a8js{R9E@AST6i@eRknDzdarY z=F901*uB5{y|7?Si=&X1j`tRduCDTm&g;*5&4+p%d(X%3{HuX45xY~&Byo@HcAOZP zVp~g`_g1ym-aF;YKLHVi$WLr#9xtq&9jDjo?)fstQY3*x(q0L?Vdn9G-K?$Mr6_o~ zLhmu1K6K(Ktx)l3!dP}~&3pf|vu=oL-m~BeLPkl1*iuL{nB^%t`oZ!jrZrdINA8bq zhH1FI4r`%Tgv~P4*@dt~>O+?5y+84(C$p%2Bx9dWc!3oLuUre6xNX<7qP}T zHAPk$F>JQ_Axr8i{`A5dZd&S?%^5A&dM|{H(mY`<`|hT2H2OopUwKY&_^f)-<7#Dq zK;^l$2NNaTRzz+sxGQg2^xI^Ed-xkb(s|jKR;FsS@@3Qa@Q@}#M&ZGNKAjKR++^aZ zOBY8A;uD%ed8ns3;>Xcop<8lF42RPZ`G@}3pYJK9&l{pWCUOi+jAs|5ceBxMm;`|w zLc3nbLAsdN2~(b^=WDL)QDt+dron~^4P}mbvFg6527`39T`wt-j286UcldBi9Mz(3 zX6n#O<0&9L_Gx&5_M`=sG1h^DP%W`US^f=XAY4c28iuEuhRqn_wFv%Tj>jf&cEClI z>Z-_R_7W~<&QMnGM|Jq5x|nYp`~CpW`@Uv%pLb9p*!4zfPayC&ZUAEMotA(b=lY(8 zzlP{eF~17W9P@< zD6j-4_Ak+_!>FPNmm9~$2K)I+y`s11UpzfLbj;pIn1OkKjU}Z#8K{b_OY$*=W57drjQY8W5;1#FNFQwaD@~n`kMjS+FG5khx$lmx^P5e zBwyFXB`rf(mCnk}_sjk+$;-_zf&rtHmNqsHwvU(Mulm+eFVElTS;kbFnp-+aAK&H* zS)S(8qK+$ks_9k|C2JP2^GL^|6VTH>>7Y|(+=^;swcXP)bzP_yE8(NogvuRbL@;r{ zMRD6Z8vy}AOt*}Tz!;G~fs>vFrXOmxow9b|rVpt=R)`Wjim-GK3+@n_Qys6NVtP^0 z`MhB3WHBNsO!&=UuqjtM0?^Y0<6RmH86~lreD=+r(5FnpJBV&7ZvS09USX-O*WW!u zJD0i%F%~*hq)%ir$XbPGvxtYh(7+szWSOQhFe$E9t1!+DwEKoCQmk`(DcFPv3mxDa zA{1Yt&VSewc6@breL8l!H(g>hU7~N20ecGatNUk+;n4lrN4V)&XoKY9K0}@00*&Ac z%Iw~_wY)Q|t8D-{ni7Yynuj(bFeKvu2cidkVYhmv#|94{7-+hhMz{J|Vj1zh$#QG|ij5N{-M1ud- z_(l4&=~h=(I$^9Gz=i<4F1Kl9Z~SDK1^q+s=cqp=9fW4Q5$aSOKHXh>Tcb;Z(nYxKJ z@DZbyUSH^T-t}fDIHwvAk))$n7whDVIn)$fR!&$ec(X#(dZ!rG8T178Ibzbn&ctD> zez6k7OGu@zj076NIz|8{TOz|3e!_--3)7Mkj@N@n-6_#AU%pzLb=H`f`|SZlHqo5o zA5}*dm9=g`&pck3(pH#-UaNmI9#t3u+oKjJ>jl5S6D3PpS(=VaP@<2seVD(cEs&s8 zjkQbH5o^>)=eTt}B{ZF)G+l0&WlT^M4K^_BoV73wxJ~w@V;r-H2I#r?CI(}}y2r+~ zekRsN&Op46@lB2Ax^R!GWrJ7Y+_U*lTekMC{2EJ7J}%Yd9llaXkQR1tM0RRQZ^o%D zoivvHE?)@6Shwd{Qnz?cLk76>*$!J>97}mS-p7Z|CZ#K`tF`L@;|Qdv?psGvZvg2#Ms>^u)L|Odi8Xc8)3+Ei z2?f~qadIPh%+r_$sbU*4_s>MTts>44ZYNy(lDqk2IBXu6n{D4e6-G#<(3?QrzE{Mw z2Iq5h>X66!;Dx@zEkDF0B6A6U-C~g0MqVop3Inqik+Hxk){2X}qPFJ_PMnKNeOe8c+~blUjX;bFm%Q**g;Inc$|!ccVkw(X z<6=l=5*EnkOkTuM;tMd0c7ieKvP8Aq7bF7>j>c^ycqMg`!Z=R4hLrraubw5h>Iz#R z3KQACWXF`#Mar_|h>+l=#ph_N&l|TR`nI}+9=fDxU7}7Ex2dbOmPMNBeU9!RD#R$> zr$@v+w*BqOuRkBPF;Ry?bC>0*U_D4g=pH#FzMPyuaIM)+BT6Uli4NMnqbaNro6d+T z3d+WM4D|PKXltzjOjUbjqNP4m61jyYlsVz6X$A>FF|AdfFZ}kNeQ=F*#P1mJwbhSe z)=^kvxa&kfrr^f@ zH6bUJ0M^!<8{OB+n?IuL`=PHsh>NlN?}gcWYl)&!%_cCxSL$DUH&&L#uZ0jWYy#%* z1Jq%vBV94DR7hh=lH;$;L1@lr){rhOMoouGAyMSH`cQWaG3}3fg zI#%T>KE~(esKf#z6nNbdY`h;^@+w)(#v(2I3++8O9@3J?E4#b0U?<_>aL6jmKP~| z1G?MNc|uZ71I&69Q}eTA!-YGnUj_6lU#4b;a(NejLUBGddbLhI_Nmqr*eX|>-OzP*%ckWBB1Z;tyd+*HZOGQ8r3uW*zQWhTs@D)_)aWDmwG2yI;yEMb0}&3 z`bI^v!YLW_C3J(4N}O`z3U?r(aYcGVBIh8sacM;3_|P+Cer>3$(K00M4)FRs$@1{Z z%1|mBSq0%KOy<0GHWJc>O0nvUD_Ao9-ieS%6GYNw3+Cx#`9gy~r8Z3sx8Qq=;w`A> zq<#7~AV!0OXxOP?)22*RGyKSu)j`{nujHXInX;m!lM2Q+>_lQBB3hUv(PqOC#b)W1 zR?e4NVazAz!OpgOXD~Kr-fDSg`>8i1KDW_55_|@|u&Q2lE>0(9MspJ-&uQCLR-EvVxC4;k&6j^g`GTC-_j8tPZq6WzK9d+ z$Po(V3}m*IFn}05iTXat(WxuxFvh26g%uWzzZ~7P4q~SI7%BQDa*H!`?pbNb@J`x` zEHy>hskRAy1EPZjU-eYqXWvVMj&y#Yq_NY{(1N}BWFo-~a6xq|0p-rj-0;j~q1m|c zsMSp_x8U=Lr*5es5&{PiKJh2hf8ZlkmL7_HM=XgdDU>wZ)qjy-lq&>`KhK0*q0=Iy1G+D7q>J zNkY5}$7aYj`XK9WuE#r_AiO`BVbhf72FQrB zZ2gLf5EowJ9hg_4n4BOb)OaB62FwqZ<8inA=uo}`p3~$@g>D{2y->Ph4 z*wrK()%FtsMyA>cv4kHxWm62>!UECA6tb`6qfwCF=fkap`)X=VX?tDD8|;akL!B`h z#&GqtQZ<;tvAQ8#{d6tvzb9CR)Tp@=v>FAN*(tbz=hIkJ61$lGKR%v!hkrY+1pe>P zdnHXC!;kF|`h)dnng4sIxoF@+nAP{IrC;Nu_l-bN))Y}f%o2|APENVycrdTM(t3)R zPWoBF0e$VbD2!Q~HYC)23)GPhzsl4ix=tdwdWQOBBmREMenW?zf?0Q3!uAK94qXld z6X!!0$~em&x=lVSgIp*4%v(b4kFrci4z8EBO(v!N)@}TuN?@4`PbC6*$~CiNs(;U` zJ7Yi(6O4Io2@40Ch4v*0-FhK6Yl-7)#1 zwk|&bZ-ob-&;gVS%Cz!fArhRN(PUA4>p2~}qpYbda-p|o3JQ^gfXv(m(7nAKNoZE| z4e3M0UW_MUeFa?IHyGr$py^v@lS~}wBaGo)gioH1CMG7nLT-<}MUHXCxuamKGsKh8 zg`D(@gW(fV{>XsAk}XNbskcs|8#)XM$=epJ=c0JGwVaxw<9~aXcf&r{c9%2 zv%kGMX54&uxawuxWEIGNrd!xBJCy5ENP_^%;30!dgkYQ3J%DLSLGDGllM-LnN?E%D_h?D4nBWx#fT9Wk*5+S)m#uY{qm7=_3XRH^Xj1g(8x^;J8Ru+N67`SKjbF-B2@^R9)hgl{{d*pn-a$RHnO&(JeR`qKaH%aiH?+;o z{D=A9IHy&sIMMneji4fA1~AM2H}^dI)hXB2y@Dj@=P~{ZJc;H zNbpH2gC+9mPf3VNeKF{M{A;@nOY!xCe&3e5I>V2T_C@jjB1y_YSzgJGhHy2H{hRB- z6b>(!kjVP3vTe3As3H{42s54hS|}@hN9aq$CByW(VAsrvyrbd}t|MEmOF_Kbs;ywo zg+Ap;u&nIX3lyO?sH*;SVA+p-6i?Fz2mPCE^1y5Zj#B~_tWRM3W(|S@GSql>%WLv~ ze~bO!Q49<{H!|w~@z`v)&snNaV#*>+4G|=tch<5&ykR58*hC_1ge3GJnEfR`6EE)h z3*9?#DJ16;Zc{sxu52ld(nZoa`Ku;)%4X+!zL@Rs?g9pI^P`t%_q?7y(jD@>Vv4$k zb|_MP3S}r;ZNg zh(aA)A3QeR!wG5up-0PH0t5Q$9VCkrg{(b8SE ztYR3IzffP5jH}W{+0rQ(zL?KEoe$>Rd~1-Op>X9-sphE=I2t{Z=@=dVHb%mBFMG3C zHoz*k4Yeg5VJJ}N(q`@n;3zY&yHj!PU5fv>eeU2#z1gB?XSN3`gl}$tM-Hza8dePnBB_MKS zTDzI_m&)`EJ+xE+){XMg+b|`BtnK}Q=~2sfU3^c&?Vab)tZiejc`MAo1QV z&&cyiojm6_#;||@X%5fTiHS%5-oGolJ7-0YUH_PyZviV%Tzk-_woqvXr`Sw)BvS7l zBtD51{Vk90CeP=DQulzOr0`!+>+i65kNw)iW%~~mkJGQ7Va?WDq8nV;e&D}=^Mm*3 zxXi5J1=`eQL=({Q7JNCSM7#vhI~{>tb=W@#pMs~ulK)z$Jlj9iQ0Q^+8otvnopPLu ze{Jg+4#)}7Z!MJydFz8|AsNYcN(^3w=c)Jt9ZY+eo^kBGxwjeIEvOq@0}DNuty4q* z79zX~eL0dH{Q?_FoU7mCh+H&!@%_0?gK9xR_`?lj&0z;WY~nB%Ii?pjBhYj05PUwZ z_G3*$$>M#hdgI#$n{+(x<{a@v4iZ{bAviS}4QAgc8`Wt1Ue087%9B(R368MxZ{9L2 zQ;kxX`VTMY_Tk{k90_(fZ8otv6=+xOB67=#tzWC6ZRM?HFkmP}Tm-yjmixmk1Z`h= zn-_XuRMd2Hw%gwR?4s)S05?AF?TnPbj>6m3z}1TT4v)URA7YXm3Usg^^ckL*iA?A8 z|5^*XIy(9BbJ^i*p-t%h_#vU6h8jyj;&|JFaD?WL~a!NPv= z*ZEn#fp@<85Qz@ctB?4_lP~zLGnq$RSAcnAPfj!{9iOaIv9XU!F6bsY^I|ZS8PQ;1 zy0rY13dATCtpu(Y0D>9P2UTUub#*oFGX-Wom>2B#LpPe@)c23TbSHpAgIHwYpHJQK ztcgx+A8)gdfNy(rlX^y{r5E`JxGPNxa80Fs-N8JOC=^^GX4?H0YbU0SiO)~T+JN#P zu_@%wn<95X+6CimP@h$aZxgP5J^&4I%RwzB=-M&be~Z;xc`NUY`FCAvoR7Gfc^%kQ zBkql8dTG3B38In$+q5q^=LsdDY?>CiWVQ2xQUa>6H)8@s9TEQKu2dLR$#0U|S*NlF z6|4n za-c6Ye=|?XiSg_;hGpoik)G6_r+F`5Z6H;CIf}ZM4rG~{`90TBS7KV>(fhEJ`sIfd zHItw(3KAs{<>I0hIMH4|^N;%K1N&M?EXBXgnaj@n=*xjrq5JfasvcM0R-|s57=1vH zwNo!-ldR+;wINblqAv0e^?e;vllo6+iY?I9J2gx#j^x~vkXs-b6+DVDs^pu{N0|Y4 z2qgGbeO|Yb3{84RGxvVAFP3}xNioefOVDKWwoDLP`4faCjz^aUZzRxu&y;3fnQXJ7t4+E`l-GLj-sH^I5D%D6!uRM zw@kXrSv{z3cl&zbPmp>+v(gA}PX%ib@mma-fZxzZ9A(CWiu=sBiO-@h;*E>X8)h;m zTI$+XZ9-0+Xk}l1487nDwlH^#nOk^<#m&1TO`tKNye~9=hjdcvX&Ai~#;YNUF=WYC zvdFt{?^$}u!*lfhu&yQQ0aMIn>{alac}ua^NQ}QZqT##rZF7{Cae^GLd!{)|_ImFV zZ+{e%u+|ng)DzMOBk*p?&DO#~k04~aIBN-Z@+cele<>Gpgv$_u_HB>XV!C_>-2;MN zTdg0CPGeS-RuvSL-yJ`hs$O6>WZsP+88owZb)Lu}xs*papd;B~Si**)VP>kDFqXIz7;J`}lan_v(I@!xb$A0Ew#y#N5mJumb^JcS> zu^}}!7cIWUl6wy)6p$Klc1gQscMvnF|sfA_Mpn znLUfUJ|ezkNo>t$b&z>a+?bh!bCxxKV7M8Iu=nz2DY}_(LEKDCu3EFsQ*2&)4gS_D zp7gS7sU@hKdfo}C>^4L6N1jx_8BK&*@>vhfQDlf~5X_%)e*xIL)T-@15-X^Ia60XJ zQQ2fxlVEQTKZPA;LTB>V-52u>BqVBlrAkYmqYwsH5~3AHOR%vS5RnZiGX5|vSoE;v zvwq0SU>wpfLEq&5zWOZsWw!oR@x&rE4$IZU-L8E5VclneO27{@Dq0){fOiTTAlUoP zUS;MgnT;Pra0W4jOYbsbbZW&{e2pa53Ico-^27Vq_5?k=04^(}+hW2(*b#MU)c=;d zX^T~d$D~8Igq8QsZV2xfMVHNl8Y5y~D9m*6u)f?-CgL_tKW!ySDh3hVlq|xeOL9QkvI353l;`Nil2WyTK}njzPuUBetCc` zT3hTIb##{-idOCFWeXi1_T*@KMD1e*2H8 zm^Vl7-RlU#sfhWSJ|Ev$GCg50|MEM5^U-5#B=u@~hw5iFU(369BAgVj2fLssK54!z zVZyJdxuop*K2X609Z7*qfV&dUCXoOuqm~?KBs!e5zZv!#>FMQx9-N84~Z zJs0yJ!~=#G`7}t2r^+7^;=)SdBf-=x_wQF2g=)d0*g~mh=og zq5WwWI8{-h%nWSL3IlDgU4*n^G7@|H-z%}BWnHHLzVLS)rCqQ&GbN0mf_EQcN1Xa* zX3_idK>FJZ)$lW(5y(tjTtP8=qd$vZpBVf^D66OFZ~ z^cc3B^;wvCcftfDm?L(i{_Tt1zq+(e@0Q8ulhQXHz4yJ2O0nc|PO#kO)zHlhT){lLtX*@!ID1`f3%;N<2C5ok#8fqVysqOC)0G9r564 zBkpP4=RbJEDP4BwP7=+`K&Kn(0PgyrsW{LFJ*0Jik!1{X`Lm_*4p_d+$goGB&b7`F zjw4v#YnvJoRs4L`nLV&aV4a3nH*!?`Hf_@opyE2{y}9K)wlT#rt`n@N+b4efnSTvB$s2Vd0Q!z2qzmC|wEINzO;UCge_X#%(i!f6pf7QACWqE9A@Tef>x4Qz(jEo(o{P4lEWtgMzOFhBiKd)ke zNAuTtd$&kV{u=k1-$1#DfGE#E?PL7=PgOki4Fzov9m({94qn(%Ds=IwdoU&|te`=fr&!WPn*mk`klWvyXGP_5Q4ny9-UD z?=cDrCJC$iNJij_1+qrSNC`QbP$)f7)n_=`dS6bf1uZ;E!bv^Q_}w(Shn{&*(_N&n z^Kh)sdOSCFr;e1!><2PCq!j{y1Xp@R1C1x8glEEY?wP%_%{bqiVQeTh!4 zME}F_>EGIxWs3IJ$-+|u&0f@jg29mf@$r`*$0&Nr8^<+9(cE8go$88x#UiG46)h~< zcv`je-r}&fr*Sz!xBU6@>?Qj+MslQqOhf$zH(!JO0QOSEW)G0X@&fcn@b3G`JUNkq zLwnv=iw(?SVJ~tJ!_(9Dd&V*~(dvA0)I10v084d1-?$H|dGkncwqK&60Ut+gf1Et3csBy=pL~ev}i^QiK z+3>b!9dE2n44xpi%(Om?fAm&~o)ceKDlNx<^-xOuaq02FZuReR<-xDovLtPoi=S6mv*J35 zp|>3b1lDef&79MxnL{*PvZI z`zF7;u-8z&>h`|2q;c}gmM^9oE18bRI~Y{?y1nx=rK(l|wOZtvqh!MLJs_p07-vz6 z-Wud~9ZVkh-u>=R)O7@u`rD=`i=%HU303u;?$13Gtiq5@nH&LUTNJD$5tQ-w!~+@@ ziDYHe3l!frjZ!60pd1UM(QKv-VUyiZ{oMOL?=%5#!&tMYrIW2a${cKiB}>d;i{Eb| z=}yU$aZ>91u#n(<9zpM!Lf|^W0ewq4CHqi8FjyOOBmhr}-{cHEh*?SlEC%8*=X{aL z=aU~30!uT%DE z51Vo*^{LdftLw+6imWG`!kH#VH|nF>4&WMh9>hBw#5;n(V%EmA51zd}mzUC>F?Y-p zg|NyUb}e)dyDTh+>Ft?4b~UbWMgDWo3xOzo&j?*;L`;C39%qNv8u$p?K0>7bXd~n+ewhPess$=;bcTB zAd1eU9&qO6w*Euk%mJcN9|}IMZsx<+%bLjhJT-7;##6sp=Y2YDPTmf09Fu;V*U|#> zRJDO+c6xPp_cJ>58TXczu7q-ut9$eU?XD^);4={>kcNWSV0In%anKHK%Giax~XY z!dyI7zo9Y3)KzqI;KZ>b7=MnYxO+@IFOHcU8Y`-S8JNC}r~39|$<|bE?e)rWm`mvG zxw(^OZcwRAF1(bNRmalsaho1giEz9&zwH3_nxnVuA-a;(nY*OoeFivE6?`qY-bhk- z8mn5YIrr2`=RV#O@3Sys8wH*F4T)CoHeSV&I}nz2`R_gAVALt@o@$??)`hl(l5sA> zEc|%e3z{{hRrSI)-CpyxaD_J>sT3rF8< z8ks^4hsz|wrS0fjMgOyiy;4f6O$NArXU-X%6#iPZjq1_wLy_Q8_B zp3VM0osS0#W!_OIL&#y3FKCo*ls4~tl6y1X`q+w)^?N$JF#W#(J~P406`fB6!ox53 z>z`lwkgO@lG03Pf;=(avI59SijCj5AamO7OK6J!B3pNWfN@MdZg7fJ}2q)1wroDNJ zfi-mu{V~TJ6AUJA48+l3Xc!e^P{~_&5#PhcI?BS7G3&Sxi<3`2IT%pN66duiLit%H zqAbjYK#cgd0gf1g#8U>;Ck_B(zEUr8D;I5o!KE&6hO`N@2W%2#HjHH*K3>JNK7@(7 zMKbH0d~zLOiAh2jesvFz5%Y?1*1IVS&IWRc6C-`fLf)k>-2wOEC?R`{w0VgW-_Brf z0*^H^9gbeG??L(HS7phR%yKDwtY0kM9D3-Xp-p%Uoi>3SK`JzZ(OG-wJZ0sWc2wSR z@Hj_^Hy=RTYp=aRJCHtUlLxg&u0q+lk9_m?PiAshYT@x!@(-b4hJtoLFM0yX0r`Pc zu|I>duz8g<&N}O?K!_n>=r!e$kT>#}tt~nd+KBS7*_YRvt1Rd+Gl&pO@}XIZ6l(_S z8L6O?=4Mb=hV4i?0@pAC$+83c7?740Cm5@c3dIrsCPZPSaIPas!mK0}D;-8hBIC5K zATxiATp4lE(V4MeKLQ2-T~88zUM^ocCOWI^aH>79&Fv`nZ_sYkRz6-NSC%CFRK4sh4RMzgLv5_ zOFEDtbXiWgxBY`YMp;-Gsrx6CVx?2Mie_JuUO}c8Z#NrDH4yL6mTmj98h~K_93`+J-vKR_7AR3mL(w zOZb~>=o@9Cov9}rCvBo~kbY14Kl!H}=pQVtK!E8R)ECPX^b7J$dyo&+d9Fem&>qA? z+c0ZK{PYco80nJt-Vh440YW??7qo-=Ipu-Kke67f^K1fHK-4zUFZ79apgqx5mhjXT zv^{0%jXtI58B$V_N1(v5q{t2#enyozMT{An(lDYJA4bfKG;ul{cO*DJiOw;~!@(0D zh7vi?}N9v99DI+6+otkax8)apyy?)Q+nE z7_Qc~QMeVttB5*W+@iEc}h#p#CX~$Z5X&NQ&H%l8O{-9I^3{0g9aqf>DU4 z=Mju{ELGqKVcq?BG??oW>mGhB{BK0*C`LLFWL(%Z_ZtO?q)B>IlR}s6;&ffl%;f zM+il=2dH(wyTTgXmOv~Lav?$y3;+Cn4~rO8)NtncU*sX%cBDKdaupyWK3#k1YK*^sxm@kYHbeBle>^}M_VjkmgWB)rLw z-!Fdgi$OT%t!uoxmY1mTcoKk62oMbAK{FQ}fv$%8ZYK*X{Js@FLS0eg7c2c2QsLJT z57XqzkbJ1CF_NaXb51WX4jIamDXW5WosfyPbG%q94$~@WBQt*0e-X}j9{AVM%d+xD zD)e`zaBWqrjCb7wgh8`i=qx%#e$gGCrRPnC=o`Ann?CtY1$xXcZw2ISew_*T5tcU~ z@?t1n{=_4i5O2~!SLr9}J2F5i)&<0RuD;2+j^x9jJrUX%XHFYS#{}M;as2Ve2iwT& zvPX;yYiWCP=ZY53OK z#E;`JNCJM9M~*xGJ1$!yNt}RR(p0^~qFB~8r~@Da3&AKVDhe;tL02*Oyda2|C-FQy zFFN7{JN)t@A6|6DR}ecA?&alM91{oO`I}b^^8zJa&_q9o>uhB^r05cDiiA$N_O)W?)_`9H> zpi3cqVPRqW--8AXvf;yr+puB7Y~a9w;h2x;@lFfbNb-c9r^p9rcWGOu(3kyJghHeO zS_9Zp(ca@A~d{R zv#~MxxEH9ZE;qhtid0r5pJ88Cy2Rruv4$3iNwd|~CGj-W)>!$ny+DU`#wO2o7EnoO>i98T{%Ep#mQ^ ztF7fLsdeE4W(jC}J_DbOfJ7k6_^_4#Ha0g|LsO%dGg(epUYr;(iuGI`^gbS)oHoDgB&){)=9%l;oBBy!(XN z!IhE>D-e+jN$#LuQAS=|#q2^%4muLZMQ;S6LOhcvPYyG>yo8Lp>q}^ux z`taV&_pPd?I`reR%8F16i``!Fcaw*yZE&0A!Z8}Qig?>o4{CqeoJ>hIxW z2H~8b>+!j4h{=_fLb)&gjgdJ>tasa+e4@eamfJ;+`KTjyl4}h9YAP$O&L!w!8x~iV z1#JSPL;2!CFz0e@qjeFzA{A0P-uOqO&|#P+262N4G;&uBoeABOO=R$xD%2b9v<99{ zDKb&t&}2*88J1U7xI?b=j^Lf8#g(98nZ5Gbv-Uqen%I7aPuM3v{VCgL|Gn*jNA3xP zqOsK-Y#<=5cI$07+Yv|n!1mpDUpwru!@>uK?zrR6J}Q01N2o9Ph~y}{^j8Pk{Um)Fce80m2iX*244oGixt`7ARfbxH?HKDFK)z8$dVo_mH59`O;a^Ugaj zgu$@XJ7FlPD7W)3xyVjF<8-TS@^Uy?sfFMKGlQ9Sa_He87#J&^?*X zUFRi2=J`yC!Qa$Vw*1gmT#v8Cq#}+Jp-=KFU^7Hea9rHOuZ}bCCp`>sd*61lP+L`P z=Ui~U{o>*aY<_i_E%ts~)>!9cJXu<5ZS;7&pCe69fX_Yu96SBIGi^cTGVd{vh?Sy` z1M;QdMA!r2_lEgEY+d(A1^J|Is1pbpgDz$$n2i`RWXOj|K)y)@vu+RzW-e~J>86lA zbx!?uN57cqShP6#B5S!5iE?*<*o;cIMbrp=dP(W>AgYe)DQ)AO_yWREgJiGT_kJ#LDx9=^k zft1~^H#j*)nwwU&yB|gLid3K&bQl_uHE7-^&zc^8uL)>V9iO}9UGgYD#DZm*T+rwS zhrCw#sHUu9xy_vQmOb_Kqc(DMu?-wtXoYL_+Iw82+&OZBW@5udZqm4JTE3WvJ6P*RN&pzL>(|>%j9d`IZcI33f zY`4$uoEVIv0vj`NwEf`d!|mwfjzKIr9LcyG|Aw( z0-p54W37K-KRfTzi$Xann`*7TRSv}+bS>YwD7VtmW%h$3jq%UAF^ef|$%}8_?7#7^InRBoIPeT8y|}G7r#O>y%qB9sKs@bZvvABzUYs_^q^(h@%BD6K zI3L1z^7T$A7R;Sz{fY-z@yKCT;^ca{8~1AOCxF?q(q-=G>Rso`D{bbx@3_voeoh=? z?<{a4=lzP=M1;PT%q&t~Cj)*=J`=~HU#?N^+;mTb346rAi#=vXcz@(m{95){okCxP zP(Wb#>MS}bJF1YlUR#jhNiUGa)RiK)?|`k^aX&{xS3&^cZ8WC5K$fnmF!cnF3H25pBMQAP`P{Lh9YLM($Hv27ca3vPU?q^9}{Nns@zeq6hf%wrQQcUUqOc*bDX?k z)RBed%YuqGJ8?#Yux!~zyjLX_38^4`<)JGm&(sZjz}Tbno$q`n%v7)!B2%&vgWd?~ zKwkLe3#aHddZvxSonwU$JlsCEG+1dxvSiB=EbRx_+tuRz8^W;Ii3WWdJF?6N5=bBz zUSHS(YzDKDfVLE=;F!8W?^IXlfSkUdrB&&K?1M-Ig1!=$bdqR2_0y}Qvchd=U6n0cp7fuxvQk^Rq}&~S_%i@qXlSgmXJ$O^zGApdoHW|Q zFAUf41Ho3hay8UDd3I@blF(e=V22-ah!41i+G%H$j$Uunq!DrC7)!nwhl5{Jpn_2>cG~kbM@E0-MM~HeP(F+w3dVki z`i3fd>E&m`EXBytL#)_GKs0tjyuzd$@J-evN0ovTd^B~@nc5YST~e| zM+~v^FFfBqm^a53`KTWpdeDJ3WY9n>b{XYi3J3IUH%lwP1$He8{{LSd+)z57?U^Ne#;8` z7uonp6RpyXREZlLy2XE9e8Kt;7+`}v?8s3gt#7}?0J3=vf=Jt9JQ!hN6xk8a#x!=l z^9zBMf$vCS=v>@6(GzA4nB`)IjSZUU5Cnkp*@!}KL^Mq@q$PFgq}oHqs%z0Ayi zHef^n;mLP#adCKhfcBvtan_7fInNVQM;&#PZMflv!GRHm5i?Jva6ceT`ZJ$f=g9&# z`Vr5RDN{lmEbjrrP!_HuJawQ=g3MeI_octO#0mF)HhjcTZ{J~7RK(0*fgSMu{r&!O zAH6RO{cD8l`vBLa2^)+Lj*xr#q&0Oye6%%rX0**Uo|3>hE?%@a?5BA1^*5}q>sRq` zAN>zUpZnTij}x6kH?O$taw}W5+=)evz4G!aer}`{I|+Gj?wmk+(diH7&JDV^<(6A{ zohA9Fyv#a2_n+t7u_QzdX9D3^*0$#<7vRKj;taglx5E=fDrZapR)>%VZB5;=*A4Q_ ztQh%Gn`eXf6z`*K`EIeMuo2z+{|fdyxpS?rwgdLx+lu-X*v)_ZLzqov`S`#?4zxbr z#v{j!w9U8LBBUMC_j4cL>Em|m?YAa>Gr;iQL4yZ|U+P7q_~3&N4)so+>GPBU-9iV@ zLB7t4&T`+xNt0}->-N^$ZWG!MR908n-uvzw?D9#j*R(C9hNabOuD#YqxSg5mgWeJ+ z7}M|klWns3Br9;895;TlZL!6UuES&PbD#gRJ@w4LY^)C~xyFEO^yn#`=5X6&;~ngu z|C|wKaF&-Xv=?4}%7%|F4)Kg0>$*BH*~2n=%s97ggYDP9zTRGa`E^^maEVnpX+Qa- z6Rdx6p`Cf&8CKI&=7Xskd*`iJZR5!k!+n#-d6|kmT|eG<%BJ@ED{nZNX|VD#uXooY zd>sP^kU8*Lmyj1y!HADh7><2iAQc*|bp<*WGYbi+NZ9d$g>(G>7AFty*dBX)&c==% zGrg{l~X)V~Ar<*dj!Osj>+hjC12X#3s9P^zGNjw%dN2U^q3k3ZZCWUxXh+ zDmK}~9k?s(X+J(S#PjaVcSHJL`O24U`k!vI*5<}AD^cpkZ1lL%HpGqPbN`uP^V|?N zd0tOD=N_?Yi5i?7j!?3TYm6@PSS~2H4Ne{Hcdr zVfWs5cTn1K6UWz6dJE4GOv+gHUH`d$!2YuiA zdmc_d;|$OL`#zc}4>Jy%Y`$4A<}bbSatQnUOD~4$>A}N?*%BYofalz45Ab_27L1U! z*30MuCrJL};4vZ?n&18IcVQZtHe*yPlF*sN;BV=RWKEKac-EpbQlXJlE}#rRP76a! zUEy@8ORjUC`ot(7;v*%I7B=ot*IR6{Mfj{X=SUmU#v_U+ope&jhx!uaj>i^h58}qS zL0TYtY|JMA04F?fkdvE{!-@rqP1L$J0<1)rU#uhFKoclQ3`$8O)4g9xO8ohMEx zl8x*Pn40YehaPA{2N&DLmt5$B1dpq=(M~(#)R5k;yMM+_M!nml#L;iH?N&C~2QWDL zIX=kYT1|b8U3vADHfr>!Fk463EZdPs^p{7mIVb&Q7Lh$q=o?Q_fhH#lS6+Qp7{u(n z>t}-Pfjk^^=pkXS^^>3f%*jq=IFD`nnJ$k_;YOhxNU6jy$`deW<2d@yl>5Y!-|JE z0oZ(^J^IgwJZzc0`~FP3>E_#9|HpfM^s&$Gx`(x3OC5Xf{VgX|1$N2h=Xo70vzu?e z&c=>%;_qbS&wsqt3uRWexXh}`YVA`y?rft+jIn<|{zSN!ev7XKU6I1Ijb4!oj1C3> zQh|}=TmM@Apfjy*WRePuGscdoE-7SZXu^5Bv0$ez&YmYTX3lyi{IW4C?BI30`R42H z0Q)&(9UeN)`~`D7ex~r6!zczr!_347AI!E*H{CcG&W$#n5;imvM>74JYzAZ6cZzqi z{{8#enZNj%H8|5>uy9_OS_BK`&$VA&a*-AL=xwNzi9thsjF!!ouzef}n~EyS-SPPd@o*otVsuYF-~k%ww#8veRb|Km72}2B)2NS~$<1B!~%htoq(`(@ooF z4EP(Tj)P~Zk-yOyHo8)`=och?b0?Nud*aE*!;)WhjT6_V8awUulWd!9x3r(1`4jJN z3F#g)W>g^Lk3arcC<}1|%0%923v3Gcg3vH>pKO)hj+Hg#Htm=rY~#%~vKwytUmwB05MshM-F?qpHg5bF+wHTT3Fl_cdN2I` z`-y*r{L!~H`er5#lF9%A8xF|lgAYC!+MoBm;8>~CYPTb=xel^ljP_?%6r6I}sn*x+ z$FHuq+!p#EgxL$0ULJ5_h%R7zARkLg7TKTwdSl4LzWW{Ogkpsc$Q79@^jjaLaEzY^9xb=8vqOr{CXY(wK36Txgq5 z-ogg-VHT&ajT}1KuDR-3tFB63_DJ8LJ|JN0GI~WSC?F$EIt_-A#$Vf{g0*BCAU7uG z9j6%ut@JW1aU)&j26#(5;)uh%0h6^umOdZ_Pd)YTpn!wCGqJ3&*Vn)1jQX=7jN-xI z+;!LALO2AXf&t=w!XS_kW;BYOot|*wae-uTeR;`Jue(nN1AF$_zi1y3jh`^qM}7mm zZtoAI1njcQR5yYH>=#Z-h#z9IZ21zq;l|&);T_~1bweLrJs9eMjtaxS?z-!OgZ|4M zciP62rvznw;mN11V)0U|uW<6?os?DteADcDPh!bBIFU>k#6yevtgD3q;FwK( zx$A8L9{Xh(0X+NMj4(q$J+sVJ;iEAeJbiP!?Y0Y#G4dihj5Q9M{Q=Xa9qA5tw3FMP z2i-y!M~@!qIyBghJo1Q8?+_PFwURgO)p*_q5iDD=Z1(z_Zv?~5W6N*8{gxBUExpZ$ z+CvZD=ONiI(-8I}9Xah#o3OzccjUjYrEY{7?U5d{c9&oNs}L_~kRC5^qAZXhoH3xT zAQj*I<~PINIE+_bdo7UIqT&G_+sd||zi>g=NXTa5J-@z}&6zvbp8e1B)~`QxRA{s2 zd|)5Uoo|2m!!5SiW?KXz@zj$uJpF{M0Q3W0WOE{Yl>V(jNIpnKCJD&5kVa>uLQj54 zs6AQ>LcvT0gu+Yh|FZ|m`y@{|cs(3`&;d5IxX3QP@I2R#3R~iBdEg-jSn=QicIDMq z*z+$vANGS`mu|lKM%!kat!(>`ZyV}^*(q!)q+-mN_+)v$gM{8`U3pTL4vepNk%Wd?SDQ?Th z+fhfKVDG#$Gvt9m<)CqkIl)7zPSJ5j{p|voU95BL=3XA}b9-t_h20xZu(Ahntk6@f} z1Q>4eLI(z2rGbIKm?mqc$xij7kC_&xV_EXxSIZi#4MPOj_W-{7)h`Ey-eHIBLpWw0 zSoXjm6ZXzKr+eObIwl$2V3Z&)2!<50aM48EK|di03m0jL zt~FenG-*O0rcXcpWIOqVG*6p$M9{GdFFfCMm7VleR#Lju9sk(i=+SBPDUoEoc>5i8 z2pb?@ZJViJ{|s%fUH%yNaTCT_VNoA@@|k~IT|=@=P~TK#KRoUz8&KTWZoKJtA-#%< z($EJcP1+#D#ViYYK)tXm#EccoG1wIJ1U*0>7`2WVIl@VTx6R8h*#sw=j8=(b{00-k z^f#kb@^-+1-w)H>%v`Wc2l<%d@iBwOMr}4HU+~Kdyq^#EG7fN}o9G$)DJHuU;+Q2v zYy+Sl^LRD0fVqItB{quoN6(Hs?zp!2i$XbKp^hb7hY<8eNCy%|e2}!i-*s0YgJ4P9 zyCrBd2^zz1vmZ73|MmXA)2Q%2f0Zs@>?+k{2k*a+4Jsl1MHKZ{g4flBdasU0cyln}l z!3Po5K2V?zXsg_;#F|I1_f!Cwri|+deX+L0 zF~A5jC3ej<*My}P)?t~}Wi*znamJxTEHJh&zw)BrzqozoB5cs~1}$k@UVtFXo-;Fq z;XL6ODKg8zwPYqC+2M}S;ctwQg!sUcC5!xjH^9|=+$VXg2@ns*8hz?M!f}mw$yW@P zTu5uy?3wOllK2_XGuos6am*MrMoSnsJ$F5K)=X>hx-4Dnd0EkH@4fqmEm)k8F|Tb~ zQtd3cwa%(r8iEot5@lVwrXpF_3r~nR7CDRN8Fq{|%Op4)IuN5s)-+>qjS&WM)w!`{ zy`PaBZHB&JfN3{~VF>RdMZja9lpz^4E^jA|JwdE@)RD*lkCQ?y^zti4bu3BX?4N%6 zX?yIk$J%wAzi|+d35Wtq0g&oUQb8D9Cmzbc22;Ya8S#Jv4hWmi2*Vx+>1-@3p<{j% z!7&5K0Lu)?yA;^#Q|daD%qT!GxX#kpk|l|*vM$aONl6?Pu5&E^l?L&#nxAcOonPXA zmwO*u^~oOEnPn`1(J!s^``dI%Znm_l%$Afa32X6aQ)5f5H8dwPe>`T*M(1Rv!moxt zlVGWjGy!^`HDvlSJ5aq1bb_*6f^@ZN=QXQC!8Rxs0(ir zZm3UmU`4~qS(>HZo|V;Q;cjBXHe#o!D}X(F38HR_SenGnG!Em1R7->c0z?yS;Ur_rBXcz`JnJLkQ8!uf7uOPbSb0Jxx6w}GI<%Tm?0ucp@PWB$T9rx=GI$p4Wmg6DPcqc2q&Slbp;#<@gp45IRQ=w zCjg;fJ@|nK9tcy|OzknsBt1>pb_Sd~%8$gy2%Q1pSks+}^T!ZZ2S<+)!tls2;@ov# z5;mFc(=P@p76=3J5l>jg@V^tn(b_%=714l@h-65(4megGAQFHVx->L2+6Qwm#PhvA zlDaD`Ew>6c))0qACku@pH^OGp6)T#ovLfk34YgI_T^Y?RMR;e&p(B7!M+c=bXvAe12%wIywf?f{lXdV<(tLuMcm5D16@ub0OF^MWEA zJsTb&st^Ro2Vwc;Jon%vxCXh{L|757^FkheWBubEW+tEm4UN@7s@y>!kPzZ)aRbH^ zQi0Gz`r6tPK?iDl7kYPG%{1 zDIc?8_uYS=&3NwFK&(Rjc$-ll637IODj^r_r%3dR9ThdM7ZvO?^U-~+JMVBFd*ybp z(H$!L!5|dOPEb!GUd=dAcY=-BE3ZQ~pjW#ctZQqhx;9b)WMgPo^2ZVfjupX~BFo$c zsy*3YPF~SFoH&olYqkv1AfN~6DuWzA7<569E)yqVrAz1)=g}7lV0jM#ngOFv@EhwQ z=^*GN`UXMKrce3|M4+TCA1q^{;{Z}YUfDda^2ZVec-|*zKk~fZkqQhxorVzvghC@c zHlHyP1--7?Gs;n7QB_M1fz0&lG`hA|Dc6 zmtoDEfdHDFz?gA=OekU^0TNP@IDLMpk7dhB!t>IVmDRRnvFDk`L;X6hCa-kE&r7r1 zr20sNj^G5sje>cdtYB1H=Z>n@<8xV2?|j*rI``JNp{VryE~+T8Wt6MUfT$zOXwxw` zjyo8gFBZT^6bFikKuQ23Q-H1s=vqENoH%@zg_xCKgBW_nQqifWo*LHfd8C-lqwGsS zN6}++nCW=*l8t7(d~Ng1HxIvDlRzH0kLi5&Z193D9?N7Zn&lm4DOhLM*HA^YlNqI$ zP;^8pS_64NFVR7C1fUn_6*_{RN#D>j>0hEd%&Jv;8(`3UM6_Uz|Ia%fn*8{Ka1SGR z%@_d;Zy15OLFE`@4r0U0F8J`h8vn=R(ltK9t*WgEc_K{+GI1w;Dk%rYlo7IsP-cK0 zBOo4-WFr+ElQ;4uJ(RH*;lQba8XhHQ-QVjd?)U1;oTpDAA{d%!gk0K6~PcCx%ZAurG}LLOhX!aLm$AN5XXoFu&MU2n70ueMD#3kc}=wl$oVq z362fb>^q=7K&Cd);}VhqsbFhnb9kF?eKY;o>xmf(A7wXr-68D5qF$gU3DDDIhPK)3 ztlXVuy*tzPPYZZR-fz+DM5D#q%LDqL#Zz$+u-mj<*dXrr@lq`WXIkU;@f(ia;S)xV z8)CtI{0*=_vO(wr;2J~$#9;wpGIc2?6*@-mMJD33SWeTu5mynm`4~#P%osk&R*pLgJiN52GJHqQ>uejn0JNMjk!#X%} z=9y=%B0R2!WS*zfK}-;id87=MfBp5>hY=qe>u?G<8Pe8B7R6Kpt!ZEwC7I~#+L{Kn zgJ3`seovlsR`OwV9pb~#MzD1ebZCeT)6=|J6DQA860GUt{Q0sdy35;A`9%*U{4L>W z6~bidIOn9tIFj3Mzdby?!ZHXm8BFElaB`VhOORRn@!zosiJ?c5W0|2?vfYp51DSCu zJ^ikGNa`qmd4nKB6d_9xSY~ipO5;&n>?50~ue|cg59yP1sw08;o^i$*!FKXD13By? zWa2l!`AyI>>@c%t%!FYdab!G>$>wUcf`q>_&$+b<|5je@Nb(D%PcCt!9A`^4DNtAF zb^rhY!AV3xRK@xSl`T_Nwo4-GBJx5i=roWDIu44A3R1If7=32lAQ>#L z#ri6d9_yO)RKk4_Uv!o|84wAkz?n@$Z$&1w)X_aCeB5TS#E4B0*YtPhJ+b5q+2Tf& zpZ^ZXMl$FTiDl-3r7xDq&=c9mY&J4eq-z|bXV^wWOL^!o@iW*WJ?t`rKg|lZKlR~x zi(BHuEwy$?g^;PKnOL8-NM(=ZqkvwM3XCBgMn;ielviuybRwKS`yiOgMljs5Fj|X6 z=M^4;g3v)^i0PoDqqMtklvAWgApHqzx~{JfR17iG{;^08Gxb<{%V;nbh(j#suJ#Mi zH_%-~hFjHZ+t ziI3xRv2c&>izU7%?G!2U4e3)XnK}sJz)(v>nsq&!K4(g%{&poP)<1OAzS;Cg_Apa2 z!^t-0s$VHmtWM-*`6Clj+^!TnMT-AMWa|q^iWDhQe6(Tq{{e$FI-!fuwQ~Rf002ov JPDHLkV1jBOQa=Cy literal 0 HcmV?d00001 diff --git a/leveldb/doc/impl.md b/leveldb/doc/impl.md new file mode 100644 index 000000000..c9bb62174 --- /dev/null +++ b/leveldb/doc/impl.md @@ -0,0 +1,172 @@ +## Files + +The implementation of leveldb is similar in spirit to the representation of a +single [Bigtable tablet (section 5.3)](https://research.google/pubs/pub27898/). +However the organization of the files that make up the representation is +somewhat different and is explained below. + +Each database is represented by a set of files stored in a directory. There are +several different types of files as documented below: + +### Log files + +A log file (*.log) stores a sequence of recent updates. Each update is appended +to the current log file. When the log file reaches a pre-determined size +(approximately 4MB by default), it is converted to a sorted table (see below) +and a new log file is created for future updates. + +A copy of the current log file is kept in an in-memory structure (the +`memtable`). This copy is consulted on every read so that read operations +reflect all logged updates. + +## Sorted tables + +A sorted table (*.ldb) stores a sequence of entries sorted by key. Each entry is +either a value for the key, or a deletion marker for the key. (Deletion markers +are kept around to hide obsolete values present in older sorted tables). + +The set of sorted tables are organized into a sequence of levels. The sorted +table generated from a log file is placed in a special **young** level (also +called level-0). When the number of young files exceeds a certain threshold +(currently four), all of the young files are merged together with all of the +overlapping level-1 files to produce a sequence of new level-1 files (we create +a new level-1 file for every 2MB of data.) + +Files in the young level may contain overlapping keys. However files in other +levels have distinct non-overlapping key ranges. Consider level number L where +L >= 1. When the combined size of files in level-L exceeds (10^L) MB (i.e., 10MB +for level-1, 100MB for level-2, ...), one file in level-L, and all of the +overlapping files in level-(L+1) are merged to form a set of new files for +level-(L+1). These merges have the effect of gradually migrating new updates +from the young level to the largest level using only bulk reads and writes +(i.e., minimizing expensive seeks). + +### Manifest + +A MANIFEST file lists the set of sorted tables that make up each level, the +corresponding key ranges, and other important metadata. A new MANIFEST file +(with a new number embedded in the file name) is created whenever the database +is reopened. The MANIFEST file is formatted as a log, and changes made to the +serving state (as files are added or removed) are appended to this log. + +### Current + +CURRENT is a simple text file that contains the name of the latest MANIFEST +file. + +### Info logs + +Informational messages are printed to files named LOG and LOG.old. + +### Others + +Other files used for miscellaneous purposes may also be present (LOCK, *.dbtmp). + +## Level 0 + +When the log file grows above a certain size (4MB by default): +Create a brand new memtable and log file and direct future updates here. + +In the background: + +1. Write the contents of the previous memtable to an sstable. +2. Discard the memtable. +3. Delete the old log file and the old memtable. +4. Add the new sstable to the young (level-0) level. + +## Compactions + +When the size of level L exceeds its limit, we compact it in a background +thread. The compaction picks a file from level L and all overlapping files from +the next level L+1. Note that if a level-L file overlaps only part of a +level-(L+1) file, the entire file at level-(L+1) is used as an input to the +compaction and will be discarded after the compaction. Aside: because level-0 +is special (files in it may overlap each other), we treat compactions from +level-0 to level-1 specially: a level-0 compaction may pick more than one +level-0 file in case some of these files overlap each other. + +A compaction merges the contents of the picked files to produce a sequence of +level-(L+1) files. We switch to producing a new level-(L+1) file after the +current output file has reached the target file size (2MB). We also switch to a +new output file when the key range of the current output file has grown enough +to overlap more than ten level-(L+2) files. This last rule ensures that a later +compaction of a level-(L+1) file will not pick up too much data from +level-(L+2). + +The old files are discarded and the new files are added to the serving state. + +Compactions for a particular level rotate through the key space. In more detail, +for each level L, we remember the ending key of the last compaction at level L. +The next compaction for level L will pick the first file that starts after this +key (wrapping around to the beginning of the key space if there is no such +file). + +Compactions drop overwritten values. They also drop deletion markers if there +are no higher numbered levels that contain a file whose range overlaps the +current key. + +### Timing + +Level-0 compactions will read up to four 1MB files from level-0, and at worst +all the level-1 files (10MB). I.e., we will read 14MB and write 14MB. + +Other than the special level-0 compactions, we will pick one 2MB file from level +L. In the worst case, this will overlap ~ 12 files from level L+1 (10 because +level-(L+1) is ten times the size of level-L, and another two at the boundaries +since the file ranges at level-L will usually not be aligned with the file +ranges at level-L+1). The compaction will therefore read 26MB and write 26MB. +Assuming a disk IO rate of 100MB/s (ballpark range for modern drives), the worst +compaction cost will be approximately 0.5 second. + +If we throttle the background writing to something small, say 10% of the full +100MB/s speed, a compaction may take up to 5 seconds. If the user is writing at +10MB/s, we might build up lots of level-0 files (~50 to hold the 5*10MB). This +may significantly increase the cost of reads due to the overhead of merging more +files together on every read. + +Solution 1: To reduce this problem, we might want to increase the log switching +threshold when the number of level-0 files is large. Though the downside is that +the larger this threshold, the more memory we will need to hold the +corresponding memtable. + +Solution 2: We might want to decrease write rate artificially when the number of +level-0 files goes up. + +Solution 3: We work on reducing the cost of very wide merges. Perhaps most of +the level-0 files will have their blocks sitting uncompressed in the cache and +we will only need to worry about the O(N) complexity in the merging iterator. + +### Number of files + +Instead of always making 2MB files, we could make larger files for larger levels +to reduce the total file count, though at the expense of more bursty +compactions. Alternatively, we could shard the set of files into multiple +directories. + +An experiment on an ext3 filesystem on Feb 04, 2011 shows the following timings +to do 100K file opens in directories with varying number of files: + + +| Files in directory | Microseconds to open a file | +|-------------------:|----------------------------:| +| 1000 | 9 | +| 10000 | 10 | +| 100000 | 16 | + +So maybe even the sharding is not necessary on modern filesystems? + +## Recovery + +* Read CURRENT to find name of the latest committed MANIFEST +* Read the named MANIFEST file +* Clean up stale files +* We could open all sstables here, but it is probably better to be lazy... +* Convert log chunk to a new level-0 sstable +* Start directing new writes to a new log file with recovered sequence# + +## Garbage collection of files + +`RemoveObsoleteFiles()` is called at the end of every compaction and at the end +of recovery. It finds the names of all files in the database. It deletes all log +files that are not the current log file. It deletes all table files that are not +referenced from some level and are not the output of an active compaction. diff --git a/leveldb/doc/index.md b/leveldb/doc/index.md new file mode 100644 index 000000000..0f6d64917 --- /dev/null +++ b/leveldb/doc/index.md @@ -0,0 +1,524 @@ +leveldb +======= + +_Jeff Dean, Sanjay Ghemawat_ + +The leveldb library provides a persistent key value store. Keys and values are +arbitrary byte arrays. The keys are ordered within the key value store +according to a user-specified comparator function. + +## Opening A Database + +A leveldb database has a name which corresponds to a file system directory. All +of the contents of database are stored in this directory. The following example +shows how to open a database, creating it if necessary: + +```c++ +#include +#include "leveldb/db.h" + +leveldb::DB* db; +leveldb::Options options; +options.create_if_missing = true; +leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); +assert(status.ok()); +... +``` + +If you want to raise an error if the database already exists, add the following +line before the `leveldb::DB::Open` call: + +```c++ +options.error_if_exists = true; +``` + +## Status + +You may have noticed the `leveldb::Status` type above. Values of this type are +returned by most functions in leveldb that may encounter an error. You can check +if such a result is ok, and also print an associated error message: + +```c++ +leveldb::Status s = ...; +if (!s.ok()) cerr << s.ToString() << endl; +``` + +## Closing A Database + +When you are done with a database, just delete the database object. Example: + +```c++ +... open the db as described above ... +... do something with db ... +delete db; +``` + +## Reads And Writes + +The database provides Put, Delete, and Get methods to modify/query the database. +For example, the following code moves the value stored under key1 to key2. + +```c++ +std::string value; +leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value); +if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value); +if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1); +``` + +## Atomic Updates + +Note that if the process dies after the Put of key2 but before the delete of +key1, the same value may be left stored under multiple keys. Such problems can +be avoided by using the `WriteBatch` class to atomically apply a set of updates: + +```c++ +#include "leveldb/write_batch.h" +... +std::string value; +leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value); +if (s.ok()) { + leveldb::WriteBatch batch; + batch.Delete(key1); + batch.Put(key2, value); + s = db->Write(leveldb::WriteOptions(), &batch); +} +``` + +The `WriteBatch` holds a sequence of edits to be made to the database, and these +edits within the batch are applied in order. Note that we called Delete before +Put so that if key1 is identical to key2, we do not end up erroneously dropping +the value entirely. + +Apart from its atomicity benefits, `WriteBatch` may also be used to speed up +bulk updates by placing lots of individual mutations into the same batch. + +## Synchronous Writes + +By default, each write to leveldb is asynchronous: it returns after pushing the +write from the process into the operating system. The transfer from operating +system memory to the underlying persistent storage happens asynchronously. The +sync flag can be turned on for a particular write to make the write operation +not return until the data being written has been pushed all the way to +persistent storage. (On Posix systems, this is implemented by calling either +`fsync(...)` or `fdatasync(...)` or `msync(..., MS_SYNC)` before the write +operation returns.) + +```c++ +leveldb::WriteOptions write_options; +write_options.sync = true; +db->Put(write_options, ...); +``` + +Asynchronous writes are often more than a thousand times as fast as synchronous +writes. The downside of asynchronous writes is that a crash of the machine may +cause the last few updates to be lost. Note that a crash of just the writing +process (i.e., not a reboot) will not cause any loss since even when sync is +false, an update is pushed from the process memory into the operating system +before it is considered done. + +Asynchronous writes can often be used safely. For example, when loading a large +amount of data into the database you can handle lost updates by restarting the +bulk load after a crash. A hybrid scheme is also possible where every Nth write +is synchronous, and in the event of a crash, the bulk load is restarted just +after the last synchronous write finished by the previous run. (The synchronous +write can update a marker that describes where to restart on a crash.) + +`WriteBatch` provides an alternative to asynchronous writes. Multiple updates +may be placed in the same WriteBatch and applied together using a synchronous +write (i.e., `write_options.sync` is set to true). The extra cost of the +synchronous write will be amortized across all of the writes in the batch. + +## Concurrency + +A database may only be opened by one process at a time. The leveldb +implementation acquires a lock from the operating system to prevent misuse. +Within a single process, the same `leveldb::DB` object may be safely shared by +multiple concurrent threads. I.e., different threads may write into or fetch +iterators or call Get on the same database without any external synchronization +(the leveldb implementation will automatically do the required synchronization). +However other objects (like Iterator and `WriteBatch`) may require external +synchronization. If two threads share such an object, they must protect access +to it using their own locking protocol. More details are available in the public +header files. + +## Iteration + +The following example demonstrates how to print all key,value pairs in a +database. + +```c++ +leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions()); +for (it->SeekToFirst(); it->Valid(); it->Next()) { + cout << it->key().ToString() << ": " << it->value().ToString() << endl; +} +assert(it->status().ok()); // Check for any errors found during the scan +delete it; +``` + +The following variation shows how to process just the keys in the range +[start,limit): + +```c++ +for (it->Seek(start); + it->Valid() && it->key().ToString() < limit; + it->Next()) { + ... +} +``` + +You can also process entries in reverse order. (Caveat: reverse iteration may be +somewhat slower than forward iteration.) + +```c++ +for (it->SeekToLast(); it->Valid(); it->Prev()) { + ... +} +``` + +## Snapshots + +Snapshots provide consistent read-only views over the entire state of the +key-value store. `ReadOptions::snapshot` may be non-NULL to indicate that a +read should operate on a particular version of the DB state. If +`ReadOptions::snapshot` is NULL, the read will operate on an implicit snapshot +of the current state. + +Snapshots are created by the `DB::GetSnapshot()` method: + +```c++ +leveldb::ReadOptions options; +options.snapshot = db->GetSnapshot(); +... apply some updates to db ... +leveldb::Iterator* iter = db->NewIterator(options); +... read using iter to view the state when the snapshot was created ... +delete iter; +db->ReleaseSnapshot(options.snapshot); +``` + +Note that when a snapshot is no longer needed, it should be released using the +`DB::ReleaseSnapshot` interface. This allows the implementation to get rid of +state that was being maintained just to support reading as of that snapshot. + +## Slice + +The return value of the `it->key()` and `it->value()` calls above are instances +of the `leveldb::Slice` type. Slice is a simple structure that contains a length +and a pointer to an external byte array. Returning a Slice is a cheaper +alternative to returning a `std::string` since we do not need to copy +potentially large keys and values. In addition, leveldb methods do not return +null-terminated C-style strings since leveldb keys and values are allowed to +contain `'\0'` bytes. + +C++ strings and null-terminated C-style strings can be easily converted to a +Slice: + +```c++ +leveldb::Slice s1 = "hello"; + +std::string str("world"); +leveldb::Slice s2 = str; +``` + +A Slice can be easily converted back to a C++ string: + +```c++ +std::string str = s1.ToString(); +assert(str == std::string("hello")); +``` + +Be careful when using Slices since it is up to the caller to ensure that the +external byte array into which the Slice points remains live while the Slice is +in use. For example, the following is buggy: + +```c++ +leveldb::Slice slice; +if (...) { + std::string str = ...; + slice = str; +} +Use(slice); +``` + +When the if statement goes out of scope, str will be destroyed and the backing +storage for slice will disappear. + +## Comparators + +The preceding examples used the default ordering function for key, which orders +bytes lexicographically. You can however supply a custom comparator when opening +a database. For example, suppose each database key consists of two numbers and +we should sort by the first number, breaking ties by the second number. First, +define a proper subclass of `leveldb::Comparator` that expresses these rules: + +```c++ +class TwoPartComparator : public leveldb::Comparator { + public: + // Three-way comparison function: + // if a < b: negative result + // if a > b: positive result + // else: zero result + int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const { + int a1, a2, b1, b2; + ParseKey(a, &a1, &a2); + ParseKey(b, &b1, &b2); + if (a1 < b1) return -1; + if (a1 > b1) return +1; + if (a2 < b2) return -1; + if (a2 > b2) return +1; + return 0; + } + + // Ignore the following methods for now: + const char* Name() const { return "TwoPartComparator"; } + void FindShortestSeparator(std::string*, const leveldb::Slice&) const {} + void FindShortSuccessor(std::string*) const {} +}; +``` + +Now create a database using this custom comparator: + +```c++ +TwoPartComparator cmp; +leveldb::DB* db; +leveldb::Options options; +options.create_if_missing = true; +options.comparator = &cmp; +leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); +... +``` + +### Backwards compatibility + +The result of the comparator's Name method is attached to the database when it +is created, and is checked on every subsequent database open. If the name +changes, the `leveldb::DB::Open` call will fail. Therefore, change the name if +and only if the new key format and comparison function are incompatible with +existing databases, and it is ok to discard the contents of all existing +databases. + +You can however still gradually evolve your key format over time with a little +bit of pre-planning. For example, you could store a version number at the end of +each key (one byte should suffice for most uses). When you wish to switch to a +new key format (e.g., adding an optional third part to the keys processed by +`TwoPartComparator`), (a) keep the same comparator name (b) increment the +version number for new keys (c) change the comparator function so it uses the +version numbers found in the keys to decide how to interpret them. + +## Performance + +Performance can be tuned by changing the default values of the types defined in +`include/options.h`. + +### Block size + +leveldb groups adjacent keys together into the same block and such a block is +the unit of transfer to and from persistent storage. The default block size is +approximately 4096 uncompressed bytes. Applications that mostly do bulk scans +over the contents of the database may wish to increase this size. Applications +that do a lot of point reads of small values may wish to switch to a smaller +block size if performance measurements indicate an improvement. There isn't much +benefit in using blocks smaller than one kilobyte, or larger than a few +megabytes. Also note that compression will be more effective with larger block +sizes. + +### Compression + +Each block is individually compressed before being written to persistent +storage. Compression is on by default since the default compression method is +very fast, and is automatically disabled for uncompressible data. In rare cases, +applications may want to disable compression entirely, but should only do so if +benchmarks show a performance improvement: + +```c++ +leveldb::Options options; +options.compression = leveldb::kNoCompression; +... leveldb::DB::Open(options, name, ...) .... +``` + +### Cache + +The contents of the database are stored in a set of files in the filesystem and +each file stores a sequence of compressed blocks. If options.block_cache is +non-NULL, it is used to cache frequently used uncompressed block contents. + +```c++ +#include "leveldb/cache.h" + +leveldb::Options options; +options.block_cache = leveldb::NewLRUCache(100 * 1048576); // 100MB cache +leveldb::DB* db; +leveldb::DB::Open(options, name, &db); +... use the db ... +delete db +delete options.block_cache; +``` + +Note that the cache holds uncompressed data, and therefore it should be sized +according to application level data sizes, without any reduction from +compression. (Caching of compressed blocks is left to the operating system +buffer cache, or any custom Env implementation provided by the client.) + +When performing a bulk read, the application may wish to disable caching so that +the data processed by the bulk read does not end up displacing most of the +cached contents. A per-iterator option can be used to achieve this: + +```c++ +leveldb::ReadOptions options; +options.fill_cache = false; +leveldb::Iterator* it = db->NewIterator(options); +for (it->SeekToFirst(); it->Valid(); it->Next()) { + ... +} +delete it; +``` + +### Key Layout + +Note that the unit of disk transfer and caching is a block. Adjacent keys +(according to the database sort order) will usually be placed in the same block. +Therefore the application can improve its performance by placing keys that are +accessed together near each other and placing infrequently used keys in a +separate region of the key space. + +For example, suppose we are implementing a simple file system on top of leveldb. +The types of entries we might wish to store are: + + filename -> permission-bits, length, list of file_block_ids + file_block_id -> data + +We might want to prefix filename keys with one letter (say '/') and the +`file_block_id` keys with a different letter (say '0') so that scans over just +the metadata do not force us to fetch and cache bulky file contents. + +### Filters + +Because of the way leveldb data is organized on disk, a single `Get()` call may +involve multiple reads from disk. The optional FilterPolicy mechanism can be +used to reduce the number of disk reads substantially. + +```c++ +leveldb::Options options; +options.filter_policy = NewBloomFilterPolicy(10); +leveldb::DB* db; +leveldb::DB::Open(options, "/tmp/testdb", &db); +... use the database ... +delete db; +delete options.filter_policy; +``` + +The preceding code associates a Bloom filter based filtering policy with the +database. Bloom filter based filtering relies on keeping some number of bits of +data in memory per key (in this case 10 bits per key since that is the argument +we passed to `NewBloomFilterPolicy`). This filter will reduce the number of +unnecessary disk reads needed for Get() calls by a factor of approximately +a 100. Increasing the bits per key will lead to a larger reduction at the cost +of more memory usage. We recommend that applications whose working set does not +fit in memory and that do a lot of random reads set a filter policy. + +If you are using a custom comparator, you should ensure that the filter policy +you are using is compatible with your comparator. For example, consider a +comparator that ignores trailing spaces when comparing keys. +`NewBloomFilterPolicy` must not be used with such a comparator. Instead, the +application should provide a custom filter policy that also ignores trailing +spaces. For example: + +```c++ +class CustomFilterPolicy : public leveldb::FilterPolicy { + private: + leveldb::FilterPolicy* builtin_policy_; + + public: + CustomFilterPolicy() : builtin_policy_(leveldb::NewBloomFilterPolicy(10)) {} + ~CustomFilterPolicy() { delete builtin_policy_; } + + const char* Name() const { return "IgnoreTrailingSpacesFilter"; } + + void CreateFilter(const leveldb::Slice* keys, int n, std::string* dst) const { + // Use builtin bloom filter code after removing trailing spaces + std::vector trimmed(n); + for (int i = 0; i < n; i++) { + trimmed[i] = RemoveTrailingSpaces(keys[i]); + } + builtin_policy_->CreateFilter(trimmed.data(), n, dst); + } +}; +``` + +Advanced applications may provide a filter policy that does not use a bloom +filter but uses some other mechanism for summarizing a set of keys. See +`leveldb/filter_policy.h` for detail. + +## Checksums + +leveldb associates checksums with all data it stores in the file system. There +are two separate controls provided over how aggressively these checksums are +verified: + +`ReadOptions::verify_checksums` may be set to true to force checksum +verification of all data that is read from the file system on behalf of a +particular read. By default, no such verification is done. + +`Options::paranoid_checks` may be set to true before opening a database to make +the database implementation raise an error as soon as it detects an internal +corruption. Depending on which portion of the database has been corrupted, the +error may be raised when the database is opened, or later by another database +operation. By default, paranoid checking is off so that the database can be used +even if parts of its persistent storage have been corrupted. + +If a database is corrupted (perhaps it cannot be opened when paranoid checking +is turned on), the `leveldb::RepairDB` function may be used to recover as much +of the data as possible + +## Approximate Sizes + +The `GetApproximateSizes` method can used to get the approximate number of bytes +of file system space used by one or more key ranges. + +```c++ +leveldb::Range ranges[2]; +ranges[0] = leveldb::Range("a", "c"); +ranges[1] = leveldb::Range("x", "z"); +uint64_t sizes[2]; +db->GetApproximateSizes(ranges, 2, sizes); +``` + +The preceding call will set `sizes[0]` to the approximate number of bytes of +file system space used by the key range `[a..c)` and `sizes[1]` to the +approximate number of bytes used by the key range `[x..z)`. + +## Environment + +All file operations (and other operating system calls) issued by the leveldb +implementation are routed through a `leveldb::Env` object. Sophisticated clients +may wish to provide their own Env implementation to get better control. +For example, an application may introduce artificial delays in the file IO +paths to limit the impact of leveldb on other activities in the system. + +```c++ +class SlowEnv : public leveldb::Env { + ... implementation of the Env interface ... +}; + +SlowEnv env; +leveldb::Options options; +options.env = &env; +Status s = leveldb::DB::Open(options, ...); +``` + +## Porting + +leveldb may be ported to a new platform by providing platform specific +implementations of the types/methods/functions exported by +`leveldb/port/port.h`. See `leveldb/port/port_example.h` for more details. + +In addition, the new platform may need a new default `leveldb::Env` +implementation. See `leveldb/util/env_posix.h` for an example. + +## Other Information + +Details about the leveldb implementation may be found in the following +documents: + +1. [Implementation notes](impl.md) +2. [Format of an immutable Table file](table_format.md) +3. [Format of a log file](log_format.md) diff --git a/leveldb/doc/log_format.md b/leveldb/doc/log_format.md new file mode 100644 index 000000000..f32cb5d7d --- /dev/null +++ b/leveldb/doc/log_format.md @@ -0,0 +1,75 @@ +leveldb Log format +================== +The log file contents are a sequence of 32KB blocks. The only exception is that +the tail of the file may contain a partial block. + +Each block consists of a sequence of records: + + block := record* trailer? + record := + checksum: uint32 // crc32c of type and data[] ; little-endian + length: uint16 // little-endian + type: uint8 // One of FULL, FIRST, MIDDLE, LAST + data: uint8[length] + +A record never starts within the last six bytes of a block (since it won't fit). +Any leftover bytes here form the trailer, which must consist entirely of zero +bytes and must be skipped by readers. + +Aside: if exactly seven bytes are left in the current block, and a new non-zero +length record is added, the writer must emit a FIRST record (which contains zero +bytes of user data) to fill up the trailing seven bytes of the block and then +emit all of the user data in subsequent blocks. + +More types may be added in the future. Some Readers may skip record types they +do not understand, others may report that some data was skipped. + + FULL == 1 + FIRST == 2 + MIDDLE == 3 + LAST == 4 + +The FULL record contains the contents of an entire user record. + +FIRST, MIDDLE, LAST are types used for user records that have been split into +multiple fragments (typically because of block boundaries). FIRST is the type +of the first fragment of a user record, LAST is the type of the last fragment of +a user record, and MIDDLE is the type of all interior fragments of a user +record. + +Example: consider a sequence of user records: + + A: length 1000 + B: length 97270 + C: length 8000 + +**A** will be stored as a FULL record in the first block. + +**B** will be split into three fragments: first fragment occupies the rest of +the first block, second fragment occupies the entirety of the second block, and +the third fragment occupies a prefix of the third block. This will leave six +bytes free in the third block, which will be left empty as the trailer. + +**C** will be stored as a FULL record in the fourth block. + +---- + +## Some benefits over the recordio format: + +1. We do not need any heuristics for resyncing - just go to next block boundary + and scan. If there is a corruption, skip to the next block. As a + side-benefit, we do not get confused when part of the contents of one log + file are embedded as a record inside another log file. + +2. Splitting at approximate boundaries (e.g., for mapreduce) is simple: find the + next block boundary and skip records until we hit a FULL or FIRST record. + +3. We do not need extra buffering for large records. + +## Some downsides compared to recordio format: + +1. No packing of tiny records. This could be fixed by adding a new record type, + so it is a shortcoming of the current implementation, not necessarily the + format. + +2. No compression. Again, this could be fixed by adding new record types. diff --git a/leveldb/doc/project1-LSMTree.md b/leveldb/doc/project1-LSMTree.md new file mode 100644 index 000000000..6ec225d05 --- /dev/null +++ b/leveldb/doc/project1-LSMTree.md @@ -0,0 +1,54 @@ +# Project1 LSM Tree +In this project, you will learn the basic architecture of LSM tree and finish the implementation of LevelDB, a representative LSM-tree-based key/value storage engine. + +The log-structured merge-tree (also known as LSM tree, or LSMT) is a data structure with performance characteristics that make it attractive for providing indexed access to files with high insert volume, such as transactional log data. LSM trees, like other search trees, maintain key-value pairs. LSM trees maintain data in two or more separate structures, each of which is optimized for its respective underlying storage medium; data is synchronized between the two structures efficiently, in batches. + +LSM tree is widely used in mainstream persistent KV storage systems like LevelDB and RocksDB. RocksDB, which is used as TiKV's storage engine, borrows significant code from the open source [Leveldb](https://code.google.com/google/leveldb/) project and does a lot of performance optimization. However, RocksDB has a higher amount of code and is more difficult to learn. As a beginner's course for KV storage, this project is based on LevelDB. + +### Architecture +![lsmtree](imgs/lsmtree.png) + +A brief introduction to the architecture of LSM tree is provided later in this article. For more details, you can also read our collection of documents at . + + +#### Memtable +`Memtable` is an in-memory component in which the data incoming from client requests is first written, which is basically a skip-list, an ordered data structure. After the skip-list has got enough entries and has hit its threshold memory it is transformed into an immutable Memtable, and then it waits to be flushed to the Disk, sorted in form of `SSTable`. + +#### SSTable +`SSTable` or sorted string table as the name explains contains the entry in sorted format on keys on disk. When the sorted data from RAM is flushed to disk it is stored in form of `SSTable`. SSTables are divided into different levels, with lower levels storing newer entries and higher levels storing older entries. The SSTable within each level are ordered(except Level0) and the SSTables between the different levels are disordered. After the memtable has reach its threshold size, it is first flushed to Level0. After a while the entries will be moved to a higher level by `compaction`. + +#### Log +`Log` in LevelDB is a write ahead log. As mentioned earlier, entries written by LevelDB is first saved to MemTable. To prevent data loss due to downtime, data is persisted to the log file before being written to MemTable. After Memtable is flushed to the disk, related data can be deleted from the log. + +### The Code +In this part, you will finish the implementation of LevelDB, which involves three important operations in LSM tree: Get/Compaction/Scan. It maintains a simple database of key/value pairs. Keys and values are strings. `Get` queries the database and fetches the newest value for a key. There may be multiple versions of the same key in the database because of the append nature of the LSM tree. `Compaction` merges some files into the next layer and does garbage collection. `Scan` fetches the current value for a series of keys. + +#### 1. Get +The code you need to implement is in `db/db_impl.cc` and `db/version_set.cc`. +- Inside `db/db_impl.cc`, you need to complete the function `DBImpl::Get` (Blank1.1), which searches Memtable, immutable Memtable, and SSTable until the key is found and returns the value. +- Inside `db/version_set.cc`, you need to complete the function `Version::ForEachOverlapping` (Blank1.2) and the function `Version::Get` (Blank1.3). + +#### 2. Compaction +In this part, the code you need to implement is in `db/db_impl.cc` and `db/version_set.cc`. +- Inside `db/db_impl.cc`, you need to complete the function `DBImpl::BackgroundCompaction` (Blank2.1) and the function `DBImpl::InstallCompactionResults` (Blank2.2). +- Inside `db/version_set.cc`, you need to complete the function `VersionSet::Finalize` (Blank2.3) and the function `VersionSet::PickCompaction` (Blank2.4) + +#### 3. Scan +In this part, the code you need to implement is in `db/version_set.cc` and `table/merger.cc`. +- Inside `db/version_set.cc`, you need to complete the function `Version::AddIterators` (Blank3.1). +- Inside `table/merger.cc`, you need to complete the function `Seek` (Blank3.2). + +### Test + +Make: +```bash +mkdir -p build && cd build +cmake -DCMAKE_BUILD_TYPE=Release .. && cmake --build . +``` +Please see the CMake documentation and `CMakeLists.txt` for more advanced usage. + +Test: +```bash +./db_bench +``` +Please see the CMake documentation and `leveldb/benchmarks/db_bench.cc` for more advanced usage. \ No newline at end of file diff --git a/leveldb/doc/table_format.md b/leveldb/doc/table_format.md new file mode 100644 index 000000000..5fe7e7241 --- /dev/null +++ b/leveldb/doc/table_format.md @@ -0,0 +1,107 @@ +leveldb File format +=================== + + + [data block 1] + [data block 2] + ... + [data block N] + [meta block 1] + ... + [meta block K] + [metaindex block] + [index block] + [Footer] (fixed size; starts at file_size - sizeof(Footer)) + + +The file contains internal pointers. Each such pointer is called +a BlockHandle and contains the following information: + + offset: varint64 + size: varint64 + +See [varints](https://developers.google.com/protocol-buffers/docs/encoding#varints) +for an explanation of varint64 format. + +1. The sequence of key/value pairs in the file are stored in sorted +order and partitioned into a sequence of data blocks. These blocks +come one after another at the beginning of the file. Each data block +is formatted according to the code in `block_builder.cc`, and then +optionally compressed. + +2. After the data blocks we store a bunch of meta blocks. The +supported meta block types are described below. More meta block types +may be added in the future. Each meta block is again formatted using +`block_builder.cc` and then optionally compressed. + +3. A "metaindex" block. It contains one entry for every other meta +block where the key is the name of the meta block and the value is a +BlockHandle pointing to that meta block. + +4. An "index" block. This block contains one entry per data block, +where the key is a string >= last key in that data block and before +the first key in the successive data block. The value is the +BlockHandle for the data block. + +5. At the very end of the file is a fixed length footer that contains +the BlockHandle of the metaindex and index blocks as well as a magic number. + + metaindex_handle: char[p]; // Block handle for metaindex + index_handle: char[q]; // Block handle for index + padding: char[40-p-q];// zeroed bytes to make fixed length + // (40==2*BlockHandle::kMaxEncodedLength) + magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) + +## "filter" Meta Block + +If a `FilterPolicy` was specified when the database was opened, a +filter block is stored in each table. The "metaindex" block contains +an entry that maps from `filter.` to the BlockHandle for the filter +block where `` is the string returned by the filter policy's +`Name()` method. + +The filter block stores a sequence of filters, where filter i contains +the output of `FilterPolicy::CreateFilter()` on all keys that are stored +in a block whose file offset falls within the range + + [ i*base ... (i+1)*base-1 ] + +Currently, "base" is 2KB. So for example, if blocks X and Y start in +the range `[ 0KB .. 2KB-1 ]`, all of the keys in X and Y will be +converted to a filter by calling `FilterPolicy::CreateFilter()`, and the +resulting filter will be stored as the first filter in the filter +block. + +The filter block is formatted as follows: + + [filter 0] + [filter 1] + [filter 2] + ... + [filter N-1] + + [offset of filter 0] : 4 bytes + [offset of filter 1] : 4 bytes + [offset of filter 2] : 4 bytes + ... + [offset of filter N-1] : 4 bytes + + [offset of beginning of offset array] : 4 bytes + lg(base) : 1 byte + +The offset array at the end of the filter block allows efficient +mapping from a data block offset to the corresponding filter. + +## "stats" Meta Block + +This meta block contains a bunch of stats. The key is the name +of the statistic. The value contains the statistic. + +TODO(postrelease): record following stats. + + data size + index size + key size (uncompressed) + value size (uncompressed) + number of entries + number of data blocks diff --git a/leveldb/helpers/memenv/memenv.cc b/leveldb/helpers/memenv/memenv.cc new file mode 100644 index 000000000..e47661330 --- /dev/null +++ b/leveldb/helpers/memenv/memenv.cc @@ -0,0 +1,390 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "helpers/memenv/memenv.h" + +#include +#include +#include +#include +#include + +#include "leveldb/env.h" +#include "leveldb/status.h" +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/mutexlock.h" + +namespace leveldb { + +namespace { + +class FileState { + public: + // FileStates are reference counted. The initial reference count is zero + // and the caller must call Ref() at least once. + FileState() : refs_(0), size_(0) {} + + // No copying allowed. + FileState(const FileState&) = delete; + FileState& operator=(const FileState&) = delete; + + // Increase the reference count. + void Ref() { + MutexLock lock(&refs_mutex_); + ++refs_; + } + + // Decrease the reference count. Delete if this is the last reference. + void Unref() { + bool do_delete = false; + + { + MutexLock lock(&refs_mutex_); + --refs_; + assert(refs_ >= 0); + if (refs_ <= 0) { + do_delete = true; + } + } + + if (do_delete) { + delete this; + } + } + + uint64_t Size() const { + MutexLock lock(&blocks_mutex_); + return size_; + } + + void Truncate() { + MutexLock lock(&blocks_mutex_); + for (char*& block : blocks_) { + delete[] block; + } + blocks_.clear(); + size_ = 0; + } + + Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { + MutexLock lock(&blocks_mutex_); + if (offset > size_) { + return Status::IOError("Offset greater than file size."); + } + const uint64_t available = size_ - offset; + if (n > available) { + n = static_cast(available); + } + if (n == 0) { + *result = Slice(); + return Status::OK(); + } + + assert(offset / kBlockSize <= std::numeric_limits::max()); + size_t block = static_cast(offset / kBlockSize); + size_t block_offset = offset % kBlockSize; + size_t bytes_to_copy = n; + char* dst = scratch; + + while (bytes_to_copy > 0) { + size_t avail = kBlockSize - block_offset; + if (avail > bytes_to_copy) { + avail = bytes_to_copy; + } + std::memcpy(dst, blocks_[block] + block_offset, avail); + + bytes_to_copy -= avail; + dst += avail; + block++; + block_offset = 0; + } + + *result = Slice(scratch, n); + return Status::OK(); + } + + Status Append(const Slice& data) { + const char* src = data.data(); + size_t src_len = data.size(); + + MutexLock lock(&blocks_mutex_); + while (src_len > 0) { + size_t avail; + size_t offset = size_ % kBlockSize; + + if (offset != 0) { + // There is some room in the last block. + avail = kBlockSize - offset; + } else { + // No room in the last block; push new one. + blocks_.push_back(new char[kBlockSize]); + avail = kBlockSize; + } + + if (avail > src_len) { + avail = src_len; + } + std::memcpy(blocks_.back() + offset, src, avail); + src_len -= avail; + src += avail; + size_ += avail; + } + + return Status::OK(); + } + + private: + enum { kBlockSize = 8 * 1024 }; + + // Private since only Unref() should be used to delete it. + ~FileState() { Truncate(); } + + port::Mutex refs_mutex_; + int refs_ GUARDED_BY(refs_mutex_); + + mutable port::Mutex blocks_mutex_; + std::vector blocks_ GUARDED_BY(blocks_mutex_); + uint64_t size_ GUARDED_BY(blocks_mutex_); +}; + +class SequentialFileImpl : public SequentialFile { + public: + explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) { + file_->Ref(); + } + + ~SequentialFileImpl() override { file_->Unref(); } + + Status Read(size_t n, Slice* result, char* scratch) override { + Status s = file_->Read(pos_, n, result, scratch); + if (s.ok()) { + pos_ += result->size(); + } + return s; + } + + Status Skip(uint64_t n) override { + if (pos_ > file_->Size()) { + return Status::IOError("pos_ > file_->Size()"); + } + const uint64_t available = file_->Size() - pos_; + if (n > available) { + n = available; + } + pos_ += n; + return Status::OK(); + } + + private: + FileState* file_; + uint64_t pos_; +}; + +class RandomAccessFileImpl : public RandomAccessFile { + public: + explicit RandomAccessFileImpl(FileState* file) : file_(file) { file_->Ref(); } + + ~RandomAccessFileImpl() override { file_->Unref(); } + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + return file_->Read(offset, n, result, scratch); + } + + private: + FileState* file_; +}; + +class WritableFileImpl : public WritableFile { + public: + WritableFileImpl(FileState* file) : file_(file) { file_->Ref(); } + + ~WritableFileImpl() override { file_->Unref(); } + + Status Append(const Slice& data) override { return file_->Append(data); } + + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + + private: + FileState* file_; +}; + +class NoOpLogger : public Logger { + public: + void Logv(const char* format, std::va_list ap) override {} +}; + +class InMemoryEnv : public EnvWrapper { + public: + explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) {} + + ~InMemoryEnv() override { + for (const auto& kvp : file_map_) { + kvp.second->Unref(); + } + } + + // Partial implementation of the Env interface. + Status NewSequentialFile(const std::string& fname, + SequentialFile** result) override { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + *result = nullptr; + return Status::IOError(fname, "File not found"); + } + + *result = new SequentialFileImpl(file_map_[fname]); + return Status::OK(); + } + + Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result) override { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + *result = nullptr; + return Status::IOError(fname, "File not found"); + } + + *result = new RandomAccessFileImpl(file_map_[fname]); + return Status::OK(); + } + + Status NewWritableFile(const std::string& fname, + WritableFile** result) override { + MutexLock lock(&mutex_); + FileSystem::iterator it = file_map_.find(fname); + + FileState* file; + if (it == file_map_.end()) { + // File is not currently open. + file = new FileState(); + file->Ref(); + file_map_[fname] = file; + } else { + file = it->second; + file->Truncate(); + } + + *result = new WritableFileImpl(file); + return Status::OK(); + } + + Status NewAppendableFile(const std::string& fname, + WritableFile** result) override { + MutexLock lock(&mutex_); + FileState** sptr = &file_map_[fname]; + FileState* file = *sptr; + if (file == nullptr) { + file = new FileState(); + file->Ref(); + } + *result = new WritableFileImpl(file); + return Status::OK(); + } + + bool FileExists(const std::string& fname) override { + MutexLock lock(&mutex_); + return file_map_.find(fname) != file_map_.end(); + } + + Status GetChildren(const std::string& dir, + std::vector* result) override { + MutexLock lock(&mutex_); + result->clear(); + + for (const auto& kvp : file_map_) { + const std::string& filename = kvp.first; + + if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' && + Slice(filename).starts_with(Slice(dir))) { + result->push_back(filename.substr(dir.size() + 1)); + } + } + + return Status::OK(); + } + + void RemoveFileInternal(const std::string& fname) + EXCLUSIVE_LOCKS_REQUIRED(mutex_) { + if (file_map_.find(fname) == file_map_.end()) { + return; + } + + file_map_[fname]->Unref(); + file_map_.erase(fname); + } + + Status RemoveFile(const std::string& fname) override { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + return Status::IOError(fname, "File not found"); + } + + RemoveFileInternal(fname); + return Status::OK(); + } + + Status CreateDir(const std::string& dirname) override { return Status::OK(); } + + Status RemoveDir(const std::string& dirname) override { return Status::OK(); } + + Status GetFileSize(const std::string& fname, uint64_t* file_size) override { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + return Status::IOError(fname, "File not found"); + } + + *file_size = file_map_[fname]->Size(); + return Status::OK(); + } + + Status RenameFile(const std::string& src, + const std::string& target) override { + MutexLock lock(&mutex_); + if (file_map_.find(src) == file_map_.end()) { + return Status::IOError(src, "File not found"); + } + + RemoveFileInternal(target); + file_map_[target] = file_map_[src]; + file_map_.erase(src); + return Status::OK(); + } + + Status LockFile(const std::string& fname, FileLock** lock) override { + *lock = new FileLock; + return Status::OK(); + } + + Status UnlockFile(FileLock* lock) override { + delete lock; + return Status::OK(); + } + + Status GetTestDirectory(std::string* path) override { + *path = "/test"; + return Status::OK(); + } + + Status NewLogger(const std::string& fname, Logger** result) override { + *result = new NoOpLogger; + return Status::OK(); + } + + private: + // Map from filenames to FileState objects, representing a simple file system. + typedef std::map FileSystem; + + port::Mutex mutex_; + FileSystem file_map_ GUARDED_BY(mutex_); +}; + +} // namespace + +Env* NewMemEnv(Env* base_env) { return new InMemoryEnv(base_env); } + +} // namespace leveldb diff --git a/leveldb/helpers/memenv/memenv.h b/leveldb/helpers/memenv/memenv.h new file mode 100644 index 000000000..3d929e4c4 --- /dev/null +++ b/leveldb/helpers/memenv/memenv.h @@ -0,0 +1,22 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ +#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ + +#include "leveldb/export.h" + +namespace leveldb { + +class Env; + +// Returns a new environment that stores its data in memory and delegates +// all non-file-storage tasks to base_env. The caller must delete the result +// when it is no longer needed. +// *base_env must remain live while the result is in use. +LEVELDB_EXPORT Env* NewMemEnv(Env* base_env); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ diff --git a/leveldb/helpers/memenv/memenv_test.cc b/leveldb/helpers/memenv/memenv_test.cc new file mode 100644 index 000000000..909a0ca41 --- /dev/null +++ b/leveldb/helpers/memenv/memenv_test.cc @@ -0,0 +1,259 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "helpers/memenv/memenv.h" + +#include +#include + +#include "gtest/gtest.h" +#include "db/db_impl.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "util/testutil.h" + +namespace leveldb { + +class MemEnvTest : public testing::Test { + public: + MemEnvTest() : env_(NewMemEnv(Env::Default())) {} + ~MemEnvTest() { delete env_; } + + Env* env_; +}; + +TEST_F(MemEnvTest, Basics) { + uint64_t file_size; + WritableFile* writable_file; + std::vector children; + + ASSERT_LEVELDB_OK(env_->CreateDir("/dir")); + + // Check that the directory is empty. + ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); + ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); + ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(0, children.size()); + + // Create a file. + ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(0, file_size); + delete writable_file; + + // Check that the file exists. + ASSERT_TRUE(env_->FileExists("/dir/f")); + ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(0, file_size); + ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(1, children.size()); + ASSERT_EQ("f", children[0]); + + // Write to the file. + ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_LEVELDB_OK(writable_file->Append("abc")); + delete writable_file; + + // Check that append works. + ASSERT_LEVELDB_OK(env_->NewAppendableFile("/dir/f", &writable_file)); + ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(3, file_size); + ASSERT_LEVELDB_OK(writable_file->Append("hello")); + delete writable_file; + + // Check for expected size. + ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(8, file_size); + + // Check that renaming works. + ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); + ASSERT_LEVELDB_OK(env_->RenameFile("/dir/f", "/dir/g")); + ASSERT_TRUE(!env_->FileExists("/dir/f")); + ASSERT_TRUE(env_->FileExists("/dir/g")); + ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/g", &file_size)); + ASSERT_EQ(8, file_size); + + // Check that opening non-existent file fails. + SequentialFile* seq_file; + RandomAccessFile* rand_file; + ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok()); + ASSERT_TRUE(!seq_file); + ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok()); + ASSERT_TRUE(!rand_file); + + // Check that deleting works. + ASSERT_TRUE(!env_->RemoveFile("/dir/non_existent").ok()); + ASSERT_LEVELDB_OK(env_->RemoveFile("/dir/g")); + ASSERT_TRUE(!env_->FileExists("/dir/g")); + ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(0, children.size()); + ASSERT_LEVELDB_OK(env_->RemoveDir("/dir")); +} + +TEST_F(MemEnvTest, ReadWrite) { + WritableFile* writable_file; + SequentialFile* seq_file; + RandomAccessFile* rand_file; + Slice result; + char scratch[100]; + + ASSERT_LEVELDB_OK(env_->CreateDir("/dir")); + + ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_LEVELDB_OK(writable_file->Append("hello ")); + ASSERT_LEVELDB_OK(writable_file->Append("world")); + delete writable_file; + + // Read sequentially. + ASSERT_LEVELDB_OK(env_->NewSequentialFile("/dir/f", &seq_file)); + ASSERT_LEVELDB_OK(seq_file->Read(5, &result, scratch)); // Read "hello". + ASSERT_EQ(0, result.compare("hello")); + ASSERT_LEVELDB_OK(seq_file->Skip(1)); + ASSERT_LEVELDB_OK(seq_file->Read(1000, &result, scratch)); // Read "world". + ASSERT_EQ(0, result.compare("world")); + ASSERT_LEVELDB_OK( + seq_file->Read(1000, &result, scratch)); // Try reading past EOF. + ASSERT_EQ(0, result.size()); + ASSERT_LEVELDB_OK(seq_file->Skip(100)); // Try to skip past end of file. + ASSERT_LEVELDB_OK(seq_file->Read(1000, &result, scratch)); + ASSERT_EQ(0, result.size()); + delete seq_file; + + // Random reads. + ASSERT_LEVELDB_OK(env_->NewRandomAccessFile("/dir/f", &rand_file)); + ASSERT_LEVELDB_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world". + ASSERT_EQ(0, result.compare("world")); + ASSERT_LEVELDB_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello". + ASSERT_EQ(0, result.compare("hello")); + ASSERT_LEVELDB_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d". + ASSERT_EQ(0, result.compare("d")); + + // Too high offset. + ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok()); + delete rand_file; +} + +TEST_F(MemEnvTest, Locks) { + FileLock* lock; + + // These are no-ops, but we test they return success. + ASSERT_LEVELDB_OK(env_->LockFile("some file", &lock)); + ASSERT_LEVELDB_OK(env_->UnlockFile(lock)); +} + +TEST_F(MemEnvTest, Misc) { + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + ASSERT_TRUE(!test_dir.empty()); + + WritableFile* writable_file; + ASSERT_LEVELDB_OK(env_->NewWritableFile("/a/b", &writable_file)); + + // These are no-ops, but we test they return success. + ASSERT_LEVELDB_OK(writable_file->Sync()); + ASSERT_LEVELDB_OK(writable_file->Flush()); + ASSERT_LEVELDB_OK(writable_file->Close()); + delete writable_file; +} + +TEST_F(MemEnvTest, LargeWrite) { + const size_t kWriteSize = 300 * 1024; + char* scratch = new char[kWriteSize * 2]; + + std::string write_data; + for (size_t i = 0; i < kWriteSize; ++i) { + write_data.append(1, static_cast(i)); + } + + WritableFile* writable_file; + ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_LEVELDB_OK(writable_file->Append("foo")); + ASSERT_LEVELDB_OK(writable_file->Append(write_data)); + delete writable_file; + + SequentialFile* seq_file; + Slice result; + ASSERT_LEVELDB_OK(env_->NewSequentialFile("/dir/f", &seq_file)); + ASSERT_LEVELDB_OK(seq_file->Read(3, &result, scratch)); // Read "foo". + ASSERT_EQ(0, result.compare("foo")); + + size_t read = 0; + std::string read_data; + while (read < kWriteSize) { + ASSERT_LEVELDB_OK(seq_file->Read(kWriteSize - read, &result, scratch)); + read_data.append(result.data(), result.size()); + read += result.size(); + } + ASSERT_TRUE(write_data == read_data); + delete seq_file; + delete[] scratch; +} + +TEST_F(MemEnvTest, OverwriteOpenFile) { + const char kWrite1Data[] = "Write #1 data"; + const size_t kFileDataLen = sizeof(kWrite1Data) - 1; + const std::string kTestFileName = testing::TempDir() + "leveldb-TestFile.dat"; + + ASSERT_LEVELDB_OK(WriteStringToFile(env_, kWrite1Data, kTestFileName)); + + RandomAccessFile* rand_file; + ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(kTestFileName, &rand_file)); + + const char kWrite2Data[] = "Write #2 data"; + ASSERT_LEVELDB_OK(WriteStringToFile(env_, kWrite2Data, kTestFileName)); + + // Verify that overwriting an open file will result in the new file data + // being read from files opened before the write. + Slice result; + char scratch[kFileDataLen]; + ASSERT_LEVELDB_OK(rand_file->Read(0, kFileDataLen, &result, scratch)); + ASSERT_EQ(0, result.compare(kWrite2Data)); + + delete rand_file; +} + +TEST_F(MemEnvTest, DBTest) { + Options options; + options.create_if_missing = true; + options.env = env_; + DB* db; + + const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; + const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; + + ASSERT_LEVELDB_OK(DB::Open(options, "/dir/db", &db)); + for (size_t i = 0; i < 3; ++i) { + ASSERT_LEVELDB_OK(db->Put(WriteOptions(), keys[i], vals[i])); + } + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_LEVELDB_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + Iterator* iterator = db->NewIterator(ReadOptions()); + iterator->SeekToFirst(); + for (size_t i = 0; i < 3; ++i) { + ASSERT_TRUE(iterator->Valid()); + ASSERT_TRUE(keys[i] == iterator->key()); + ASSERT_TRUE(vals[i] == iterator->value()); + iterator->Next(); + } + ASSERT_TRUE(!iterator->Valid()); + delete iterator; + + DBImpl* dbi = reinterpret_cast(db); + ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable()); + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_LEVELDB_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + delete db; +} + +} // namespace leveldb diff --git a/leveldb/include/leveldb/c.h b/leveldb/include/leveldb/c.h new file mode 100644 index 000000000..62e1f6439 --- /dev/null +++ b/leveldb/include/leveldb/c.h @@ -0,0 +1,270 @@ +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. + + C bindings for leveldb. May be useful as a stable ABI that can be + used by programs that keep leveldb in a shared library, or for + a JNI api. + + Does not support: + . getters for the option types + . custom comparators that implement key shortening + . custom iter, db, env, cache implementations using just the C bindings + + Some conventions: + + (1) We expose just opaque struct pointers and functions to clients. + This allows us to change internal representations without having to + recompile clients. + + (2) For simplicity, there is no equivalent to the Slice type. Instead, + the caller has to pass the pointer and length as separate + arguments. + + (3) Errors are represented by a null-terminated c string. NULL + means no error. All operations that can raise an error are passed + a "char** errptr" as the last argument. One of the following must + be true on entry: + *errptr == NULL + *errptr points to a malloc()ed null-terminated error message + (On Windows, *errptr must have been malloc()-ed by this library.) + On success, a leveldb routine leaves *errptr unchanged. + On failure, leveldb frees the old value of *errptr and + set *errptr to a malloc()ed error message. + + (4) Bools have the type uint8_t (0 == false; rest == true) + + (5) All of the pointer arguments must be non-NULL. +*/ + +#ifndef STORAGE_LEVELDB_INCLUDE_C_H_ +#define STORAGE_LEVELDB_INCLUDE_C_H_ + +#include +#include +#include + +#include "leveldb/export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Exported types */ + +typedef struct leveldb_t leveldb_t; +typedef struct leveldb_cache_t leveldb_cache_t; +typedef struct leveldb_comparator_t leveldb_comparator_t; +typedef struct leveldb_env_t leveldb_env_t; +typedef struct leveldb_filelock_t leveldb_filelock_t; +typedef struct leveldb_filterpolicy_t leveldb_filterpolicy_t; +typedef struct leveldb_iterator_t leveldb_iterator_t; +typedef struct leveldb_logger_t leveldb_logger_t; +typedef struct leveldb_options_t leveldb_options_t; +typedef struct leveldb_randomfile_t leveldb_randomfile_t; +typedef struct leveldb_readoptions_t leveldb_readoptions_t; +typedef struct leveldb_seqfile_t leveldb_seqfile_t; +typedef struct leveldb_snapshot_t leveldb_snapshot_t; +typedef struct leveldb_writablefile_t leveldb_writablefile_t; +typedef struct leveldb_writebatch_t leveldb_writebatch_t; +typedef struct leveldb_writeoptions_t leveldb_writeoptions_t; + +/* DB operations */ + +LEVELDB_EXPORT leveldb_t* leveldb_open(const leveldb_options_t* options, + const char* name, char** errptr); + +LEVELDB_EXPORT void leveldb_close(leveldb_t* db); + +LEVELDB_EXPORT void leveldb_put(leveldb_t* db, + const leveldb_writeoptions_t* options, + const char* key, size_t keylen, const char* val, + size_t vallen, char** errptr); + +LEVELDB_EXPORT void leveldb_delete(leveldb_t* db, + const leveldb_writeoptions_t* options, + const char* key, size_t keylen, + char** errptr); + +LEVELDB_EXPORT void leveldb_write(leveldb_t* db, + const leveldb_writeoptions_t* options, + leveldb_writebatch_t* batch, char** errptr); + +/* Returns NULL if not found. A malloc()ed array otherwise. + Stores the length of the array in *vallen. */ +LEVELDB_EXPORT char* leveldb_get(leveldb_t* db, + const leveldb_readoptions_t* options, + const char* key, size_t keylen, size_t* vallen, + char** errptr); + +LEVELDB_EXPORT leveldb_iterator_t* leveldb_create_iterator( + leveldb_t* db, const leveldb_readoptions_t* options); + +LEVELDB_EXPORT const leveldb_snapshot_t* leveldb_create_snapshot(leveldb_t* db); + +LEVELDB_EXPORT void leveldb_release_snapshot( + leveldb_t* db, const leveldb_snapshot_t* snapshot); + +/* Returns NULL if property name is unknown. + Else returns a pointer to a malloc()-ed null-terminated value. */ +LEVELDB_EXPORT char* leveldb_property_value(leveldb_t* db, + const char* propname); + +LEVELDB_EXPORT void leveldb_approximate_sizes( + leveldb_t* db, int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes); + +LEVELDB_EXPORT void leveldb_compact_range(leveldb_t* db, const char* start_key, + size_t start_key_len, + const char* limit_key, + size_t limit_key_len); + +/* Management operations */ + +LEVELDB_EXPORT void leveldb_destroy_db(const leveldb_options_t* options, + const char* name, char** errptr); + +LEVELDB_EXPORT void leveldb_repair_db(const leveldb_options_t* options, + const char* name, char** errptr); + +/* Iterator */ + +LEVELDB_EXPORT void leveldb_iter_destroy(leveldb_iterator_t*); +LEVELDB_EXPORT uint8_t leveldb_iter_valid(const leveldb_iterator_t*); +LEVELDB_EXPORT void leveldb_iter_seek_to_first(leveldb_iterator_t*); +LEVELDB_EXPORT void leveldb_iter_seek_to_last(leveldb_iterator_t*); +LEVELDB_EXPORT void leveldb_iter_seek(leveldb_iterator_t*, const char* k, + size_t klen); +LEVELDB_EXPORT void leveldb_iter_next(leveldb_iterator_t*); +LEVELDB_EXPORT void leveldb_iter_prev(leveldb_iterator_t*); +LEVELDB_EXPORT const char* leveldb_iter_key(const leveldb_iterator_t*, + size_t* klen); +LEVELDB_EXPORT const char* leveldb_iter_value(const leveldb_iterator_t*, + size_t* vlen); +LEVELDB_EXPORT void leveldb_iter_get_error(const leveldb_iterator_t*, + char** errptr); + +/* Write batch */ + +LEVELDB_EXPORT leveldb_writebatch_t* leveldb_writebatch_create(void); +LEVELDB_EXPORT void leveldb_writebatch_destroy(leveldb_writebatch_t*); +LEVELDB_EXPORT void leveldb_writebatch_clear(leveldb_writebatch_t*); +LEVELDB_EXPORT void leveldb_writebatch_put(leveldb_writebatch_t*, + const char* key, size_t klen, + const char* val, size_t vlen); +LEVELDB_EXPORT void leveldb_writebatch_delete(leveldb_writebatch_t*, + const char* key, size_t klen); +LEVELDB_EXPORT void leveldb_writebatch_iterate( + const leveldb_writebatch_t*, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +LEVELDB_EXPORT void leveldb_writebatch_append( + leveldb_writebatch_t* destination, const leveldb_writebatch_t* source); + +/* Options */ + +LEVELDB_EXPORT leveldb_options_t* leveldb_options_create(void); +LEVELDB_EXPORT void leveldb_options_destroy(leveldb_options_t*); +LEVELDB_EXPORT void leveldb_options_set_comparator(leveldb_options_t*, + leveldb_comparator_t*); +LEVELDB_EXPORT void leveldb_options_set_filter_policy(leveldb_options_t*, + leveldb_filterpolicy_t*); +LEVELDB_EXPORT void leveldb_options_set_create_if_missing(leveldb_options_t*, + uint8_t); +LEVELDB_EXPORT void leveldb_options_set_error_if_exists(leveldb_options_t*, + uint8_t); +LEVELDB_EXPORT void leveldb_options_set_paranoid_checks(leveldb_options_t*, + uint8_t); +LEVELDB_EXPORT void leveldb_options_set_env(leveldb_options_t*, leveldb_env_t*); +LEVELDB_EXPORT void leveldb_options_set_info_log(leveldb_options_t*, + leveldb_logger_t*); +LEVELDB_EXPORT void leveldb_options_set_write_buffer_size(leveldb_options_t*, + size_t); +LEVELDB_EXPORT void leveldb_options_set_max_open_files(leveldb_options_t*, int); +LEVELDB_EXPORT void leveldb_options_set_cache(leveldb_options_t*, + leveldb_cache_t*); +LEVELDB_EXPORT void leveldb_options_set_block_size(leveldb_options_t*, size_t); +LEVELDB_EXPORT void leveldb_options_set_block_restart_interval( + leveldb_options_t*, int); +LEVELDB_EXPORT void leveldb_options_set_max_file_size(leveldb_options_t*, + size_t); + +enum { leveldb_no_compression = 0, leveldb_snappy_compression = 1 }; +LEVELDB_EXPORT void leveldb_options_set_compression(leveldb_options_t*, int); + +/* Comparator */ + +LEVELDB_EXPORT leveldb_comparator_t* leveldb_comparator_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + const char* (*name)(void*)); +LEVELDB_EXPORT void leveldb_comparator_destroy(leveldb_comparator_t*); + +/* Filter policy */ + +LEVELDB_EXPORT leveldb_filterpolicy_t* leveldb_filterpolicy_create( + void* state, void (*destructor)(void*), + char* (*create_filter)(void*, const char* const* key_array, + const size_t* key_length_array, int num_keys, + size_t* filter_length), + uint8_t (*key_may_match)(void*, const char* key, size_t length, + const char* filter, size_t filter_length), + const char* (*name)(void*)); +LEVELDB_EXPORT void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t*); + +LEVELDB_EXPORT leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom( + int bits_per_key); + +/* Read options */ + +LEVELDB_EXPORT leveldb_readoptions_t* leveldb_readoptions_create(void); +LEVELDB_EXPORT void leveldb_readoptions_destroy(leveldb_readoptions_t*); +LEVELDB_EXPORT void leveldb_readoptions_set_verify_checksums( + leveldb_readoptions_t*, uint8_t); +LEVELDB_EXPORT void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t*, + uint8_t); +LEVELDB_EXPORT void leveldb_readoptions_set_snapshot(leveldb_readoptions_t*, + const leveldb_snapshot_t*); + +/* Write options */ + +LEVELDB_EXPORT leveldb_writeoptions_t* leveldb_writeoptions_create(void); +LEVELDB_EXPORT void leveldb_writeoptions_destroy(leveldb_writeoptions_t*); +LEVELDB_EXPORT void leveldb_writeoptions_set_sync(leveldb_writeoptions_t*, + uint8_t); + +/* Cache */ + +LEVELDB_EXPORT leveldb_cache_t* leveldb_cache_create_lru(size_t capacity); +LEVELDB_EXPORT void leveldb_cache_destroy(leveldb_cache_t* cache); + +/* Env */ + +LEVELDB_EXPORT leveldb_env_t* leveldb_create_default_env(void); +LEVELDB_EXPORT void leveldb_env_destroy(leveldb_env_t*); + +/* If not NULL, the returned buffer must be released using leveldb_free(). */ +LEVELDB_EXPORT char* leveldb_env_get_test_directory(leveldb_env_t*); + +/* Utility */ + +/* Calls free(ptr). + REQUIRES: ptr was malloc()-ed and returned by one of the routines + in this file. Note that in certain cases (typically on Windows), you + may need to call this routine instead of free(ptr) to dispose of + malloc()-ed memory returned by this library. */ +LEVELDB_EXPORT void leveldb_free(void* ptr); + +/* Return the major version number for this release. */ +LEVELDB_EXPORT int leveldb_major_version(void); + +/* Return the minor version number for this release. */ +LEVELDB_EXPORT int leveldb_minor_version(void); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif /* STORAGE_LEVELDB_INCLUDE_C_H_ */ diff --git a/leveldb/include/leveldb/cache.h b/leveldb/include/leveldb/cache.h new file mode 100644 index 000000000..a94c68398 --- /dev/null +++ b/leveldb/include/leveldb/cache.h @@ -0,0 +1,103 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A Cache is an interface that maps keys to values. It has internal +// synchronization and may be safely accessed concurrently from +// multiple threads. It may automatically evict entries to make room +// for new entries. Values have a specified charge against the cache +// capacity. For example, a cache where the values are variable +// length strings, may use the length of the string as the charge for +// the string. +// +// A builtin cache implementation with a least-recently-used eviction +// policy is provided. Clients may use their own implementations if +// they want something more sophisticated (like scan-resistance, a +// custom eviction policy, variable cache sizing, etc.) + +#ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_ +#define STORAGE_LEVELDB_INCLUDE_CACHE_H_ + +#include + +#include "leveldb/export.h" +#include "leveldb/slice.h" + +namespace leveldb { + +class LEVELDB_EXPORT Cache; + +// Create a new cache with a fixed size capacity. This implementation +// of Cache uses a least-recently-used eviction policy. +LEVELDB_EXPORT Cache* NewLRUCache(size_t capacity); + +class LEVELDB_EXPORT Cache { + public: + Cache() = default; + + Cache(const Cache&) = delete; + Cache& operator=(const Cache&) = delete; + + // Destroys all existing entries by calling the "deleter" + // function that was passed to the constructor. + virtual ~Cache(); + + // Opaque handle to an entry stored in the cache. + struct Handle {}; + + // Insert a mapping from key->value into the cache and assign it + // the specified charge against the total cache capacity. + // + // Returns a handle that corresponds to the mapping. The caller + // must call this->Release(handle) when the returned mapping is no + // longer needed. + // + // When the inserted entry is no longer needed, the key and + // value will be passed to "deleter". + virtual Handle* Insert(const Slice& key, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value)) = 0; + + // If the cache has no mapping for "key", returns nullptr. + // + // Else return a handle that corresponds to the mapping. The caller + // must call this->Release(handle) when the returned mapping is no + // longer needed. + virtual Handle* Lookup(const Slice& key) = 0; + + // Release a mapping returned by a previous Lookup(). + // REQUIRES: handle must not have been released yet. + // REQUIRES: handle must have been returned by a method on *this. + virtual void Release(Handle* handle) = 0; + + // Return the value encapsulated in a handle returned by a + // successful Lookup(). + // REQUIRES: handle must not have been released yet. + // REQUIRES: handle must have been returned by a method on *this. + virtual void* Value(Handle* handle) = 0; + + // If the cache contains entry for key, erase it. Note that the + // underlying entry will be kept around until all existing handles + // to it have been released. + virtual void Erase(const Slice& key) = 0; + + // Return a new numeric id. May be used by multiple clients who are + // sharing the same cache to partition the key space. Typically the + // client will allocate a new id at startup and prepend the id to + // its cache keys. + virtual uint64_t NewId() = 0; + + // Remove all cache entries that are not actively in use. Memory-constrained + // applications may wish to call this method to reduce memory usage. + // Default implementation of Prune() does nothing. Subclasses are strongly + // encouraged to override the default implementation. A future release of + // leveldb may change Prune() to a pure abstract method. + virtual void Prune() {} + + // Return an estimate of the combined charges of all elements stored in the + // cache. + virtual size_t TotalCharge() const = 0; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_CACHE_H_ diff --git a/leveldb/include/leveldb/comparator.h b/leveldb/include/leveldb/comparator.h new file mode 100644 index 000000000..a85b51ebd --- /dev/null +++ b/leveldb/include/leveldb/comparator.h @@ -0,0 +1,64 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ +#define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ + +#include + +#include "leveldb/export.h" + +namespace leveldb { + +class Slice; + +// A Comparator object provides a total order across slices that are +// used as keys in an sstable or a database. A Comparator implementation +// must be thread-safe since leveldb may invoke its methods concurrently +// from multiple threads. +class LEVELDB_EXPORT Comparator { + public: + virtual ~Comparator(); + + // Three-way comparison. Returns value: + // < 0 iff "a" < "b", + // == 0 iff "a" == "b", + // > 0 iff "a" > "b" + virtual int Compare(const Slice& a, const Slice& b) const = 0; + + // The name of the comparator. Used to check for comparator + // mismatches (i.e., a DB created with one comparator is + // accessed using a different comparator. + // + // The client of this package should switch to a new name whenever + // the comparator implementation changes in a way that will cause + // the relative ordering of any two keys to change. + // + // Names starting with "leveldb." are reserved and should not be used + // by any clients of this package. + virtual const char* Name() const = 0; + + // Advanced functions: these are used to reduce the space requirements + // for internal data structures like index blocks. + + // If *start < limit, changes *start to a short string in [start,limit). + // Simple comparator implementations may return with *start unchanged, + // i.e., an implementation of this method that does nothing is correct. + virtual void FindShortestSeparator(std::string* start, + const Slice& limit) const = 0; + + // Changes *key to a short string >= *key. + // Simple comparator implementations may return with *key unchanged, + // i.e., an implementation of this method that does nothing is correct. + virtual void FindShortSuccessor(std::string* key) const = 0; +}; + +// Return a builtin comparator that uses lexicographic byte-wise +// ordering. The result remains the property of this module and +// must not be deleted. +LEVELDB_EXPORT const Comparator* BytewiseComparator(); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ diff --git a/leveldb/include/leveldb/db.h b/leveldb/include/leveldb/db.h new file mode 100644 index 000000000..a13d14716 --- /dev/null +++ b/leveldb/include/leveldb/db.h @@ -0,0 +1,167 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_INCLUDE_DB_H_ +#define STORAGE_LEVELDB_INCLUDE_DB_H_ + +#include +#include + +#include "leveldb/export.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" + +namespace leveldb { + +// Update CMakeLists.txt if you change these +static const int kMajorVersion = 1; +static const int kMinorVersion = 23; + +struct Options; +struct ReadOptions; +struct WriteOptions; +class WriteBatch; + +// Abstract handle to particular state of a DB. +// A Snapshot is an immutable object and can therefore be safely +// accessed from multiple threads without any external synchronization. +class LEVELDB_EXPORT Snapshot { + protected: + virtual ~Snapshot(); +}; + +// A range of keys +struct LEVELDB_EXPORT Range { + Range() = default; + Range(const Slice& s, const Slice& l) : start(s), limit(l) {} + + Slice start; // Included in the range + Slice limit; // Not included in the range +}; + +// A DB is a persistent ordered map from keys to values. +// A DB is safe for concurrent access from multiple threads without +// any external synchronization. +class LEVELDB_EXPORT DB { + public: + // Open the database with the specified "name". + // Stores a pointer to a heap-allocated database in *dbptr and returns + // OK on success. + // Stores nullptr in *dbptr and returns a non-OK status on error. + // Caller should delete *dbptr when it is no longer needed. + static Status Open(const Options& options, const std::string& name, + DB** dbptr); + + DB() = default; + + DB(const DB&) = delete; + DB& operator=(const DB&) = delete; + + virtual ~DB(); + + // Set the database entry for "key" to "value". Returns OK on success, + // and a non-OK status on error. + // Note: consider setting options.sync = true. + virtual Status Put(const WriteOptions& options, const Slice& key, + const Slice& value) = 0; + + // Remove the database entry (if any) for "key". Returns OK on + // success, and a non-OK status on error. It is not an error if "key" + // did not exist in the database. + // Note: consider setting options.sync = true. + virtual Status Delete(const WriteOptions& options, const Slice& key) = 0; + + // Apply the specified updates to the database. + // Returns OK on success, non-OK on failure. + // Note: consider setting options.sync = true. + virtual Status Write(const WriteOptions& options, WriteBatch* updates) = 0; + + // If the database contains an entry for "key" store the + // corresponding value in *value and return OK. + // + // If there is no entry for "key" leave *value unchanged and return + // a status for which Status::IsNotFound() returns true. + // + // May return some other Status on an error. + virtual Status Get(const ReadOptions& options, const Slice& key, + std::string* value) = 0; + + // Return a heap-allocated iterator over the contents of the database. + // The result of NewIterator() is initially invalid (caller must + // call one of the Seek methods on the iterator before using it). + // + // Caller should delete the iterator when it is no longer needed. + // The returned iterator should be deleted before this db is deleted. + virtual Iterator* NewIterator(const ReadOptions& options) = 0; + + // Return a handle to the current DB state. Iterators created with + // this handle will all observe a stable snapshot of the current DB + // state. The caller must call ReleaseSnapshot(result) when the + // snapshot is no longer needed. + virtual const Snapshot* GetSnapshot() = 0; + + // Release a previously acquired snapshot. The caller must not + // use "snapshot" after this call. + virtual void ReleaseSnapshot(const Snapshot* snapshot) = 0; + + // DB implementations can export properties about their state + // via this method. If "property" is a valid property understood by this + // DB implementation, fills "*value" with its current value and returns + // true. Otherwise returns false. + // + // + // Valid property names include: + // + // "leveldb.num-files-at-level" - return the number of files at level , + // where is an ASCII representation of a level number (e.g. "0"). + // "leveldb.stats" - returns a multi-line string that describes statistics + // about the internal operation of the DB. + // "leveldb.sstables" - returns a multi-line string that describes all + // of the sstables that make up the db contents. + // "leveldb.approximate-memory-usage" - returns the approximate number of + // bytes of memory in use by the DB. + virtual bool GetProperty(const Slice& property, std::string* value) = 0; + + // For each i in [0,n-1], store in "sizes[i]", the approximate + // file system space used by keys in "[range[i].start .. range[i].limit)". + // + // Note that the returned sizes measure file system space usage, so + // if the user data compresses by a factor of ten, the returned + // sizes will be one-tenth the size of the corresponding user data size. + // + // The results may not include the sizes of recently written data. + virtual void GetApproximateSizes(const Range* range, int n, + uint64_t* sizes) = 0; + + // Compact the underlying storage for the key range [*begin,*end]. + // In particular, deleted and overwritten versions are discarded, + // and the data is rearranged to reduce the cost of operations + // needed to access the data. This operation should typically only + // be invoked by users who understand the underlying implementation. + // + // begin==nullptr is treated as a key before all keys in the database. + // end==nullptr is treated as a key after all keys in the database. + // Therefore the following call will compact the entire database: + // db->CompactRange(nullptr, nullptr); + virtual void CompactRange(const Slice* begin, const Slice* end) = 0; +}; + +// Destroy the contents of the specified database. +// Be very careful using this method. +// +// Note: For backwards compatibility, if DestroyDB is unable to list the +// database files, Status::OK() will still be returned masking this failure. +LEVELDB_EXPORT Status DestroyDB(const std::string& name, + const Options& options); + +// If a DB cannot be opened, you may attempt to call this method to +// resurrect as much of the contents of the database as possible. +// Some data may be lost, so be careful when calling this function +// on a database that contains important information. +LEVELDB_EXPORT Status RepairDB(const std::string& dbname, + const Options& options); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_DB_H_ diff --git a/leveldb/include/leveldb/dumpfile.h b/leveldb/include/leveldb/dumpfile.h new file mode 100644 index 000000000..a58bc6b36 --- /dev/null +++ b/leveldb/include/leveldb/dumpfile.h @@ -0,0 +1,28 @@ +// Copyright (c) 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ +#define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ + +#include + +#include "leveldb/env.h" +#include "leveldb/export.h" +#include "leveldb/status.h" + +namespace leveldb { + +// Dump the contents of the file named by fname in text format to +// *dst. Makes a sequence of dst->Append() calls; each call is passed +// the newline-terminated text corresponding to a single item found +// in the file. +// +// Returns a non-OK result if fname does not name a leveldb storage +// file, or if the file cannot be read. +LEVELDB_EXPORT Status DumpFile(Env* env, const std::string& fname, + WritableFile* dst); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ diff --git a/leveldb/include/leveldb/env.h b/leveldb/include/leveldb/env.h new file mode 100644 index 000000000..e00895a2b --- /dev/null +++ b/leveldb/include/leveldb/env.h @@ -0,0 +1,417 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// An Env is an interface used by the leveldb implementation to access +// operating system functionality like the filesystem etc. Callers +// may wish to provide a custom Env object when opening a database to +// get fine gain control; e.g., to rate limit file system operations. +// +// All Env implementations are safe for concurrent access from +// multiple threads without any external synchronization. + +#ifndef STORAGE_LEVELDB_INCLUDE_ENV_H_ +#define STORAGE_LEVELDB_INCLUDE_ENV_H_ + +#include +#include +#include +#include + +#include "leveldb/export.h" +#include "leveldb/status.h" + +// This workaround can be removed when leveldb::Env::DeleteFile is removed. +#if defined(_WIN32) +// On Windows, the method name DeleteFile (below) introduces the risk of +// triggering undefined behavior by exposing the compiler to different +// declarations of the Env class in different translation units. +// +// This is because , a fairly popular header file for Windows +// applications, defines a DeleteFile macro. So, files that include the Windows +// header before this header will contain an altered Env declaration. +// +// This workaround ensures that the compiler sees the same Env declaration, +// independently of whether was included. +#if defined(DeleteFile) +#undef DeleteFile +#define LEVELDB_DELETEFILE_UNDEFINED +#endif // defined(DeleteFile) +#endif // defined(_WIN32) + +namespace leveldb { + +class FileLock; +class Logger; +class RandomAccessFile; +class SequentialFile; +class Slice; +class WritableFile; + +class LEVELDB_EXPORT Env { + public: + Env(); + + Env(const Env&) = delete; + Env& operator=(const Env&) = delete; + + virtual ~Env(); + + // Return a default environment suitable for the current operating + // system. Sophisticated users may wish to provide their own Env + // implementation instead of relying on this default environment. + // + // The result of Default() belongs to leveldb and must never be deleted. + static Env* Default(); + + // Create an object that sequentially reads the file with the specified name. + // On success, stores a pointer to the new file in *result and returns OK. + // On failure stores nullptr in *result and returns non-OK. If the file does + // not exist, returns a non-OK status. Implementations should return a + // NotFound status when the file does not exist. + // + // The returned file will only be accessed by one thread at a time. + virtual Status NewSequentialFile(const std::string& fname, + SequentialFile** result) = 0; + + // Create an object supporting random-access reads from the file with the + // specified name. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. If the file does not exist, returns a non-OK + // status. Implementations should return a NotFound status when the file does + // not exist. + // + // The returned file may be concurrently accessed by multiple threads. + virtual Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result) = 0; + + // Create an object that writes to a new file with the specified + // name. Deletes any existing file with the same name and creates a + // new file. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result) = 0; + + // Create an object that either appends to an existing file, or + // writes to a new file (if the file does not exist to begin with). + // On success, stores a pointer to the new file in *result and + // returns OK. On failure stores nullptr in *result and returns + // non-OK. + // + // The returned file will only be accessed by one thread at a time. + // + // May return an IsNotSupportedError error if this Env does + // not allow appending to an existing file. Users of Env (including + // the leveldb implementation) must be prepared to deal with + // an Env that does not support appending. + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result); + + // Returns true iff the named file exists. + virtual bool FileExists(const std::string& fname) = 0; + + // Store in *result the names of the children of the specified directory. + // The names are relative to "dir". + // Original contents of *results are dropped. + virtual Status GetChildren(const std::string& dir, + std::vector* result) = 0; + // Delete the named file. + // + // The default implementation calls DeleteFile, to support legacy Env + // implementations. Updated Env implementations must override RemoveFile and + // ignore the existence of DeleteFile. Updated code calling into the Env API + // must call RemoveFile instead of DeleteFile. + // + // A future release will remove DeleteDir and the default implementation of + // RemoveDir. + virtual Status RemoveFile(const std::string& fname); + + // DEPRECATED: Modern Env implementations should override RemoveFile instead. + // + // The default implementation calls RemoveFile, to support legacy Env user + // code that calls this method on modern Env implementations. Modern Env user + // code should call RemoveFile. + // + // A future release will remove this method. + virtual Status DeleteFile(const std::string& fname); + + // Create the specified directory. + virtual Status CreateDir(const std::string& dirname) = 0; + + // Delete the specified directory. + // + // The default implementation calls DeleteDir, to support legacy Env + // implementations. Updated Env implementations must override RemoveDir and + // ignore the existence of DeleteDir. Modern code calling into the Env API + // must call RemoveDir instead of DeleteDir. + // + // A future release will remove DeleteDir and the default implementation of + // RemoveDir. + virtual Status RemoveDir(const std::string& dirname); + + // DEPRECATED: Modern Env implementations should override RemoveDir instead. + // + // The default implementation calls RemoveDir, to support legacy Env user + // code that calls this method on modern Env implementations. Modern Env user + // code should call RemoveDir. + // + // A future release will remove this method. + virtual Status DeleteDir(const std::string& dirname); + + // Store the size of fname in *file_size. + virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) = 0; + + // Rename file src to target. + virtual Status RenameFile(const std::string& src, + const std::string& target) = 0; + + // Lock the specified file. Used to prevent concurrent access to + // the same db by multiple processes. On failure, stores nullptr in + // *lock and returns non-OK. + // + // On success, stores a pointer to the object that represents the + // acquired lock in *lock and returns OK. The caller should call + // UnlockFile(*lock) to release the lock. If the process exits, + // the lock will be automatically released. + // + // If somebody else already holds the lock, finishes immediately + // with a failure. I.e., this call does not wait for existing locks + // to go away. + // + // May create the named file if it does not already exist. + virtual Status LockFile(const std::string& fname, FileLock** lock) = 0; + + // Release the lock acquired by a previous successful call to LockFile. + // REQUIRES: lock was returned by a successful LockFile() call + // REQUIRES: lock has not already been unlocked. + virtual Status UnlockFile(FileLock* lock) = 0; + + // Arrange to run "(*function)(arg)" once in a background thread. + // + // "function" may run in an unspecified thread. Multiple functions + // added to the same Env may run concurrently in different threads. + // I.e., the caller may not assume that background work items are + // serialized. + virtual void Schedule(void (*function)(void* arg), void* arg) = 0; + + // Start a new thread, invoking "function(arg)" within the new thread. + // When "function(arg)" returns, the thread will be destroyed. + virtual void StartThread(void (*function)(void* arg), void* arg) = 0; + + // *path is set to a temporary directory that can be used for testing. It may + // or may not have just been created. The directory may or may not differ + // between runs of the same process, but subsequent calls will return the + // same directory. + virtual Status GetTestDirectory(std::string* path) = 0; + + // Create and return a log file for storing informational messages. + virtual Status NewLogger(const std::string& fname, Logger** result) = 0; + + // Returns the number of micro-seconds since some fixed point in time. Only + // useful for computing deltas of time. + virtual uint64_t NowMicros() = 0; + + // Sleep/delay the thread for the prescribed number of micro-seconds. + virtual void SleepForMicroseconds(int micros) = 0; +}; + +// A file abstraction for reading sequentially through a file +class LEVELDB_EXPORT SequentialFile { + public: + SequentialFile() = default; + + SequentialFile(const SequentialFile&) = delete; + SequentialFile& operator=(const SequentialFile&) = delete; + + virtual ~SequentialFile(); + + // Read up to "n" bytes from the file. "scratch[0..n-1]" may be + // written by this routine. Sets "*result" to the data that was + // read (including if fewer than "n" bytes were successfully read). + // May set "*result" to point at data in "scratch[0..n-1]", so + // "scratch[0..n-1]" must be live when "*result" is used. + // If an error was encountered, returns a non-OK status. + // + // REQUIRES: External synchronization + virtual Status Read(size_t n, Slice* result, char* scratch) = 0; + + // Skip "n" bytes from the file. This is guaranteed to be no + // slower that reading the same data, but may be faster. + // + // If end of file is reached, skipping will stop at the end of the + // file, and Skip will return OK. + // + // REQUIRES: External synchronization + virtual Status Skip(uint64_t n) = 0; +}; + +// A file abstraction for randomly reading the contents of a file. +class LEVELDB_EXPORT RandomAccessFile { + public: + RandomAccessFile() = default; + + RandomAccessFile(const RandomAccessFile&) = delete; + RandomAccessFile& operator=(const RandomAccessFile&) = delete; + + virtual ~RandomAccessFile(); + + // Read up to "n" bytes from the file starting at "offset". + // "scratch[0..n-1]" may be written by this routine. Sets "*result" + // to the data that was read (including if fewer than "n" bytes were + // successfully read). May set "*result" to point at data in + // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when + // "*result" is used. If an error was encountered, returns a non-OK + // status. + // + // Safe for concurrent use by multiple threads. + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const = 0; +}; + +// A file abstraction for sequential writing. The implementation +// must provide buffering since callers may append small fragments +// at a time to the file. +class LEVELDB_EXPORT WritableFile { + public: + WritableFile() = default; + + WritableFile(const WritableFile&) = delete; + WritableFile& operator=(const WritableFile&) = delete; + + virtual ~WritableFile(); + + virtual Status Append(const Slice& data) = 0; + virtual Status Close() = 0; + virtual Status Flush() = 0; + virtual Status Sync() = 0; +}; + +// An interface for writing log messages. +class LEVELDB_EXPORT Logger { + public: + Logger() = default; + + Logger(const Logger&) = delete; + Logger& operator=(const Logger&) = delete; + + virtual ~Logger(); + + // Write an entry to the log file with the specified format. + virtual void Logv(const char* format, std::va_list ap) = 0; +}; + +// Identifies a locked file. +class LEVELDB_EXPORT FileLock { + public: + FileLock() = default; + + FileLock(const FileLock&) = delete; + FileLock& operator=(const FileLock&) = delete; + + virtual ~FileLock(); +}; + +// Log the specified data to *info_log if info_log is non-null. +void Log(Logger* info_log, const char* format, ...) +#if defined(__GNUC__) || defined(__clang__) + __attribute__((__format__(__printf__, 2, 3))) +#endif + ; + +// A utility routine: write "data" to the named file. +LEVELDB_EXPORT Status WriteStringToFile(Env* env, const Slice& data, + const std::string& fname); + +// A utility routine: read contents of named file into *data +LEVELDB_EXPORT Status ReadFileToString(Env* env, const std::string& fname, + std::string* data); + +// An implementation of Env that forwards all calls to another Env. +// May be useful to clients who wish to override just part of the +// functionality of another Env. +class LEVELDB_EXPORT EnvWrapper : public Env { + public: + // Initialize an EnvWrapper that delegates all calls to *t. + explicit EnvWrapper(Env* t) : target_(t) {} + virtual ~EnvWrapper(); + + // Return the target to which this Env forwards all calls. + Env* target() const { return target_; } + + // The following text is boilerplate that forwards all methods to target(). + Status NewSequentialFile(const std::string& f, SequentialFile** r) override { + return target_->NewSequentialFile(f, r); + } + Status NewRandomAccessFile(const std::string& f, + RandomAccessFile** r) override { + return target_->NewRandomAccessFile(f, r); + } + Status NewWritableFile(const std::string& f, WritableFile** r) override { + return target_->NewWritableFile(f, r); + } + Status NewAppendableFile(const std::string& f, WritableFile** r) override { + return target_->NewAppendableFile(f, r); + } + bool FileExists(const std::string& f) override { + return target_->FileExists(f); + } + Status GetChildren(const std::string& dir, + std::vector* r) override { + return target_->GetChildren(dir, r); + } + Status RemoveFile(const std::string& f) override { + return target_->RemoveFile(f); + } + Status CreateDir(const std::string& d) override { + return target_->CreateDir(d); + } + Status RemoveDir(const std::string& d) override { + return target_->RemoveDir(d); + } + Status GetFileSize(const std::string& f, uint64_t* s) override { + return target_->GetFileSize(f, s); + } + Status RenameFile(const std::string& s, const std::string& t) override { + return target_->RenameFile(s, t); + } + Status LockFile(const std::string& f, FileLock** l) override { + return target_->LockFile(f, l); + } + Status UnlockFile(FileLock* l) override { return target_->UnlockFile(l); } + void Schedule(void (*f)(void*), void* a) override { + return target_->Schedule(f, a); + } + void StartThread(void (*f)(void*), void* a) override { + return target_->StartThread(f, a); + } + Status GetTestDirectory(std::string* path) override { + return target_->GetTestDirectory(path); + } + Status NewLogger(const std::string& fname, Logger** result) override { + return target_->NewLogger(fname, result); + } + uint64_t NowMicros() override { return target_->NowMicros(); } + void SleepForMicroseconds(int micros) override { + target_->SleepForMicroseconds(micros); + } + + private: + Env* target_; +}; + +} // namespace leveldb + +// This workaround can be removed when leveldb::Env::DeleteFile is removed. +// Redefine DeleteFile if it was undefined earlier. +#if defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED) +#if defined(UNICODE) +#define DeleteFile DeleteFileW +#else +#define DeleteFile DeleteFileA +#endif // defined(UNICODE) +#endif // defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED) + +#endif // STORAGE_LEVELDB_INCLUDE_ENV_H_ diff --git a/leveldb/include/leveldb/export.h b/leveldb/include/leveldb/export.h new file mode 100644 index 000000000..6ba9b183d --- /dev/null +++ b/leveldb/include/leveldb/export.h @@ -0,0 +1,33 @@ +// Copyright (c) 2017 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_INCLUDE_EXPORT_H_ +#define STORAGE_LEVELDB_INCLUDE_EXPORT_H_ + +#if !defined(LEVELDB_EXPORT) + +#if defined(LEVELDB_SHARED_LIBRARY) +#if defined(_WIN32) + +#if defined(LEVELDB_COMPILE_LIBRARY) +#define LEVELDB_EXPORT __declspec(dllexport) +#else +#define LEVELDB_EXPORT __declspec(dllimport) +#endif // defined(LEVELDB_COMPILE_LIBRARY) + +#else // defined(_WIN32) +#if defined(LEVELDB_COMPILE_LIBRARY) +#define LEVELDB_EXPORT __attribute__((visibility("default"))) +#else +#define LEVELDB_EXPORT +#endif +#endif // defined(_WIN32) + +#else // defined(LEVELDB_SHARED_LIBRARY) +#define LEVELDB_EXPORT +#endif + +#endif // !defined(LEVELDB_EXPORT) + +#endif // STORAGE_LEVELDB_INCLUDE_EXPORT_H_ diff --git a/leveldb/include/leveldb/filter_policy.h b/leveldb/include/leveldb/filter_policy.h new file mode 100644 index 000000000..49c8eda77 --- /dev/null +++ b/leveldb/include/leveldb/filter_policy.h @@ -0,0 +1,72 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A database can be configured with a custom FilterPolicy object. +// This object is responsible for creating a small filter from a set +// of keys. These filters are stored in leveldb and are consulted +// automatically by leveldb to decide whether or not to read some +// information from disk. In many cases, a filter can cut down the +// number of disk seeks form a handful to a single disk seek per +// DB::Get() call. +// +// Most people will want to use the builtin bloom filter support (see +// NewBloomFilterPolicy() below). + +#ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ +#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ + +#include + +#include "leveldb/export.h" + +namespace leveldb { + +class Slice; + +class LEVELDB_EXPORT FilterPolicy { + public: + virtual ~FilterPolicy(); + + // Return the name of this policy. Note that if the filter encoding + // changes in an incompatible way, the name returned by this method + // must be changed. Otherwise, old incompatible filters may be + // passed to methods of this type. + virtual const char* Name() const = 0; + + // keys[0,n-1] contains a list of keys (potentially with duplicates) + // that are ordered according to the user supplied comparator. + // Append a filter that summarizes keys[0,n-1] to *dst. + // + // Warning: do not change the initial contents of *dst. Instead, + // append the newly constructed filter to *dst. + virtual void CreateFilter(const Slice* keys, int n, + std::string* dst) const = 0; + + // "filter" contains the data appended by a preceding call to + // CreateFilter() on this class. This method must return true if + // the key was in the list of keys passed to CreateFilter(). + // This method may return true or false if the key was not on the + // list, but it should aim to return false with a high probability. + virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const = 0; +}; + +// Return a new filter policy that uses a bloom filter with approximately +// the specified number of bits per key. A good value for bits_per_key +// is 10, which yields a filter with ~ 1% false positive rate. +// +// Callers must delete the result after any database that is using the +// result has been closed. +// +// Note: if you are using a custom comparator that ignores some parts +// of the keys being compared, you must not use NewBloomFilterPolicy() +// and must provide your own FilterPolicy that also ignores the +// corresponding parts of the keys. For example, if the comparator +// ignores trailing spaces, it would be incorrect to use a +// FilterPolicy (like NewBloomFilterPolicy) that does not ignore +// trailing spaces in keys. +LEVELDB_EXPORT const FilterPolicy* NewBloomFilterPolicy(int bits_per_key); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ diff --git a/leveldb/include/leveldb/iterator.h b/leveldb/include/leveldb/iterator.h new file mode 100644 index 000000000..bb9a5df8f --- /dev/null +++ b/leveldb/include/leveldb/iterator.h @@ -0,0 +1,112 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// An iterator yields a sequence of key/value pairs from a source. +// The following class defines the interface. Multiple implementations +// are provided by this library. In particular, iterators are provided +// to access the contents of a Table or a DB. +// +// Multiple threads can invoke const methods on an Iterator without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Iterator must use +// external synchronization. + +#ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ +#define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ + +#include "leveldb/export.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" + +namespace leveldb { + +class LEVELDB_EXPORT Iterator { + public: + Iterator(); + + Iterator(const Iterator&) = delete; + Iterator& operator=(const Iterator&) = delete; + + virtual ~Iterator(); + + // An iterator is either positioned at a key/value pair, or + // not valid. This method returns true iff the iterator is valid. + virtual bool Valid() const = 0; + + // Position at the first key in the source. The iterator is Valid() + // after this call iff the source is not empty. + virtual void SeekToFirst() = 0; + + // Position at the last key in the source. The iterator is + // Valid() after this call iff the source is not empty. + virtual void SeekToLast() = 0; + + // Position at the first key in the source that is at or past target. + // The iterator is Valid() after this call iff the source contains + // an entry that comes at or past target. + virtual void Seek(const Slice& target) = 0; + + // Moves to the next entry in the source. After this call, Valid() is + // true iff the iterator was not positioned at the last entry in the source. + // REQUIRES: Valid() + virtual void Next() = 0; + + // Moves to the previous entry in the source. After this call, Valid() is + // true iff the iterator was not positioned at the first entry in source. + // REQUIRES: Valid() + virtual void Prev() = 0; + + // Return the key for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of + // the iterator. + // REQUIRES: Valid() + virtual Slice key() const = 0; + + // Return the value for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of + // the iterator. + // REQUIRES: Valid() + virtual Slice value() const = 0; + + // If an error has occurred, return it. Else return an ok status. + virtual Status status() const = 0; + + // Clients are allowed to register function/arg1/arg2 triples that + // will be invoked when this iterator is destroyed. + // + // Note that unlike all of the preceding methods, this method is + // not abstract and therefore clients should not override it. + using CleanupFunction = void (*)(void* arg1, void* arg2); + void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2); + + private: + // Cleanup functions are stored in a single-linked list. + // The list's head node is inlined in the iterator. + struct CleanupNode { + // True if the node is not used. Only head nodes might be unused. + bool IsEmpty() const { return function == nullptr; } + // Invokes the cleanup function. + void Run() { + assert(function != nullptr); + (*function)(arg1, arg2); + } + + // The head node is used if the function pointer is not null. + CleanupFunction function; + void* arg1; + void* arg2; + CleanupNode* next; + }; + CleanupNode cleanup_head_; +}; + +// Return an empty iterator (yields nothing). +LEVELDB_EXPORT Iterator* NewEmptyIterator(); + +// Return an empty iterator with the specified status. +LEVELDB_EXPORT Iterator* NewErrorIterator(const Status& status); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ diff --git a/leveldb/include/leveldb/options.h b/leveldb/include/leveldb/options.h new file mode 100644 index 000000000..79bcdbb2b --- /dev/null +++ b/leveldb/include/leveldb/options.h @@ -0,0 +1,186 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ +#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ + +#include + +#include "leveldb/export.h" + +namespace leveldb { + +class Cache; +class Comparator; +class Env; +class FilterPolicy; +class Logger; +class Snapshot; + +// DB contents are stored in a set of blocks, each of which holds a +// sequence of key,value pairs. Each block may be compressed before +// being stored in a file. The following enum describes which +// compression method (if any) is used to compress a block. +enum CompressionType { + // NOTE: do not change the values of existing entries, as these are + // part of the persistent format on disk. + kNoCompression = 0x0, + kSnappyCompression = 0x1, + kZstdCompression = 0x2, +}; + +// Options to control the behavior of a database (passed to DB::Open) +struct LEVELDB_EXPORT Options { + // Create an Options object with default values for all fields. + Options(); + + // ------------------- + // Parameters that affect behavior + + // Comparator used to define the order of keys in the table. + // Default: a comparator that uses lexicographic byte-wise ordering + // + // REQUIRES: The client must ensure that the comparator supplied + // here has the same name and orders keys *exactly* the same as the + // comparator provided to previous open calls on the same DB. + const Comparator* comparator; + + // If true, the database will be created if it is missing. + bool create_if_missing = false; + + // If true, an error is raised if the database already exists. + bool error_if_exists = false; + + // If true, the implementation will do aggressive checking of the + // data it is processing and will stop early if it detects any + // errors. This may have unforeseen ramifications: for example, a + // corruption of one DB entry may cause a large number of entries to + // become unreadable or for the entire DB to become unopenable. + bool paranoid_checks = false; + + // Use the specified object to interact with the environment, + // e.g. to read/write files, schedule background work, etc. + // Default: Env::Default() + Env* env; + + // Any internal progress/error information generated by the db will + // be written to info_log if it is non-null, or to a file stored + // in the same directory as the DB contents if info_log is null. + Logger* info_log = nullptr; + + // ------------------- + // Parameters that affect performance + + // Amount of data to build up in memory (backed by an unsorted log + // on disk) before converting to a sorted on-disk file. + // + // Larger values increase performance, especially during bulk loads. + // Up to two write buffers may be held in memory at the same time, + // so you may wish to adjust this parameter to control memory usage. + // Also, a larger write buffer will result in a longer recovery time + // the next time the database is opened. + size_t write_buffer_size = 4 * 1024 * 1024; + + // Number of open files that can be used by the DB. You may need to + // increase this if your database has a large working set (budget + // one open file per 2MB of working set). + int max_open_files = 1000; + + // Control over blocks (user data is stored in a set of blocks, and + // a block is the unit of reading from disk). + + // If non-null, use the specified cache for blocks. + // If null, leveldb will automatically create and use an 8MB internal cache. + Cache* block_cache = nullptr; + + // Approximate size of user data packed per block. Note that the + // block size specified here corresponds to uncompressed data. The + // actual size of the unit read from disk may be smaller if + // compression is enabled. This parameter can be changed dynamically. + size_t block_size = 4 * 1024; + + // Number of keys between restart points for delta encoding of keys. + // This parameter can be changed dynamically. Most clients should + // leave this parameter alone. + int block_restart_interval = 16; + + // Leveldb will write up to this amount of bytes to a file before + // switching to a new one. + // Most clients should leave this parameter alone. However if your + // filesystem is more efficient with larger files, you could + // consider increasing the value. The downside will be longer + // compactions and hence longer latency/performance hiccups. + // Another reason to increase this parameter might be when you are + // initially populating a large database. + size_t max_file_size = 2 * 1024 * 1024; + + // Compress blocks using the specified compression algorithm. This + // parameter can be changed dynamically. + // + // Default: kSnappyCompression, which gives lightweight but fast + // compression. + // + // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz: + // ~200-500MB/s compression + // ~400-800MB/s decompression + // Note that these speeds are significantly faster than most + // persistent storage speeds, and therefore it is typically never + // worth switching to kNoCompression. Even if the input data is + // incompressible, the kSnappyCompression implementation will + // efficiently detect that and will switch to uncompressed mode. + CompressionType compression = kSnappyCompression; + + // EXPERIMENTAL: If true, append to existing MANIFEST and log files + // when a database is opened. This can significantly speed up open. + // + // Default: currently false, but may become true later. + bool reuse_logs = false; + + // If non-null, use the specified filter policy to reduce disk reads. + // Many applications will benefit from passing the result of + // NewBloomFilterPolicy() here. + const FilterPolicy* filter_policy = nullptr; +}; + +// Options that control read operations +struct LEVELDB_EXPORT ReadOptions { + // If true, all data read from underlying storage will be + // verified against corresponding checksums. + bool verify_checksums = false; + + // Should the data read for this iteration be cached in memory? + // Callers may wish to set this field to false for bulk scans. + bool fill_cache = true; + + // If "snapshot" is non-null, read as of the supplied snapshot + // (which must belong to the DB that is being read and which must + // not have been released). If "snapshot" is null, use an implicit + // snapshot of the state at the beginning of this read operation. + const Snapshot* snapshot = nullptr; +}; + +// Options that control write operations +struct LEVELDB_EXPORT WriteOptions { + WriteOptions() = default; + + // If true, the write will be flushed from the operating system + // buffer cache (by calling WritableFile::Sync()) before the write + // is considered complete. If this flag is true, writes will be + // slower. + // + // If this flag is false, and the machine crashes, some recent + // writes may be lost. Note that if it is just the process that + // crashes (i.e., the machine does not reboot), no writes will be + // lost even if sync==false. + // + // In other words, a DB write with sync==false has similar + // crash semantics as the "write()" system call. A DB write + // with sync==true has similar crash semantics to a "write()" + // system call followed by "fsync()". + bool sync = false; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ diff --git a/leveldb/include/leveldb/slice.h b/leveldb/include/leveldb/slice.h new file mode 100644 index 000000000..37cb82178 --- /dev/null +++ b/leveldb/include/leveldb/slice.h @@ -0,0 +1,114 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Slice is a simple structure containing a pointer into some external +// storage and a size. The user of a Slice must ensure that the slice +// is not used after the corresponding external storage has been +// deallocated. +// +// Multiple threads can invoke const methods on a Slice without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Slice must use +// external synchronization. + +#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_ +#define STORAGE_LEVELDB_INCLUDE_SLICE_H_ + +#include +#include +#include +#include + +#include "leveldb/export.h" + +namespace leveldb { + +class LEVELDB_EXPORT Slice { + public: + // Create an empty slice. + Slice() : data_(""), size_(0) {} + + // Create a slice that refers to d[0,n-1]. + Slice(const char* d, size_t n) : data_(d), size_(n) {} + + // Create a slice that refers to the contents of "s" + Slice(const std::string& s) : data_(s.data()), size_(s.size()) {} + + // Create a slice that refers to s[0,strlen(s)-1] + Slice(const char* s) : data_(s), size_(strlen(s)) {} + + // Intentionally copyable. + Slice(const Slice&) = default; + Slice& operator=(const Slice&) = default; + + // Return a pointer to the beginning of the referenced data + const char* data() const { return data_; } + + // Return the length (in bytes) of the referenced data + size_t size() const { return size_; } + + // Return true iff the length of the referenced data is zero + bool empty() const { return size_ == 0; } + + // Return the ith byte in the referenced data. + // REQUIRES: n < size() + char operator[](size_t n) const { + assert(n < size()); + return data_[n]; + } + + // Change this slice to refer to an empty array + void clear() { + data_ = ""; + size_ = 0; + } + + // Drop the first "n" bytes from this slice. + void remove_prefix(size_t n) { + assert(n <= size()); + data_ += n; + size_ -= n; + } + + // Return a string that contains the copy of the referenced data. + std::string ToString() const { return std::string(data_, size_); } + + // Three-way comparison. Returns value: + // < 0 iff "*this" < "b", + // == 0 iff "*this" == "b", + // > 0 iff "*this" > "b" + int compare(const Slice& b) const; + + // Return true iff "x" is a prefix of "*this" + bool starts_with(const Slice& x) const { + return ((size_ >= x.size_) && (memcmp(data_, x.data_, x.size_) == 0)); + } + + private: + const char* data_; + size_t size_; +}; + +inline bool operator==(const Slice& x, const Slice& y) { + return ((x.size() == y.size()) && + (memcmp(x.data(), y.data(), x.size()) == 0)); +} + +inline bool operator!=(const Slice& x, const Slice& y) { return !(x == y); } + +inline int Slice::compare(const Slice& b) const { + const size_t min_len = (size_ < b.size_) ? size_ : b.size_; + int r = memcmp(data_, b.data_, min_len); + if (r == 0) { + if (size_ < b.size_) + r = -1; + else if (size_ > b.size_) + r = +1; + } + return r; +} + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_ diff --git a/leveldb/include/leveldb/status.h b/leveldb/include/leveldb/status.h new file mode 100644 index 000000000..e3273144e --- /dev/null +++ b/leveldb/include/leveldb/status.h @@ -0,0 +1,122 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A Status encapsulates the result of an operation. It may indicate success, +// or it may indicate an error with an associated error message. +// +// Multiple threads can invoke const methods on a Status without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Status must use +// external synchronization. + +#ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_ +#define STORAGE_LEVELDB_INCLUDE_STATUS_H_ + +#include +#include + +#include "leveldb/export.h" +#include "leveldb/slice.h" + +namespace leveldb { + +class LEVELDB_EXPORT Status { + public: + // Create a success status. + Status() noexcept : state_(nullptr) {} + ~Status() { delete[] state_; } + + Status(const Status& rhs); + Status& operator=(const Status& rhs); + + Status(Status&& rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; } + Status& operator=(Status&& rhs) noexcept; + + // Return a success status. + static Status OK() { return Status(); } + + // Return error status of an appropriate type. + static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kNotFound, msg, msg2); + } + static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kCorruption, msg, msg2); + } + static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kNotSupported, msg, msg2); + } + static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kInvalidArgument, msg, msg2); + } + static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIOError, msg, msg2); + } + + // Returns true iff the status indicates success. + bool ok() const { return (state_ == nullptr); } + + // Returns true iff the status indicates a NotFound error. + bool IsNotFound() const { return code() == kNotFound; } + + // Returns true iff the status indicates a Corruption error. + bool IsCorruption() const { return code() == kCorruption; } + + // Returns true iff the status indicates an IOError. + bool IsIOError() const { return code() == kIOError; } + + // Returns true iff the status indicates a NotSupportedError. + bool IsNotSupportedError() const { return code() == kNotSupported; } + + // Returns true iff the status indicates an InvalidArgument. + bool IsInvalidArgument() const { return code() == kInvalidArgument; } + + // Return a string representation of this status suitable for printing. + // Returns the string "OK" for success. + std::string ToString() const; + + private: + enum Code { + kOk = 0, + kNotFound = 1, + kCorruption = 2, + kNotSupported = 3, + kInvalidArgument = 4, + kIOError = 5 + }; + + Code code() const { + return (state_ == nullptr) ? kOk : static_cast(state_[4]); + } + + Status(Code code, const Slice& msg, const Slice& msg2); + static const char* CopyState(const char* s); + + // OK status has a null state_. Otherwise, state_ is a new[] array + // of the following form: + // state_[0..3] == length of message + // state_[4] == code + // state_[5..] == message + const char* state_; +}; + +inline Status::Status(const Status& rhs) { + state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); +} +inline Status& Status::operator=(const Status& rhs) { + // The following condition catches both aliasing (when this == &rhs), + // and the common case where both rhs and *this are ok. + if (state_ != rhs.state_) { + delete[] state_; + state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); + } + return *this; +} +inline Status& Status::operator=(Status&& rhs) noexcept { + std::swap(state_, rhs.state_); + return *this; +} + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_STATUS_H_ diff --git a/leveldb/include/leveldb/table.h b/leveldb/include/leveldb/table.h new file mode 100644 index 000000000..a30e90301 --- /dev/null +++ b/leveldb/include/leveldb/table.h @@ -0,0 +1,84 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_H_ +#define STORAGE_LEVELDB_INCLUDE_TABLE_H_ + +#include + +#include "leveldb/export.h" +#include "leveldb/iterator.h" + +namespace leveldb { + +class Block; +class BlockHandle; +class Footer; +struct Options; +class RandomAccessFile; +struct ReadOptions; +class TableCache; + +// A Table is a sorted map from strings to strings. Tables are +// immutable and persistent. A Table may be safely accessed from +// multiple threads without external synchronization. +class LEVELDB_EXPORT Table { + public: + // Attempt to open the table that is stored in bytes [0..file_size) + // of "file", and read the metadata entries necessary to allow + // retrieving data from the table. + // + // If successful, returns ok and sets "*table" to the newly opened + // table. The client should delete "*table" when no longer needed. + // If there was an error while initializing the table, sets "*table" + // to nullptr and returns a non-ok status. Does not take ownership of + // "*source", but the client must ensure that "source" remains live + // for the duration of the returned table's lifetime. + // + // *file must remain live while this Table is in use. + static Status Open(const Options& options, RandomAccessFile* file, + uint64_t file_size, Table** table); + + Table(const Table&) = delete; + Table& operator=(const Table&) = delete; + + ~Table(); + + // Returns a new iterator over the table contents. + // The result of NewIterator() is initially invalid (caller must + // call one of the Seek methods on the iterator before using it). + Iterator* NewIterator(const ReadOptions&) const; + + // Given a key, return an approximate byte offset in the file where + // the data for that key begins (or would begin if the key were + // present in the file). The returned value is in terms of file + // bytes, and so includes effects like compression of the underlying data. + // E.g., the approximate offset of the last key in the table will + // be close to the file length. + uint64_t ApproximateOffsetOf(const Slice& key) const; + + private: + friend class TableCache; + struct Rep; + + static Iterator* BlockReader(void*, const ReadOptions&, const Slice&); + + explicit Table(Rep* rep) : rep_(rep) {} + + // Calls (*handle_result)(arg, ...) with the entry found after a call + // to Seek(key). May not make such a call if filter policy says + // that key is not present. + Status InternalGet(const ReadOptions&, const Slice& key, void* arg, + void (*handle_result)(void* arg, const Slice& k, + const Slice& v)); + + void ReadMeta(const Footer& footer); + void ReadFilter(const Slice& filter_handle_value); + + Rep* const rep_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_TABLE_H_ diff --git a/leveldb/include/leveldb/table_builder.h b/leveldb/include/leveldb/table_builder.h new file mode 100644 index 000000000..85710c332 --- /dev/null +++ b/leveldb/include/leveldb/table_builder.h @@ -0,0 +1,93 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// TableBuilder provides the interface used to build a Table +// (an immutable and sorted map from keys to values). +// +// Multiple threads can invoke const methods on a TableBuilder without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same TableBuilder must use +// external synchronization. + +#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ +#define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ + +#include + +#include "leveldb/export.h" +#include "leveldb/options.h" +#include "leveldb/status.h" + +namespace leveldb { + +class BlockBuilder; +class BlockHandle; +class WritableFile; + +class LEVELDB_EXPORT TableBuilder { + public: + // Create a builder that will store the contents of the table it is + // building in *file. Does not close the file. It is up to the + // caller to close the file after calling Finish(). + TableBuilder(const Options& options, WritableFile* file); + + TableBuilder(const TableBuilder&) = delete; + TableBuilder& operator=(const TableBuilder&) = delete; + + // REQUIRES: Either Finish() or Abandon() has been called. + ~TableBuilder(); + + // Change the options used by this builder. Note: only some of the + // option fields can be changed after construction. If a field is + // not allowed to change dynamically and its value in the structure + // passed to the constructor is different from its value in the + // structure passed to this method, this method will return an error + // without changing any fields. + Status ChangeOptions(const Options& options); + + // Add key,value to the table being constructed. + // REQUIRES: key is after any previously added key according to comparator. + // REQUIRES: Finish(), Abandon() have not been called + void Add(const Slice& key, const Slice& value); + + // Advanced operation: flush any buffered key/value pairs to file. + // Can be used to ensure that two adjacent entries never live in + // the same data block. Most clients should not need to use this method. + // REQUIRES: Finish(), Abandon() have not been called + void Flush(); + + // Return non-ok iff some error has been detected. + Status status() const; + + // Finish building the table. Stops using the file passed to the + // constructor after this function returns. + // REQUIRES: Finish(), Abandon() have not been called + Status Finish(); + + // Indicate that the contents of this builder should be abandoned. Stops + // using the file passed to the constructor after this function returns. + // If the caller is not going to call Finish(), it must call Abandon() + // before destroying this builder. + // REQUIRES: Finish(), Abandon() have not been called + void Abandon(); + + // Number of calls to Add() so far. + uint64_t NumEntries() const; + + // Size of the file generated so far. If invoked after a successful + // Finish() call, returns the size of the final generated file. + uint64_t FileSize() const; + + private: + bool ok() const { return status().ok(); } + void WriteBlock(BlockBuilder* block, BlockHandle* handle); + void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle); + + struct Rep; + Rep* rep_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ diff --git a/leveldb/include/leveldb/write_batch.h b/leveldb/include/leveldb/write_batch.h new file mode 100644 index 000000000..94d4115fe --- /dev/null +++ b/leveldb/include/leveldb/write_batch.h @@ -0,0 +1,83 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// WriteBatch holds a collection of updates to apply atomically to a DB. +// +// The updates are applied in the order in which they are added +// to the WriteBatch. For example, the value of "key" will be "v3" +// after the following batch is written: +// +// batch.Put("key", "v1"); +// batch.Delete("key"); +// batch.Put("key", "v2"); +// batch.Put("key", "v3"); +// +// Multiple threads can invoke const methods on a WriteBatch without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same WriteBatch must use +// external synchronization. + +#ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ +#define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ + +#include + +#include "leveldb/export.h" +#include "leveldb/status.h" + +namespace leveldb { + +class Slice; + +class LEVELDB_EXPORT WriteBatch { + public: + class LEVELDB_EXPORT Handler { + public: + virtual ~Handler(); + virtual void Put(const Slice& key, const Slice& value) = 0; + virtual void Delete(const Slice& key) = 0; + }; + + WriteBatch(); + + // Intentionally copyable. + WriteBatch(const WriteBatch&) = default; + WriteBatch& operator=(const WriteBatch&) = default; + + ~WriteBatch(); + + // Store the mapping "key->value" in the database. + void Put(const Slice& key, const Slice& value); + + // If the database contains a mapping for "key", erase it. Else do nothing. + void Delete(const Slice& key); + + // Clear all updates buffered in this batch. + void Clear(); + + // The size of the database changes caused by this batch. + // + // This number is tied to implementation details, and may change across + // releases. It is intended for LevelDB usage metrics. + size_t ApproximateSize() const; + + // Copies the operations in "source" to this batch. + // + // This runs in O(source size) time. However, the constant factor is better + // than calling Iterate() over the source batch with a Handler that replicates + // the operations into this batch. + void Append(const WriteBatch& source); + + // Support for iterating over the contents of a batch. + Status Iterate(Handler* handler) const; + + private: + friend class WriteBatchInternal; + + std::string rep_; // See comment in write_batch.cc for the format of rep_ +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ diff --git a/leveldb/issues/issue178_test.cc b/leveldb/issues/issue178_test.cc new file mode 100644 index 000000000..5cd5862d5 --- /dev/null +++ b/leveldb/issues/issue178_test.cc @@ -0,0 +1,85 @@ +// Copyright (c) 2013 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Test for issue 178: a manual compaction causes deleted data to reappear. +#include +#include +#include + +#include "gtest/gtest.h" +#include "leveldb/db.h" +#include "leveldb/write_batch.h" +#include "util/testutil.h" + +namespace { + +const int kNumKeys = 1100000; + +std::string Key1(int i) { + char buf[100]; + std::snprintf(buf, sizeof(buf), "my_key_%d", i); + return buf; +} + +std::string Key2(int i) { return Key1(i) + "_xxx"; } + +TEST(Issue178, Test) { + // Get rid of any state from an old run. + std::string dbpath = testing::TempDir() + "leveldb_cbug_test"; + DestroyDB(dbpath, leveldb::Options()); + + // Open database. Disable compression since it affects the creation + // of layers and the code below is trying to test against a very + // specific scenario. + leveldb::DB* db; + leveldb::Options db_options; + db_options.create_if_missing = true; + db_options.compression = leveldb::kNoCompression; + ASSERT_LEVELDB_OK(leveldb::DB::Open(db_options, dbpath, &db)); + + // create first key range + leveldb::WriteBatch batch; + for (size_t i = 0; i < kNumKeys; i++) { + batch.Put(Key1(i), "value for range 1 key"); + } + ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // create second key range + batch.Clear(); + for (size_t i = 0; i < kNumKeys; i++) { + batch.Put(Key2(i), "value for range 2 key"); + } + ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // delete second key range + batch.Clear(); + for (size_t i = 0; i < kNumKeys; i++) { + batch.Delete(Key2(i)); + } + ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // compact database + std::string start_key = Key1(0); + std::string end_key = Key1(kNumKeys - 1); + leveldb::Slice least(start_key.data(), start_key.size()); + leveldb::Slice greatest(end_key.data(), end_key.size()); + + // commenting out the line below causes the example to work correctly + db->CompactRange(&least, &greatest); + + // count the keys + leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions()); + size_t num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + num_keys++; + } + delete iter; + ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; + + // close database + delete db; + DestroyDB(dbpath, leveldb::Options()); +} + +} // anonymous namespace diff --git a/leveldb/issues/issue200_test.cc b/leveldb/issues/issue200_test.cc new file mode 100644 index 000000000..959b3713c --- /dev/null +++ b/leveldb/issues/issue200_test.cc @@ -0,0 +1,54 @@ +// Copyright (c) 2013 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Test for issue 200: when iterator switches direction from backward +// to forward, the current key can be yielded unexpectedly if a new +// mutation has been added just before the current key. + +#include "gtest/gtest.h" +#include "leveldb/db.h" +#include "util/testutil.h" + +namespace leveldb { + +TEST(Issue200, Test) { + // Get rid of any state from an old run. + std::string dbpath = testing::TempDir() + "leveldb_issue200_test"; + DestroyDB(dbpath, Options()); + + DB* db; + Options options; + options.create_if_missing = true; + ASSERT_LEVELDB_OK(DB::Open(options, dbpath, &db)); + + WriteOptions write_options; + ASSERT_LEVELDB_OK(db->Put(write_options, "1", "b")); + ASSERT_LEVELDB_OK(db->Put(write_options, "2", "c")); + ASSERT_LEVELDB_OK(db->Put(write_options, "3", "d")); + ASSERT_LEVELDB_OK(db->Put(write_options, "4", "e")); + ASSERT_LEVELDB_OK(db->Put(write_options, "5", "f")); + + ReadOptions read_options; + Iterator* iter = db->NewIterator(read_options); + + // Add an element that should not be reflected in the iterator. + ASSERT_LEVELDB_OK(db->Put(write_options, "25", "cd")); + + iter->Seek("5"); + ASSERT_EQ(iter->key().ToString(), "5"); + iter->Prev(); + ASSERT_EQ(iter->key().ToString(), "4"); + iter->Prev(); + ASSERT_EQ(iter->key().ToString(), "3"); + iter->Next(); + ASSERT_EQ(iter->key().ToString(), "4"); + iter->Next(); + ASSERT_EQ(iter->key().ToString(), "5"); + + delete iter; + delete db; + DestroyDB(dbpath, options); +} + +} // namespace leveldb diff --git a/leveldb/issues/issue320_test.cc b/leveldb/issues/issue320_test.cc new file mode 100644 index 000000000..9d7fa7b75 --- /dev/null +++ b/leveldb/issues/issue320_test.cc @@ -0,0 +1,126 @@ +// Copyright (c) 2019 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "leveldb/db.h" +#include "leveldb/write_batch.h" +#include "util/testutil.h" + +namespace leveldb { + +namespace { + +// Creates a random number in the range of [0, max). +int GenerateRandomNumber(int max) { return std::rand() % max; } + +std::string CreateRandomString(int32_t index) { + static const size_t len = 1024; + char bytes[len]; + size_t i = 0; + while (i < 8) { + bytes[i] = 'a' + ((index >> (4 * i)) & 0xf); + ++i; + } + while (i < sizeof(bytes)) { + bytes[i] = 'a' + GenerateRandomNumber(26); + ++i; + } + return std::string(bytes, sizeof(bytes)); +} + +} // namespace + +TEST(Issue320, Test) { + std::srand(0); + + bool delete_before_put = false; + bool keep_snapshots = true; + + std::vector>> test_map( + 10000); + std::vector snapshots(100, nullptr); + + DB* db; + Options options; + options.create_if_missing = true; + + std::string dbpath = testing::TempDir() + "leveldb_issue320_test"; + ASSERT_LEVELDB_OK(DB::Open(options, dbpath, &db)); + + uint32_t target_size = 10000; + uint32_t num_items = 0; + uint32_t count = 0; + std::string key; + std::string value, old_value; + + WriteOptions writeOptions; + ReadOptions readOptions; + while (count < 200000) { + if ((++count % 1000) == 0) { + std::cout << "count: " << count << std::endl; + } + + int index = GenerateRandomNumber(test_map.size()); + WriteBatch batch; + + if (test_map[index] == nullptr) { + num_items++; + test_map[index].reset(new std::pair( + CreateRandomString(index), CreateRandomString(index))); + batch.Put(test_map[index]->first, test_map[index]->second); + } else { + ASSERT_LEVELDB_OK( + db->Get(readOptions, test_map[index]->first, &old_value)); + if (old_value != test_map[index]->second) { + std::cout << "ERROR incorrect value returned by Get" << std::endl; + std::cout << " count=" << count << std::endl; + std::cout << " old value=" << old_value << std::endl; + std::cout << " test_map[index]->second=" << test_map[index]->second + << std::endl; + std::cout << " test_map[index]->first=" << test_map[index]->first + << std::endl; + std::cout << " index=" << index << std::endl; + ASSERT_EQ(old_value, test_map[index]->second); + } + + if (num_items >= target_size && GenerateRandomNumber(100) > 30) { + batch.Delete(test_map[index]->first); + test_map[index] = nullptr; + --num_items; + } else { + test_map[index]->second = CreateRandomString(index); + if (delete_before_put) batch.Delete(test_map[index]->first); + batch.Put(test_map[index]->first, test_map[index]->second); + } + } + + ASSERT_LEVELDB_OK(db->Write(writeOptions, &batch)); + + if (keep_snapshots && GenerateRandomNumber(10) == 0) { + int i = GenerateRandomNumber(snapshots.size()); + if (snapshots[i] != nullptr) { + db->ReleaseSnapshot(snapshots[i]); + } + snapshots[i] = db->GetSnapshot(); + } + } + + for (Snapshot const* snapshot : snapshots) { + if (snapshot) { + db->ReleaseSnapshot(snapshot); + } + } + + delete db; + DestroyDB(dbpath, options); +} + +} // namespace leveldb diff --git a/leveldb/port/README.md b/leveldb/port/README.md new file mode 100644 index 000000000..8b171532e --- /dev/null +++ b/leveldb/port/README.md @@ -0,0 +1,10 @@ +This directory contains interfaces and implementations that isolate the +rest of the package from platform details. + +Code in the rest of the package includes "port.h" from this directory. +"port.h" in turn includes a platform specific "port_.h" file +that provides the platform specific implementation. + +See port_stdcxx.h for an example of what must be provided in a platform +specific header file. + diff --git a/leveldb/port/port.h b/leveldb/port/port.h new file mode 100644 index 000000000..4b247f74f --- /dev/null +++ b/leveldb/port/port.h @@ -0,0 +1,19 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_PORT_PORT_H_ +#define STORAGE_LEVELDB_PORT_PORT_H_ + +#include + +// Include the appropriate platform specific file below. If you are +// porting to a new platform, see "port_example.h" for documentation +// of what the new port_.h file must provide. +#if defined(LEVELDB_PLATFORM_POSIX) || defined(LEVELDB_PLATFORM_WINDOWS) +#include "port/port_stdcxx.h" +#elif defined(LEVELDB_PLATFORM_CHROMIUM) +#include "port/port_chromium.h" +#endif + +#endif // STORAGE_LEVELDB_PORT_PORT_H_ diff --git a/leveldb/port/port_config.h.in b/leveldb/port/port_config.h.in new file mode 100644 index 000000000..34bf66a68 --- /dev/null +++ b/leveldb/port/port_config.h.in @@ -0,0 +1,38 @@ +// Copyright 2017 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ +#define STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ + +// Define to 1 if you have a definition for fdatasync() in . +#if !defined(HAVE_FDATASYNC) +#cmakedefine01 HAVE_FDATASYNC +#endif // !defined(HAVE_FDATASYNC) + +// Define to 1 if you have a definition for F_FULLFSYNC in . +#if !defined(HAVE_FULLFSYNC) +#cmakedefine01 HAVE_FULLFSYNC +#endif // !defined(HAVE_FULLFSYNC) + +// Define to 1 if you have a definition for O_CLOEXEC in . +#if !defined(HAVE_O_CLOEXEC) +#cmakedefine01 HAVE_O_CLOEXEC +#endif // !defined(HAVE_O_CLOEXEC) + +// Define to 1 if you have Google CRC32C. +#if !defined(HAVE_CRC32C) +#cmakedefine01 HAVE_CRC32C +#endif // !defined(HAVE_CRC32C) + +// Define to 1 if you have Google Snappy. +#if !defined(HAVE_SNAPPY) +#cmakedefine01 HAVE_SNAPPY +#endif // !defined(HAVE_SNAPPY) + +// Define to 1 if you have Zstd. +#if !defined(HAVE_Zstd) +#cmakedefine01 HAVE_ZSTD +#endif // !defined(HAVE_ZSTD) + +#endif // STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ diff --git a/leveldb/port/port_example.h b/leveldb/port/port_example.h new file mode 100644 index 000000000..b1a1c3226 --- /dev/null +++ b/leveldb/port/port_example.h @@ -0,0 +1,119 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// This file contains the specification, but not the implementations, +// of the types/operations/etc. that should be defined by a platform +// specific port_.h file. Use this file as a reference for +// how to port this package to a new platform. + +#ifndef STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ +#define STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ + +#include "port/thread_annotations.h" + +namespace leveldb { +namespace port { + +// TODO(jorlow): Many of these belong more in the environment class rather than +// here. We should try moving them and see if it affects perf. + +// ------------------ Threading ------------------- + +// A Mutex represents an exclusive lock. +class LOCKABLE Mutex { + public: + Mutex(); + ~Mutex(); + + // Lock the mutex. Waits until other lockers have exited. + // Will deadlock if the mutex is already locked by this thread. + void Lock() EXCLUSIVE_LOCK_FUNCTION(); + + // Unlock the mutex. + // REQUIRES: This mutex was locked by this thread. + void Unlock() UNLOCK_FUNCTION(); + + // Optionally crash if this thread does not hold this mutex. + // The implementation must be fast, especially if NDEBUG is + // defined. The implementation is allowed to skip all checks. + void AssertHeld() ASSERT_EXCLUSIVE_LOCK(); +}; + +class CondVar { + public: + explicit CondVar(Mutex* mu); + ~CondVar(); + + // Atomically release *mu and block on this condition variable until + // either a call to SignalAll(), or a call to Signal() that picks + // this thread to wakeup. + // REQUIRES: this thread holds *mu + void Wait(); + + // If there are some threads waiting, wake up at least one of them. + void Signal(); + + // Wake up all waiting threads. + void SignalAll(); +}; + +// ------------------ Compression ------------------- + +// Store the snappy compression of "input[0,input_length-1]" in *output. +// Returns false if snappy is not supported by this port. +bool Snappy_Compress(const char* input, size_t input_length, + std::string* output); + +// If input[0,input_length-1] looks like a valid snappy compressed +// buffer, store the size of the uncompressed data in *result and +// return true. Else return false. +bool Snappy_GetUncompressedLength(const char* input, size_t length, + size_t* result); + +// Attempt to snappy uncompress input[0,input_length-1] into *output. +// Returns true if successful, false if the input is invalid snappy +// compressed data. +// +// REQUIRES: at least the first "n" bytes of output[] must be writable +// where "n" is the result of a successful call to +// Snappy_GetUncompressedLength. +bool Snappy_Uncompress(const char* input_data, size_t input_length, + char* output); + +// Store the zstd compression of "input[0,input_length-1]" in *output. +// Returns false if zstd is not supported by this port. +bool Zstd_Compress(const char* input, size_t input_length, std::string* output); + +// If input[0,input_length-1] looks like a valid zstd compressed +// buffer, store the size of the uncompressed data in *result and +// return true. Else return false. +bool Zstd_GetUncompressedLength(const char* input, size_t length, + size_t* result); + +// Attempt to zstd uncompress input[0,input_length-1] into *output. +// Returns true if successful, false if the input is invalid zstd +// compressed data. +// +// REQUIRES: at least the first "n" bytes of output[] must be writable +// where "n" is the result of a successful call to +// Zstd_GetUncompressedLength. +bool Zstd_Uncompress(const char* input_data, size_t input_length, char* output); + +// ------------------ Miscellaneous ------------------- + +// If heap profiling is not supported, returns false. +// Else repeatedly calls (*func)(arg, data, n) and then returns true. +// The concatenation of all "data[0,n-1]" fragments is the heap profile. +bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg); + +// Extend the CRC to include the first n bytes of buf. +// +// Returns zero if the CRC cannot be extended using acceleration, else returns +// the newly extended CRC value (which may also be zero). +uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size); + +} // namespace port +} // namespace leveldb + +#endif // STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ diff --git a/leveldb/port/port_stdcxx.h b/leveldb/port/port_stdcxx.h new file mode 100644 index 000000000..ca961e657 --- /dev/null +++ b/leveldb/port/port_stdcxx.h @@ -0,0 +1,218 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_PORT_PORT_STDCXX_H_ +#define STORAGE_LEVELDB_PORT_PORT_STDCXX_H_ + +// port/port_config.h availability is automatically detected via __has_include +// in newer compilers. If LEVELDB_HAS_PORT_CONFIG_H is defined, it overrides the +// configuration detection. +#if defined(LEVELDB_HAS_PORT_CONFIG_H) + +#if LEVELDB_HAS_PORT_CONFIG_H +#include "port/port_config.h" +#endif // LEVELDB_HAS_PORT_CONFIG_H + +#elif defined(__has_include) + +#if __has_include("port/port_config.h") +#include "port/port_config.h" +#endif // __has_include("port/port_config.h") + +#endif // defined(LEVELDB_HAS_PORT_CONFIG_H) + +#if HAVE_CRC32C +#include +#endif // HAVE_CRC32C +#if HAVE_SNAPPY +#include +#endif // HAVE_SNAPPY +#if HAVE_ZSTD +#include +#endif // HAVE_ZSTD + +#include +#include // NOLINT +#include +#include +#include // NOLINT +#include + +#include "port/thread_annotations.h" + +namespace leveldb { +namespace port { + +class CondVar; + +// Thinly wraps std::mutex. +class LOCKABLE Mutex { + public: + Mutex() = default; + ~Mutex() = default; + + Mutex(const Mutex&) = delete; + Mutex& operator=(const Mutex&) = delete; + + void Lock() EXCLUSIVE_LOCK_FUNCTION() { mu_.lock(); } + void Unlock() UNLOCK_FUNCTION() { mu_.unlock(); } + void AssertHeld() ASSERT_EXCLUSIVE_LOCK() {} + + private: + friend class CondVar; + std::mutex mu_; +}; + +// Thinly wraps std::condition_variable. +class CondVar { + public: + explicit CondVar(Mutex* mu) : mu_(mu) { assert(mu != nullptr); } + ~CondVar() = default; + + CondVar(const CondVar&) = delete; + CondVar& operator=(const CondVar&) = delete; + + void Wait() { + std::unique_lock lock(mu_->mu_, std::adopt_lock); + cv_.wait(lock); + lock.release(); + } + void Signal() { cv_.notify_one(); } + void SignalAll() { cv_.notify_all(); } + + private: + std::condition_variable cv_; + Mutex* const mu_; +}; + +inline bool Snappy_Compress(const char* input, size_t length, + std::string* output) { +#if HAVE_SNAPPY + output->resize(snappy::MaxCompressedLength(length)); + size_t outlen; + snappy::RawCompress(input, length, &(*output)[0], &outlen); + output->resize(outlen); + return true; +#else + // Silence compiler warnings about unused arguments. + (void)input; + (void)length; + (void)output; +#endif // HAVE_SNAPPY + + return false; +} + +inline bool Snappy_GetUncompressedLength(const char* input, size_t length, + size_t* result) { +#if HAVE_SNAPPY + return snappy::GetUncompressedLength(input, length, result); +#else + // Silence compiler warnings about unused arguments. + (void)input; + (void)length; + (void)result; + return false; +#endif // HAVE_SNAPPY +} + +inline bool Snappy_Uncompress(const char* input, size_t length, char* output) { +#if HAVE_SNAPPY + return snappy::RawUncompress(input, length, output); +#else + // Silence compiler warnings about unused arguments. + (void)input; + (void)length; + (void)output; + return false; +#endif // HAVE_SNAPPY +} + +inline bool Zstd_Compress(const char* input, size_t length, + std::string* output) { +#if HAVE_ZSTD + // Get the MaxCompressedLength. + size_t outlen = ZSTD_compressBound(length); + if (ZSTD_isError(outlen)) { + return false; + } + output->resize(outlen); + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + outlen = ZSTD_compress2(ctx, &(*output)[0], output->size(), input, length); + ZSTD_freeCCtx(ctx); + if (ZSTD_isError(outlen)) { + return false; + } + output->resize(outlen); + return true; +#else + // Silence compiler warnings about unused arguments. + (void)input; + (void)length; + (void)output; + return false; +#endif // HAVE_ZSTD +} + +inline bool Zstd_GetUncompressedLength(const char* input, size_t length, + size_t* result) { +#if HAVE_ZSTD + size_t size = ZSTD_getFrameContentSize(input, length); + if (size == 0) return false; + *result = size; + return true; +#else + // Silence compiler warnings about unused arguments. + (void)input; + (void)length; + (void)result; + return false; +#endif // HAVE_ZSTD +} + +inline bool Zstd_Uncompress(const char* input, size_t length, char* output) { +#if HAVE_ZSTD + size_t outlen; + if (!Zstd_GetUncompressedLength(input, length, &outlen)) { + return false; + } + ZSTD_DCtx* ctx = ZSTD_createDCtx(); + outlen = ZSTD_decompressDCtx(ctx, output, outlen, input, length); + ZSTD_freeDCtx(ctx); + if (ZSTD_isError(outlen)) { + return false; + } + return true; +#else + // Silence compiler warnings about unused arguments. + (void)input; + (void)length; + (void)output; + return false; +#endif // HAVE_ZSTD +} + +inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { + // Silence compiler warnings about unused arguments. + (void)func; + (void)arg; + return false; +} + +inline uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) { +#if HAVE_CRC32C + return ::crc32c::Extend(crc, reinterpret_cast(buf), size); +#else + // Silence compiler warnings about unused arguments. + (void)crc; + (void)buf; + (void)size; + return 0; +#endif // HAVE_CRC32C +} + +} // namespace port +} // namespace leveldb + +#endif // STORAGE_LEVELDB_PORT_PORT_STDCXX_H_ diff --git a/leveldb/port/thread_annotations.h b/leveldb/port/thread_annotations.h new file mode 100644 index 000000000..1547df908 --- /dev/null +++ b/leveldb/port/thread_annotations.h @@ -0,0 +1,108 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ +#define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ + +// Use Clang's thread safety analysis annotations when available. In other +// environments, the macros receive empty definitions. +// Usage documentation: https://clang.llvm.org/docs/ThreadSafetyAnalysis.html + +#if !defined(THREAD_ANNOTATION_ATTRIBUTE__) + +#if defined(__clang__) + +#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#else +#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op +#endif + +#endif // !defined(THREAD_ANNOTATION_ATTRIBUTE__) + +#ifndef GUARDED_BY +#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) +#endif + +#ifndef PT_GUARDED_BY +#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) +#endif + +#ifndef ACQUIRED_AFTER +#define ACQUIRED_AFTER(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) +#endif + +#ifndef ACQUIRED_BEFORE +#define ACQUIRED_BEFORE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) +#endif + +#ifndef EXCLUSIVE_LOCKS_REQUIRED +#define EXCLUSIVE_LOCKS_REQUIRED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) +#endif + +#ifndef SHARED_LOCKS_REQUIRED +#define SHARED_LOCKS_REQUIRED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) +#endif + +#ifndef LOCKS_EXCLUDED +#define LOCKS_EXCLUDED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) +#endif + +#ifndef LOCK_RETURNED +#define LOCK_RETURNED(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) +#endif + +#ifndef LOCKABLE +#define LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(lockable) +#endif + +#ifndef SCOPED_LOCKABLE +#define SCOPED_LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) +#endif + +#ifndef EXCLUSIVE_LOCK_FUNCTION +#define EXCLUSIVE_LOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) +#endif + +#ifndef SHARED_LOCK_FUNCTION +#define SHARED_LOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) +#endif + +#ifndef EXCLUSIVE_TRYLOCK_FUNCTION +#define EXCLUSIVE_TRYLOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) +#endif + +#ifndef SHARED_TRYLOCK_FUNCTION +#define SHARED_TRYLOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) +#endif + +#ifndef UNLOCK_FUNCTION +#define UNLOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) +#endif + +#ifndef NO_THREAD_SAFETY_ANALYSIS +#define NO_THREAD_SAFETY_ANALYSIS \ + THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) +#endif + +#ifndef ASSERT_EXCLUSIVE_LOCK +#define ASSERT_EXCLUSIVE_LOCK(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) +#endif + +#ifndef ASSERT_SHARED_LOCK +#define ASSERT_SHARED_LOCK(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) +#endif + +#endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ diff --git a/leveldb/table/block.cc b/leveldb/table/block.cc new file mode 100644 index 000000000..3b1525770 --- /dev/null +++ b/leveldb/table/block.cc @@ -0,0 +1,292 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Decodes the blocks generated by block_builder.cc. + +#include "table/block.h" + +#include +#include +#include + +#include "leveldb/comparator.h" +#include "table/format.h" +#include "util/coding.h" +#include "util/logging.h" + +namespace leveldb { + +inline uint32_t Block::NumRestarts() const { + assert(size_ >= sizeof(uint32_t)); + return DecodeFixed32(data_ + size_ - sizeof(uint32_t)); +} + +Block::Block(const BlockContents& contents) + : data_(contents.data.data()), + size_(contents.data.size()), + owned_(contents.heap_allocated) { + if (size_ < sizeof(uint32_t)) { + size_ = 0; // Error marker + } else { + size_t max_restarts_allowed = (size_ - sizeof(uint32_t)) / sizeof(uint32_t); + if (NumRestarts() > max_restarts_allowed) { + // The size is too small for NumRestarts() + size_ = 0; + } else { + restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t); + } + } +} + +Block::~Block() { + if (owned_) { + delete[] data_; + } +} + +// Helper routine: decode the next block entry starting at "p", +// storing the number of shared key bytes, non_shared key bytes, +// and the length of the value in "*shared", "*non_shared", and +// "*value_length", respectively. Will not dereference past "limit". +// +// If any errors are detected, returns nullptr. Otherwise, returns a +// pointer to the key delta (just past the three decoded values). +static inline const char* DecodeEntry(const char* p, const char* limit, + uint32_t* shared, uint32_t* non_shared, + uint32_t* value_length) { + if (limit - p < 3) return nullptr; + *shared = reinterpret_cast(p)[0]; + *non_shared = reinterpret_cast(p)[1]; + *value_length = reinterpret_cast(p)[2]; + if ((*shared | *non_shared | *value_length) < 128) { + // Fast path: all three values are encoded in one byte each + p += 3; + } else { + if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, value_length)) == nullptr) return nullptr; + } + + if (static_cast(limit - p) < (*non_shared + *value_length)) { + return nullptr; + } + return p; +} + +class Block::Iter : public Iterator { + private: + const Comparator* const comparator_; + const char* const data_; // underlying block contents + uint32_t const restarts_; // Offset of restart array (list of fixed32) + uint32_t const num_restarts_; // Number of uint32_t entries in restart array + + // current_ is offset in data_ of current entry. >= restarts_ if !Valid + uint32_t current_; + uint32_t restart_index_; // Index of restart block in which current_ falls + std::string key_; + Slice value_; + Status status_; + + inline int Compare(const Slice& a, const Slice& b) const { + return comparator_->Compare(a, b); + } + + // Return the offset in data_ just past the end of the current entry. + inline uint32_t NextEntryOffset() const { + return (value_.data() + value_.size()) - data_; + } + + uint32_t GetRestartPoint(uint32_t index) { + assert(index < num_restarts_); + return DecodeFixed32(data_ + restarts_ + index * sizeof(uint32_t)); + } + + void SeekToRestartPoint(uint32_t index) { + key_.clear(); + restart_index_ = index; + // current_ will be fixed by ParseNextKey(); + + // ParseNextKey() starts at the end of value_, so set value_ accordingly + uint32_t offset = GetRestartPoint(index); + value_ = Slice(data_ + offset, 0); + } + + public: + Iter(const Comparator* comparator, const char* data, uint32_t restarts, + uint32_t num_restarts) + : comparator_(comparator), + data_(data), + restarts_(restarts), + num_restarts_(num_restarts), + current_(restarts_), + restart_index_(num_restarts_) { + assert(num_restarts_ > 0); + } + + bool Valid() const override { return current_ < restarts_; } + Status status() const override { return status_; } + Slice key() const override { + assert(Valid()); + return key_; + } + Slice value() const override { + assert(Valid()); + return value_; + } + + void Next() override { + assert(Valid()); + ParseNextKey(); + } + + void Prev() override { + assert(Valid()); + + // Scan backwards to a restart point before current_ + const uint32_t original = current_; + while (GetRestartPoint(restart_index_) >= original) { + if (restart_index_ == 0) { + // No more entries + current_ = restarts_; + restart_index_ = num_restarts_; + return; + } + restart_index_--; + } + + SeekToRestartPoint(restart_index_); + do { + // Loop until end of current entry hits the start of original entry + } while (ParseNextKey() && NextEntryOffset() < original); + } + + void Seek(const Slice& target) override { + // Binary search in restart array to find the last restart point + // with a key < target + uint32_t left = 0; + uint32_t right = num_restarts_ - 1; + int current_key_compare = 0; + + if (Valid()) { + // If we're already scanning, use the current position as a starting + // point. This is beneficial if the key we're seeking to is ahead of the + // current position. + current_key_compare = Compare(key_, target); + if (current_key_compare < 0) { + // key_ is smaller than target + left = restart_index_; + } else if (current_key_compare > 0) { + right = restart_index_; + } else { + // We're seeking to the key we're already at. + return; + } + } + + while (left < right) { + uint32_t mid = (left + right + 1) / 2; + uint32_t region_offset = GetRestartPoint(mid); + uint32_t shared, non_shared, value_length; + const char* key_ptr = + DecodeEntry(data_ + region_offset, data_ + restarts_, &shared, + &non_shared, &value_length); + if (key_ptr == nullptr || (shared != 0)) { + CorruptionError(); + return; + } + Slice mid_key(key_ptr, non_shared); + if (Compare(mid_key, target) < 0) { + // Key at "mid" is smaller than "target". Therefore all + // blocks before "mid" are uninteresting. + left = mid; + } else { + // Key at "mid" is >= "target". Therefore all blocks at or + // after "mid" are uninteresting. + right = mid - 1; + } + } + + // We might be able to use our current position within the restart block. + // This is true if we determined the key we desire is in the current block + // and is after than the current key. + assert(current_key_compare == 0 || Valid()); + bool skip_seek = left == restart_index_ && current_key_compare < 0; + if (!skip_seek) { + SeekToRestartPoint(left); + } + // Linear search (within restart block) for first key >= target + while (true) { + if (!ParseNextKey()) { + return; + } + if (Compare(key_, target) >= 0) { + return; + } + } + } + + void SeekToFirst() override { + SeekToRestartPoint(0); + ParseNextKey(); + } + + void SeekToLast() override { + SeekToRestartPoint(num_restarts_ - 1); + while (ParseNextKey() && NextEntryOffset() < restarts_) { + // Keep skipping + } + } + + private: + void CorruptionError() { + current_ = restarts_; + restart_index_ = num_restarts_; + status_ = Status::Corruption("bad entry in block"); + key_.clear(); + value_.clear(); + } + + bool ParseNextKey() { + current_ = NextEntryOffset(); + const char* p = data_ + current_; + const char* limit = data_ + restarts_; // Restarts come right after data + if (p >= limit) { + // No more entries to return. Mark as invalid. + current_ = restarts_; + restart_index_ = num_restarts_; + return false; + } + + // Decode next entry + uint32_t shared, non_shared, value_length; + p = DecodeEntry(p, limit, &shared, &non_shared, &value_length); + if (p == nullptr || key_.size() < shared) { + CorruptionError(); + return false; + } else { + key_.resize(shared); + key_.append(p, non_shared); + value_ = Slice(p + non_shared, value_length); + while (restart_index_ + 1 < num_restarts_ && + GetRestartPoint(restart_index_ + 1) < current_) { + ++restart_index_; + } + return true; + } + } +}; + +Iterator* Block::NewIterator(const Comparator* comparator) { + if (size_ < sizeof(uint32_t)) { + return NewErrorIterator(Status::Corruption("bad block contents")); + } + const uint32_t num_restarts = NumRestarts(); + if (num_restarts == 0) { + return NewEmptyIterator(); + } else { + return new Iter(comparator, data_, restart_offset_, num_restarts); + } +} + +} // namespace leveldb diff --git a/leveldb/table/block.h b/leveldb/table/block.h new file mode 100644 index 000000000..522410855 --- /dev/null +++ b/leveldb/table/block.h @@ -0,0 +1,44 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_ +#define STORAGE_LEVELDB_TABLE_BLOCK_H_ + +#include +#include + +#include "leveldb/iterator.h" + +namespace leveldb { + +struct BlockContents; +class Comparator; + +class Block { + public: + // Initialize the block with the specified contents. + explicit Block(const BlockContents& contents); + + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + + ~Block(); + + size_t size() const { return size_; } + Iterator* NewIterator(const Comparator* comparator); + + private: + class Iter; + + uint32_t NumRestarts() const; + + const char* data_; + size_t size_; + uint32_t restart_offset_; // Offset in data_ of restart array + bool owned_; // Block owns data_[] +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_TABLE_BLOCK_H_ diff --git a/leveldb/table/block_builder.cc b/leveldb/table/block_builder.cc new file mode 100644 index 000000000..37d400866 --- /dev/null +++ b/leveldb/table/block_builder.cc @@ -0,0 +1,107 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// BlockBuilder generates blocks where keys are prefix-compressed: +// +// When we store a key, we drop the prefix shared with the previous +// string. This helps reduce the space requirement significantly. +// Furthermore, once every K keys, we do not apply the prefix +// compression and store the entire key. We call this a "restart +// point". The tail end of the block stores the offsets of all of the +// restart points, and can be used to do a binary search when looking +// for a particular key. Values are stored as-is (without compression) +// immediately following the corresponding key. +// +// An entry for a particular key-value pair has the form: +// shared_bytes: varint32 +// unshared_bytes: varint32 +// value_length: varint32 +// key_delta: char[unshared_bytes] +// value: char[value_length] +// shared_bytes == 0 for restart points. +// +// The trailer of the block has the form: +// restarts: uint32[num_restarts] +// num_restarts: uint32 +// restarts[i] contains the offset within the block of the ith restart point. + +#include "table/block_builder.h" + +#include +#include + +#include "leveldb/comparator.h" +#include "leveldb/options.h" +#include "util/coding.h" + +namespace leveldb { + +BlockBuilder::BlockBuilder(const Options* options) + : options_(options), restarts_(), counter_(0), finished_(false) { + assert(options->block_restart_interval >= 1); + restarts_.push_back(0); // First restart point is at offset 0 +} + +void BlockBuilder::Reset() { + buffer_.clear(); + restarts_.clear(); + restarts_.push_back(0); // First restart point is at offset 0 + counter_ = 0; + finished_ = false; + last_key_.clear(); +} + +size_t BlockBuilder::CurrentSizeEstimate() const { + return (buffer_.size() + // Raw data buffer + restarts_.size() * sizeof(uint32_t) + // Restart array + sizeof(uint32_t)); // Restart array length +} + +Slice BlockBuilder::Finish() { + // Append restart array + for (size_t i = 0; i < restarts_.size(); i++) { + PutFixed32(&buffer_, restarts_[i]); + } + PutFixed32(&buffer_, restarts_.size()); + finished_ = true; + return Slice(buffer_); +} + +void BlockBuilder::Add(const Slice& key, const Slice& value) { + Slice last_key_piece(last_key_); + assert(!finished_); + assert(counter_ <= options_->block_restart_interval); + assert(buffer_.empty() // No values yet? + || options_->comparator->Compare(key, last_key_piece) > 0); + size_t shared = 0; + if (counter_ < options_->block_restart_interval) { + // See how much sharing to do with previous string + const size_t min_length = std::min(last_key_piece.size(), key.size()); + while ((shared < min_length) && (last_key_piece[shared] == key[shared])) { + shared++; + } + } else { + // Restart compression + restarts_.push_back(buffer_.size()); + counter_ = 0; + } + const size_t non_shared = key.size() - shared; + + // Add "" to buffer_ + PutVarint32(&buffer_, shared); + PutVarint32(&buffer_, non_shared); + PutVarint32(&buffer_, value.size()); + + // Add string delta to buffer_ followed by value + buffer_.append(key.data() + shared, non_shared); + buffer_.append(value.data(), value.size()); + + // Update state + last_key_.resize(shared); + last_key_.append(key.data() + shared, non_shared); + assert(Slice(last_key_) == key); + counter_++; +} + +} // namespace leveldb diff --git a/leveldb/table/block_builder.h b/leveldb/table/block_builder.h new file mode 100644 index 000000000..7a481cd29 --- /dev/null +++ b/leveldb/table/block_builder.h @@ -0,0 +1,54 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ +#define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ + +#include +#include + +#include "leveldb/slice.h" + +namespace leveldb { + +struct Options; + +class BlockBuilder { + public: + explicit BlockBuilder(const Options* options); + + BlockBuilder(const BlockBuilder&) = delete; + BlockBuilder& operator=(const BlockBuilder&) = delete; + + // Reset the contents as if the BlockBuilder was just constructed. + void Reset(); + + // REQUIRES: Finish() has not been called since the last call to Reset(). + // REQUIRES: key is larger than any previously added key + void Add(const Slice& key, const Slice& value); + + // Finish building the block and return a slice that refers to the + // block contents. The returned slice will remain valid for the + // lifetime of this builder or until Reset() is called. + Slice Finish(); + + // Returns an estimate of the current (uncompressed) size of the block + // we are building. + size_t CurrentSizeEstimate() const; + + // Return true iff no entries have been added since the last Reset() + bool empty() const { return buffer_.empty(); } + + private: + const Options* options_; + std::string buffer_; // Destination buffer + std::vector restarts_; // Restart points + int counter_; // Number of entries emitted since restart + bool finished_; // Has Finish() been called? + std::string last_key_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ diff --git a/leveldb/table/filter_block.cc b/leveldb/table/filter_block.cc new file mode 100644 index 000000000..09ec0094b --- /dev/null +++ b/leveldb/table/filter_block.cc @@ -0,0 +1,106 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/filter_block.h" + +#include "leveldb/filter_policy.h" +#include "util/coding.h" + +namespace leveldb { + +// See doc/table_format.md for an explanation of the filter block format. + +// Generate new filter every 2KB of data +static const size_t kFilterBaseLg = 11; +static const size_t kFilterBase = 1 << kFilterBaseLg; + +FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy) + : policy_(policy) {} + +void FilterBlockBuilder::StartBlock(uint64_t block_offset) { + uint64_t filter_index = (block_offset / kFilterBase); + assert(filter_index >= filter_offsets_.size()); + while (filter_index > filter_offsets_.size()) { + GenerateFilter(); + } +} + +void FilterBlockBuilder::AddKey(const Slice& key) { + Slice k = key; + start_.push_back(keys_.size()); + keys_.append(k.data(), k.size()); +} + +Slice FilterBlockBuilder::Finish() { + if (!start_.empty()) { + GenerateFilter(); + } + + // Append array of per-filter offsets + const uint32_t array_offset = result_.size(); + for (size_t i = 0; i < filter_offsets_.size(); i++) { + PutFixed32(&result_, filter_offsets_[i]); + } + + PutFixed32(&result_, array_offset); + result_.push_back(kFilterBaseLg); // Save encoding parameter in result + return Slice(result_); +} + +void FilterBlockBuilder::GenerateFilter() { + const size_t num_keys = start_.size(); + if (num_keys == 0) { + // Fast path if there are no keys for this filter + filter_offsets_.push_back(result_.size()); + return; + } + + // Make list of keys from flattened key structure + start_.push_back(keys_.size()); // Simplify length computation + tmp_keys_.resize(num_keys); + for (size_t i = 0; i < num_keys; i++) { + const char* base = keys_.data() + start_[i]; + size_t length = start_[i + 1] - start_[i]; + tmp_keys_[i] = Slice(base, length); + } + + // Generate filter for current set of keys and append to result_. + filter_offsets_.push_back(result_.size()); + policy_->CreateFilter(&tmp_keys_[0], static_cast(num_keys), &result_); + + tmp_keys_.clear(); + keys_.clear(); + start_.clear(); +} + +FilterBlockReader::FilterBlockReader(const FilterPolicy* policy, + const Slice& contents) + : policy_(policy), data_(nullptr), offset_(nullptr), num_(0), base_lg_(0) { + size_t n = contents.size(); + if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array + base_lg_ = contents[n - 1]; + uint32_t last_word = DecodeFixed32(contents.data() + n - 5); + if (last_word > n - 5) return; + data_ = contents.data(); + offset_ = data_ + last_word; + num_ = (n - 5 - last_word) / 4; +} + +bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) { + uint64_t index = block_offset >> base_lg_; + if (index < num_) { + uint32_t start = DecodeFixed32(offset_ + index * 4); + uint32_t limit = DecodeFixed32(offset_ + index * 4 + 4); + if (start <= limit && limit <= static_cast(offset_ - data_)) { + Slice filter = Slice(data_ + start, limit - start); + return policy_->KeyMayMatch(key, filter); + } else if (start == limit) { + // Empty filters do not match any keys + return false; + } + } + return true; // Errors are treated as potential matches +} + +} // namespace leveldb diff --git a/leveldb/table/filter_block.h b/leveldb/table/filter_block.h new file mode 100644 index 000000000..25ab75bf9 --- /dev/null +++ b/leveldb/table/filter_block.h @@ -0,0 +1,68 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A filter block is stored near the end of a Table file. It contains +// filters (e.g., bloom filters) for all data blocks in the table combined +// into a single filter block. + +#ifndef STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ +#define STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ + +#include +#include +#include +#include + +#include "leveldb/slice.h" +#include "util/hash.h" + +namespace leveldb { + +class FilterPolicy; + +// A FilterBlockBuilder is used to construct all of the filters for a +// particular Table. It generates a single string which is stored as +// a special block in the Table. +// +// The sequence of calls to FilterBlockBuilder must match the regexp: +// (StartBlock AddKey*)* Finish +class FilterBlockBuilder { + public: + explicit FilterBlockBuilder(const FilterPolicy*); + + FilterBlockBuilder(const FilterBlockBuilder&) = delete; + FilterBlockBuilder& operator=(const FilterBlockBuilder&) = delete; + + void StartBlock(uint64_t block_offset); + void AddKey(const Slice& key); + Slice Finish(); + + private: + void GenerateFilter(); + + const FilterPolicy* policy_; + std::string keys_; // Flattened key contents + std::vector start_; // Starting index in keys_ of each key + std::string result_; // Filter data computed so far + std::vector tmp_keys_; // policy_->CreateFilter() argument + std::vector filter_offsets_; +}; + +class FilterBlockReader { + public: + // REQUIRES: "contents" and *policy must stay live while *this is live. + FilterBlockReader(const FilterPolicy* policy, const Slice& contents); + bool KeyMayMatch(uint64_t block_offset, const Slice& key); + + private: + const FilterPolicy* policy_; + const char* data_; // Pointer to filter data (at block-start) + const char* offset_; // Pointer to beginning of offset array (at block-end) + size_t num_; // Number of entries in offset array + size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file) +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ diff --git a/leveldb/table/filter_block_test.cc b/leveldb/table/filter_block_test.cc new file mode 100644 index 000000000..3ee41cf58 --- /dev/null +++ b/leveldb/table/filter_block_test.cc @@ -0,0 +1,122 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/filter_block.h" + +#include "gtest/gtest.h" +#include "leveldb/filter_policy.h" +#include "util/coding.h" +#include "util/hash.h" +#include "util/logging.h" +#include "util/testutil.h" + +namespace leveldb { + +// For testing: emit an array with one hash value per key +class TestHashFilter : public FilterPolicy { + public: + const char* Name() const override { return "TestHashFilter"; } + + void CreateFilter(const Slice* keys, int n, std::string* dst) const override { + for (int i = 0; i < n; i++) { + uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); + PutFixed32(dst, h); + } + } + + bool KeyMayMatch(const Slice& key, const Slice& filter) const override { + uint32_t h = Hash(key.data(), key.size(), 1); + for (size_t i = 0; i + 4 <= filter.size(); i += 4) { + if (h == DecodeFixed32(filter.data() + i)) { + return true; + } + } + return false; + } +}; + +class FilterBlockTest : public testing::Test { + public: + TestHashFilter policy_; +}; + +TEST_F(FilterBlockTest, EmptyBuilder) { + FilterBlockBuilder builder(&policy_); + Slice block = builder.Finish(); + ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); + FilterBlockReader reader(&policy_, block); + ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); + ASSERT_TRUE(reader.KeyMayMatch(100000, "foo")); +} + +TEST_F(FilterBlockTest, SingleChunk) { + FilterBlockBuilder builder(&policy_); + builder.StartBlock(100); + builder.AddKey("foo"); + builder.AddKey("bar"); + builder.AddKey("box"); + builder.StartBlock(200); + builder.AddKey("box"); + builder.StartBlock(300); + builder.AddKey("hello"); + Slice block = builder.Finish(); + FilterBlockReader reader(&policy_, block); + ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); + ASSERT_TRUE(reader.KeyMayMatch(100, "bar")); + ASSERT_TRUE(reader.KeyMayMatch(100, "box")); + ASSERT_TRUE(reader.KeyMayMatch(100, "hello")); + ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); + ASSERT_TRUE(!reader.KeyMayMatch(100, "missing")); + ASSERT_TRUE(!reader.KeyMayMatch(100, "other")); +} + +TEST_F(FilterBlockTest, MultiChunk) { + FilterBlockBuilder builder(&policy_); + + // First filter + builder.StartBlock(0); + builder.AddKey("foo"); + builder.StartBlock(2000); + builder.AddKey("bar"); + + // Second filter + builder.StartBlock(3100); + builder.AddKey("box"); + + // Third filter is empty + + // Last filter + builder.StartBlock(9000); + builder.AddKey("box"); + builder.AddKey("hello"); + + Slice block = builder.Finish(); + FilterBlockReader reader(&policy_, block); + + // Check first filter + ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); + ASSERT_TRUE(reader.KeyMayMatch(2000, "bar")); + ASSERT_TRUE(!reader.KeyMayMatch(0, "box")); + ASSERT_TRUE(!reader.KeyMayMatch(0, "hello")); + + // Check second filter + ASSERT_TRUE(reader.KeyMayMatch(3100, "box")); + ASSERT_TRUE(!reader.KeyMayMatch(3100, "foo")); + ASSERT_TRUE(!reader.KeyMayMatch(3100, "bar")); + ASSERT_TRUE(!reader.KeyMayMatch(3100, "hello")); + + // Check third filter (empty) + ASSERT_TRUE(!reader.KeyMayMatch(4100, "foo")); + ASSERT_TRUE(!reader.KeyMayMatch(4100, "bar")); + ASSERT_TRUE(!reader.KeyMayMatch(4100, "box")); + ASSERT_TRUE(!reader.KeyMayMatch(4100, "hello")); + + // Check last filter + ASSERT_TRUE(reader.KeyMayMatch(9000, "box")); + ASSERT_TRUE(reader.KeyMayMatch(9000, "hello")); + ASSERT_TRUE(!reader.KeyMayMatch(9000, "foo")); + ASSERT_TRUE(!reader.KeyMayMatch(9000, "bar")); +} + +} // namespace leveldb diff --git a/leveldb/table/format.cc b/leveldb/table/format.cc new file mode 100644 index 000000000..7647372a8 --- /dev/null +++ b/leveldb/table/format.cc @@ -0,0 +1,160 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/format.h" + +#include "leveldb/env.h" +#include "leveldb/options.h" +#include "port/port.h" +#include "table/block.h" +#include "util/coding.h" +#include "util/crc32c.h" + +namespace leveldb { + +void BlockHandle::EncodeTo(std::string* dst) const { + // Sanity check that all fields have been set + assert(offset_ != ~static_cast(0)); + assert(size_ != ~static_cast(0)); + PutVarint64(dst, offset_); + PutVarint64(dst, size_); +} + +Status BlockHandle::DecodeFrom(Slice* input) { + if (GetVarint64(input, &offset_) && GetVarint64(input, &size_)) { + return Status::OK(); + } else { + return Status::Corruption("bad block handle"); + } +} + +void Footer::EncodeTo(std::string* dst) const { + const size_t original_size = dst->size(); + metaindex_handle_.EncodeTo(dst); + index_handle_.EncodeTo(dst); + dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding + PutFixed32(dst, static_cast(kTableMagicNumber & 0xffffffffu)); + PutFixed32(dst, static_cast(kTableMagicNumber >> 32)); + assert(dst->size() == original_size + kEncodedLength); + (void)original_size; // Disable unused variable warning. +} + +Status Footer::DecodeFrom(Slice* input) { + const char* magic_ptr = input->data() + kEncodedLength - 8; + const uint32_t magic_lo = DecodeFixed32(magic_ptr); + const uint32_t magic_hi = DecodeFixed32(magic_ptr + 4); + const uint64_t magic = ((static_cast(magic_hi) << 32) | + (static_cast(magic_lo))); + if (magic != kTableMagicNumber) { + return Status::Corruption("not an sstable (bad magic number)"); + } + + Status result = metaindex_handle_.DecodeFrom(input); + if (result.ok()) { + result = index_handle_.DecodeFrom(input); + } + if (result.ok()) { + // We skip over any leftover data (just padding for now) in "input" + const char* end = magic_ptr + 8; + *input = Slice(end, input->data() + input->size() - end); + } + return result; +} + +Status ReadBlock(RandomAccessFile* file, const ReadOptions& options, + const BlockHandle& handle, BlockContents* result) { + result->data = Slice(); + result->cachable = false; + result->heap_allocated = false; + + // Read the block contents as well as the type/crc footer. + // See table_builder.cc for the code that built this structure. + size_t n = static_cast(handle.size()); + char* buf = new char[n + kBlockTrailerSize]; + Slice contents; + Status s = file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf); + if (!s.ok()) { + delete[] buf; + return s; + } + if (contents.size() != n + kBlockTrailerSize) { + delete[] buf; + return Status::Corruption("truncated block read"); + } + + // Check the crc of the type and the block contents + const char* data = contents.data(); // Pointer to where Read put the data + if (options.verify_checksums) { + const uint32_t crc = crc32c::Unmask(DecodeFixed32(data + n + 1)); + const uint32_t actual = crc32c::Value(data, n + 1); + if (actual != crc) { + delete[] buf; + s = Status::Corruption("block checksum mismatch"); + return s; + } + } + + switch (data[n]) { + case kNoCompression: + if (data != buf) { + // File implementation gave us pointer to some other data. + // Use it directly under the assumption that it will be live + // while the file is open. + delete[] buf; + result->data = Slice(data, n); + result->heap_allocated = false; + result->cachable = false; // Do not double-cache + } else { + result->data = Slice(buf, n); + result->heap_allocated = true; + result->cachable = true; + } + + // Ok + break; + case kSnappyCompression: { + size_t ulength = 0; + if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) { + delete[] buf; + return Status::Corruption("corrupted snappy compressed block length"); + } + char* ubuf = new char[ulength]; + if (!port::Snappy_Uncompress(data, n, ubuf)) { + delete[] buf; + delete[] ubuf; + return Status::Corruption("corrupted snappy compressed block contents"); + } + delete[] buf; + result->data = Slice(ubuf, ulength); + result->heap_allocated = true; + result->cachable = true; + break; + } + case kZstdCompression: { + size_t ulength = 0; + if (!port::Zstd_GetUncompressedLength(data, n, &ulength)) { + delete[] buf; + return Status::Corruption("corrupted zstd compressed block length"); + } + char* ubuf = new char[ulength]; + if (!port::Zstd_Uncompress(data, n, ubuf)) { + delete[] buf; + delete[] ubuf; + return Status::Corruption("corrupted zstd compressed block contents"); + } + delete[] buf; + result->data = Slice(ubuf, ulength); + result->heap_allocated = true; + result->cachable = true; + break; + } + default: + delete[] buf; + return Status::Corruption("bad block type"); + } + + return Status::OK(); +} + +} // namespace leveldb diff --git a/leveldb/table/format.h b/leveldb/table/format.h new file mode 100644 index 000000000..f6ea30433 --- /dev/null +++ b/leveldb/table/format.h @@ -0,0 +1,99 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_ +#define STORAGE_LEVELDB_TABLE_FORMAT_H_ + +#include +#include + +#include "leveldb/slice.h" +#include "leveldb/status.h" +#include "leveldb/table_builder.h" + +namespace leveldb { + +class Block; +class RandomAccessFile; +struct ReadOptions; + +// BlockHandle is a pointer to the extent of a file that stores a data +// block or a meta block. +class BlockHandle { + public: + // Maximum encoding length of a BlockHandle + enum { kMaxEncodedLength = 10 + 10 }; + + BlockHandle(); + + // The offset of the block in the file. + uint64_t offset() const { return offset_; } + void set_offset(uint64_t offset) { offset_ = offset; } + + // The size of the stored block + uint64_t size() const { return size_; } + void set_size(uint64_t size) { size_ = size; } + + void EncodeTo(std::string* dst) const; + Status DecodeFrom(Slice* input); + + private: + uint64_t offset_; + uint64_t size_; +}; + +// Footer encapsulates the fixed information stored at the tail +// end of every table file. +class Footer { + public: + // Encoded length of a Footer. Note that the serialization of a + // Footer will always occupy exactly this many bytes. It consists + // of two block handles and a magic number. + enum { kEncodedLength = 2 * BlockHandle::kMaxEncodedLength + 8 }; + + Footer() = default; + + // The block handle for the metaindex block of the table + const BlockHandle& metaindex_handle() const { return metaindex_handle_; } + void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; } + + // The block handle for the index block of the table + const BlockHandle& index_handle() const { return index_handle_; } + void set_index_handle(const BlockHandle& h) { index_handle_ = h; } + + void EncodeTo(std::string* dst) const; + Status DecodeFrom(Slice* input); + + private: + BlockHandle metaindex_handle_; + BlockHandle index_handle_; +}; + +// kTableMagicNumber was picked by running +// echo http://code.google.com/p/leveldb/ | sha1sum +// and taking the leading 64 bits. +static const uint64_t kTableMagicNumber = 0xdb4775248b80fb57ull; + +// 1-byte type + 32-bit crc +static const size_t kBlockTrailerSize = 5; + +struct BlockContents { + Slice data; // Actual contents of data + bool cachable; // True iff data can be cached + bool heap_allocated; // True iff caller should delete[] data.data() +}; + +// Read the block identified by "handle" from "file". On failure +// return non-OK. On success fill *result and return OK. +Status ReadBlock(RandomAccessFile* file, const ReadOptions& options, + const BlockHandle& handle, BlockContents* result); + +// Implementation details follow. Clients should ignore, + +inline BlockHandle::BlockHandle() + : offset_(~static_cast(0)), size_(~static_cast(0)) {} + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_TABLE_FORMAT_H_ diff --git a/leveldb/table/iterator.cc b/leveldb/table/iterator.cc new file mode 100644 index 000000000..dfef083d4 --- /dev/null +++ b/leveldb/table/iterator.cc @@ -0,0 +1,76 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/iterator.h" + +namespace leveldb { + +Iterator::Iterator() { + cleanup_head_.function = nullptr; + cleanup_head_.next = nullptr; +} + +Iterator::~Iterator() { + if (!cleanup_head_.IsEmpty()) { + cleanup_head_.Run(); + for (CleanupNode* node = cleanup_head_.next; node != nullptr;) { + node->Run(); + CleanupNode* next_node = node->next; + delete node; + node = next_node; + } + } +} + +void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) { + assert(func != nullptr); + CleanupNode* node; + if (cleanup_head_.IsEmpty()) { + node = &cleanup_head_; + } else { + node = new CleanupNode(); + node->next = cleanup_head_.next; + cleanup_head_.next = node; + } + node->function = func; + node->arg1 = arg1; + node->arg2 = arg2; +} + +namespace { + +class EmptyIterator : public Iterator { + public: + EmptyIterator(const Status& s) : status_(s) {} + ~EmptyIterator() override = default; + + bool Valid() const override { return false; } + void Seek(const Slice& target) override {} + void SeekToFirst() override {} + void SeekToLast() override {} + void Next() override { assert(false); } + void Prev() override { assert(false); } + Slice key() const override { + assert(false); + return Slice(); + } + Slice value() const override { + assert(false); + return Slice(); + } + Status status() const override { return status_; } + + private: + Status status_; +}; + +} // anonymous namespace + +Iterator* NewEmptyIterator() { return new EmptyIterator(Status::OK()); } + +Iterator* NewErrorIterator(const Status& status) { + return new EmptyIterator(status); +} + +} // namespace leveldb diff --git a/leveldb/table/iterator_wrapper.h b/leveldb/table/iterator_wrapper.h new file mode 100644 index 000000000..c23057252 --- /dev/null +++ b/leveldb/table/iterator_wrapper.h @@ -0,0 +1,92 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ +#define STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ + +#include "leveldb/iterator.h" +#include "leveldb/slice.h" + +namespace leveldb { + +// A internal wrapper class with an interface similar to Iterator that +// caches the valid() and key() results for an underlying iterator. +// This can help avoid virtual function calls and also gives better +// cache locality. +class IteratorWrapper { + public: + IteratorWrapper() : iter_(nullptr), valid_(false) {} + explicit IteratorWrapper(Iterator* iter) : iter_(nullptr) { Set(iter); } + ~IteratorWrapper() { delete iter_; } + Iterator* iter() const { return iter_; } + + // Takes ownership of "iter" and will delete it when destroyed, or + // when Set() is invoked again. + void Set(Iterator* iter) { + delete iter_; + iter_ = iter; + if (iter_ == nullptr) { + valid_ = false; + } else { + Update(); + } + } + + // Iterator interface methods + bool Valid() const { return valid_; } + Slice key() const { + assert(Valid()); + return key_; + } + Slice value() const { + assert(Valid()); + return iter_->value(); + } + // Methods below require iter() != nullptr + Status status() const { + assert(iter_); + return iter_->status(); + } + void Next() { + assert(iter_); + iter_->Next(); + Update(); + } + void Prev() { + assert(iter_); + iter_->Prev(); + Update(); + } + void Seek(const Slice& k) { + assert(iter_); + iter_->Seek(k); + Update(); + } + void SeekToFirst() { + assert(iter_); + iter_->SeekToFirst(); + Update(); + } + void SeekToLast() { + assert(iter_); + iter_->SeekToLast(); + Update(); + } + + private: + void Update() { + valid_ = iter_->Valid(); + if (valid_) { + key_ = iter_->key(); + } + } + + Iterator* iter_; + bool valid_; + Slice key_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ diff --git a/leveldb/table/merger.cc b/leveldb/table/merger.cc new file mode 100644 index 000000000..ab25a20e2 --- /dev/null +++ b/leveldb/table/merger.cc @@ -0,0 +1,189 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/merger.h" + +#include "leveldb/comparator.h" +#include "leveldb/iterator.h" +#include "table/iterator_wrapper.h" + +namespace leveldb { + +namespace { +class MergingIterator : public Iterator { + public: + MergingIterator(const Comparator* comparator, Iterator** children, int n) + : comparator_(comparator), + children_(new IteratorWrapper[n]), + n_(n), + current_(nullptr), + direction_(kForward) { + for (int i = 0; i < n; i++) { + children_[i].Set(children[i]); + } + } + + ~MergingIterator() override { delete[] children_; } + + bool Valid() const override { return (current_ != nullptr); } + + void SeekToFirst() override { + for (int i = 0; i < n_; i++) { + children_[i].SeekToFirst(); + } + FindSmallest(); + direction_ = kForward; + } + + void SeekToLast() override { + for (int i = 0; i < n_; i++) { + children_[i].SeekToLast(); + } + FindLargest(); + direction_ = kReverse; + } + + void Seek(const Slice& target) override { + //blank3.2 + //TODO: complete the seek function, + //Tips: `SeekToFirst` can be a reference. + } + + void Next() override { + assert(Valid()); + + // Ensure that all children are positioned after key(). + // If we are moving in the forward direction, it is already + // true for all of the non-current_ children since current_ is + // the smallest child and key() == current_->key(). Otherwise, + // we explicitly position the non-current_ children. + if (direction_ != kForward) { + for (int i = 0; i < n_; i++) { + IteratorWrapper* child = &children_[i]; + if (child != current_) { + child->Seek(key()); + if (child->Valid() && + comparator_->Compare(key(), child->key()) == 0) { + child->Next(); + } + } + } + direction_ = kForward; + } + + current_->Next(); + FindSmallest(); + } + + void Prev() override { + assert(Valid()); + + // Ensure that all children are positioned before key(). + // If we are moving in the reverse direction, it is already + // true for all of the non-current_ children since current_ is + // the largest child and key() == current_->key(). Otherwise, + // we explicitly position the non-current_ children. + if (direction_ != kReverse) { + for (int i = 0; i < n_; i++) { + IteratorWrapper* child = &children_[i]; + if (child != current_) { + child->Seek(key()); + if (child->Valid()) { + // Child is at first entry >= key(). Step back one to be < key() + child->Prev(); + } else { + // Child has no entries >= key(). Position at last entry. + child->SeekToLast(); + } + } + } + direction_ = kReverse; + } + + current_->Prev(); + FindLargest(); + } + + Slice key() const override { + assert(Valid()); + return current_->key(); + } + + Slice value() const override { + assert(Valid()); + return current_->value(); + } + + Status status() const override { + Status status; + for (int i = 0; i < n_; i++) { + status = children_[i].status(); + if (!status.ok()) { + break; + } + } + return status; + } + + private: + // Which direction is the iterator moving? + enum Direction { kForward, kReverse }; + + void FindSmallest(); + void FindLargest(); + + // We might want to use a heap in case there are lots of children. + // For now we use a simple array since we expect a very small number + // of children in leveldb. + const Comparator* comparator_; + IteratorWrapper* children_; + int n_; + IteratorWrapper* current_; + Direction direction_; +}; + +void MergingIterator::FindSmallest() { + IteratorWrapper* smallest = nullptr; + for (int i = 0; i < n_; i++) { + IteratorWrapper* child = &children_[i]; + if (child->Valid()) { + if (smallest == nullptr) { + smallest = child; + } else if (comparator_->Compare(child->key(), smallest->key()) < 0) { + smallest = child; + } + } + } + current_ = smallest; +} + +void MergingIterator::FindLargest() { + IteratorWrapper* largest = nullptr; + for (int i = n_ - 1; i >= 0; i--) { + IteratorWrapper* child = &children_[i]; + if (child->Valid()) { + if (largest == nullptr) { + largest = child; + } else if (comparator_->Compare(child->key(), largest->key()) > 0) { + largest = child; + } + } + } + current_ = largest; +} +} // namespace + +Iterator* NewMergingIterator(const Comparator* comparator, Iterator** children, + int n) { + assert(n >= 0); + if (n == 0) { + return NewEmptyIterator(); + } else if (n == 1) { + return children[0]; + } else { + return new MergingIterator(comparator, children, n); + } +} + +} // namespace leveldb diff --git a/leveldb/table/merger.h b/leveldb/table/merger.h new file mode 100644 index 000000000..41cedc525 --- /dev/null +++ b/leveldb/table/merger.h @@ -0,0 +1,26 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_TABLE_MERGER_H_ +#define STORAGE_LEVELDB_TABLE_MERGER_H_ + +namespace leveldb { + +class Comparator; +class Iterator; + +// Return an iterator that provided the union of the data in +// children[0,n-1]. Takes ownership of the child iterators and +// will delete them when the result iterator is deleted. +// +// The result does no duplicate suppression. I.e., if a particular +// key is present in K child iterators, it will be yielded K times. +// +// REQUIRES: n >= 0 +Iterator* NewMergingIterator(const Comparator* comparator, Iterator** children, + int n); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_TABLE_MERGER_H_ diff --git a/leveldb/table/table.cc b/leveldb/table/table.cc new file mode 100644 index 000000000..29e835f71 --- /dev/null +++ b/leveldb/table/table.cc @@ -0,0 +1,271 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/table.h" + +#include "leveldb/cache.h" +#include "leveldb/comparator.h" +#include "leveldb/env.h" +#include "leveldb/filter_policy.h" +#include "leveldb/options.h" +#include "table/block.h" +#include "table/filter_block.h" +#include "table/format.h" +#include "table/two_level_iterator.h" +#include "util/coding.h" + +namespace leveldb { + +struct Table::Rep { + ~Rep() { + delete filter; + delete[] filter_data; + delete index_block; + } + + Options options; + Status status; + RandomAccessFile* file; + uint64_t cache_id; + FilterBlockReader* filter; + const char* filter_data; + + BlockHandle metaindex_handle; // Handle to metaindex_block: saved from footer + Block* index_block; +}; + +Status Table::Open(const Options& options, RandomAccessFile* file, + uint64_t size, Table** table) { + *table = nullptr; + if (size < Footer::kEncodedLength) { + return Status::Corruption("file is too short to be an sstable"); + } + + char footer_space[Footer::kEncodedLength]; + Slice footer_input; + Status s = file->Read(size - Footer::kEncodedLength, Footer::kEncodedLength, + &footer_input, footer_space); + if (!s.ok()) return s; + + Footer footer; + s = footer.DecodeFrom(&footer_input); + if (!s.ok()) return s; + + // Read the index block + BlockContents index_block_contents; + ReadOptions opt; + if (options.paranoid_checks) { + opt.verify_checksums = true; + } + s = ReadBlock(file, opt, footer.index_handle(), &index_block_contents); + + if (s.ok()) { + // We've successfully read the footer and the index block: we're + // ready to serve requests. + Block* index_block = new Block(index_block_contents); + Rep* rep = new Table::Rep; + rep->options = options; + rep->file = file; + rep->metaindex_handle = footer.metaindex_handle(); + rep->index_block = index_block; + rep->cache_id = (options.block_cache ? options.block_cache->NewId() : 0); + rep->filter_data = nullptr; + rep->filter = nullptr; + *table = new Table(rep); + (*table)->ReadMeta(footer); + } + + return s; +} + +void Table::ReadMeta(const Footer& footer) { + if (rep_->options.filter_policy == nullptr) { + return; // Do not need any metadata + } + + // TODO(sanjay): Skip this if footer.metaindex_handle() size indicates + // it is an empty block. + ReadOptions opt; + if (rep_->options.paranoid_checks) { + opt.verify_checksums = true; + } + BlockContents contents; + if (!ReadBlock(rep_->file, opt, footer.metaindex_handle(), &contents).ok()) { + // Do not propagate errors since meta info is not needed for operation + return; + } + Block* meta = new Block(contents); + + Iterator* iter = meta->NewIterator(BytewiseComparator()); + std::string key = "filter."; + key.append(rep_->options.filter_policy->Name()); + iter->Seek(key); + if (iter->Valid() && iter->key() == Slice(key)) { + ReadFilter(iter->value()); + } + delete iter; + delete meta; +} + +void Table::ReadFilter(const Slice& filter_handle_value) { + Slice v = filter_handle_value; + BlockHandle filter_handle; + if (!filter_handle.DecodeFrom(&v).ok()) { + return; + } + + // We might want to unify with ReadBlock() if we start + // requiring checksum verification in Table::Open. + ReadOptions opt; + if (rep_->options.paranoid_checks) { + opt.verify_checksums = true; + } + BlockContents block; + if (!ReadBlock(rep_->file, opt, filter_handle, &block).ok()) { + return; + } + if (block.heap_allocated) { + rep_->filter_data = block.data.data(); // Will need to delete later + } + rep_->filter = new FilterBlockReader(rep_->options.filter_policy, block.data); +} + +Table::~Table() { delete rep_; } + +static void DeleteBlock(void* arg, void* ignored) { + delete reinterpret_cast(arg); +} + +static void DeleteCachedBlock(const Slice& key, void* value) { + Block* block = reinterpret_cast(value); + delete block; +} + +static void ReleaseBlock(void* arg, void* h) { + Cache* cache = reinterpret_cast(arg); + Cache::Handle* handle = reinterpret_cast(h); + cache->Release(handle); +} + +// Convert an index iterator value (i.e., an encoded BlockHandle) +// into an iterator over the contents of the corresponding block. +Iterator* Table::BlockReader(void* arg, const ReadOptions& options, + const Slice& index_value) { + Table* table = reinterpret_cast(arg); + Cache* block_cache = table->rep_->options.block_cache; + Block* block = nullptr; + Cache::Handle* cache_handle = nullptr; + + BlockHandle handle; + Slice input = index_value; + Status s = handle.DecodeFrom(&input); + // We intentionally allow extra stuff in index_value so that we + // can add more features in the future. + + if (s.ok()) { + BlockContents contents; + if (block_cache != nullptr) { + char cache_key_buffer[16]; + EncodeFixed64(cache_key_buffer, table->rep_->cache_id); + EncodeFixed64(cache_key_buffer + 8, handle.offset()); + Slice key(cache_key_buffer, sizeof(cache_key_buffer)); + cache_handle = block_cache->Lookup(key); + if (cache_handle != nullptr) { + block = reinterpret_cast(block_cache->Value(cache_handle)); + } else { + s = ReadBlock(table->rep_->file, options, handle, &contents); + if (s.ok()) { + block = new Block(contents); + if (contents.cachable && options.fill_cache) { + cache_handle = block_cache->Insert(key, block, block->size(), + &DeleteCachedBlock); + } + } + } + } else { + s = ReadBlock(table->rep_->file, options, handle, &contents); + if (s.ok()) { + block = new Block(contents); + } + } + } + + Iterator* iter; + if (block != nullptr) { + iter = block->NewIterator(table->rep_->options.comparator); + if (cache_handle == nullptr) { + iter->RegisterCleanup(&DeleteBlock, block, nullptr); + } else { + iter->RegisterCleanup(&ReleaseBlock, block_cache, cache_handle); + } + } else { + iter = NewErrorIterator(s); + } + return iter; +} + +Iterator* Table::NewIterator(const ReadOptions& options) const { + return NewTwoLevelIterator( + rep_->index_block->NewIterator(rep_->options.comparator), + &Table::BlockReader, const_cast(this), options); +} + +Status Table::InternalGet(const ReadOptions& options, const Slice& k, void* arg, + void (*handle_result)(void*, const Slice&, + const Slice&)) { + Status s; + Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator); + iiter->Seek(k); + if (iiter->Valid()) { + Slice handle_value = iiter->value(); + FilterBlockReader* filter = rep_->filter; + BlockHandle handle; + if (filter != nullptr && handle.DecodeFrom(&handle_value).ok() && + !filter->KeyMayMatch(handle.offset(), k)) { + // Not found + } else { + Iterator* block_iter = BlockReader(this, options, iiter->value()); + block_iter->Seek(k); + if (block_iter->Valid()) { + (*handle_result)(arg, block_iter->key(), block_iter->value()); + } + s = block_iter->status(); + delete block_iter; + } + } + if (s.ok()) { + s = iiter->status(); + } + delete iiter; + return s; +} + +uint64_t Table::ApproximateOffsetOf(const Slice& key) const { + Iterator* index_iter = + rep_->index_block->NewIterator(rep_->options.comparator); + index_iter->Seek(key); + uint64_t result; + if (index_iter->Valid()) { + BlockHandle handle; + Slice input = index_iter->value(); + Status s = handle.DecodeFrom(&input); + if (s.ok()) { + result = handle.offset(); + } else { + // Strange: we can't decode the block handle in the index block. + // We'll just return the offset of the metaindex block, which is + // close to the whole file size for this case. + result = rep_->metaindex_handle.offset(); + } + } else { + // key is past the last key in the file. Approximate the offset + // by returning the offset of the metaindex block (which is + // right near the end of the file). + result = rep_->metaindex_handle.offset(); + } + delete index_iter; + return result; +} + +} // namespace leveldb diff --git a/leveldb/table/table_builder.cc b/leveldb/table/table_builder.cc new file mode 100644 index 000000000..ba3df9e60 --- /dev/null +++ b/leveldb/table/table_builder.cc @@ -0,0 +1,279 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/table_builder.h" + +#include + +#include "leveldb/comparator.h" +#include "leveldb/env.h" +#include "leveldb/filter_policy.h" +#include "leveldb/options.h" +#include "table/block_builder.h" +#include "table/filter_block.h" +#include "table/format.h" +#include "util/coding.h" +#include "util/crc32c.h" + +namespace leveldb { + +struct TableBuilder::Rep { + Rep(const Options& opt, WritableFile* f) + : options(opt), + index_block_options(opt), + file(f), + offset(0), + data_block(&options), + index_block(&index_block_options), + num_entries(0), + closed(false), + filter_block(opt.filter_policy == nullptr + ? nullptr + : new FilterBlockBuilder(opt.filter_policy)), + pending_index_entry(false) { + index_block_options.block_restart_interval = 1; + } + + Options options; + Options index_block_options; + WritableFile* file; + uint64_t offset; + Status status; + BlockBuilder data_block; + BlockBuilder index_block; + std::string last_key; + int64_t num_entries; + bool closed; // Either Finish() or Abandon() has been called. + FilterBlockBuilder* filter_block; + + // We do not emit the index entry for a block until we have seen the + // first key for the next data block. This allows us to use shorter + // keys in the index block. For example, consider a block boundary + // between the keys "the quick brown fox" and "the who". We can use + // "the r" as the key for the index block entry since it is >= all + // entries in the first block and < all entries in subsequent + // blocks. + // + // Invariant: r->pending_index_entry is true only if data_block is empty. + bool pending_index_entry; + BlockHandle pending_handle; // Handle to add to index block + + std::string compressed_output; +}; + +TableBuilder::TableBuilder(const Options& options, WritableFile* file) + : rep_(new Rep(options, file)) { + if (rep_->filter_block != nullptr) { + rep_->filter_block->StartBlock(0); + } +} + +TableBuilder::~TableBuilder() { + assert(rep_->closed); // Catch errors where caller forgot to call Finish() + delete rep_->filter_block; + delete rep_; +} + +Status TableBuilder::ChangeOptions(const Options& options) { + // Note: if more fields are added to Options, update + // this function to catch changes that should not be allowed to + // change in the middle of building a Table. + if (options.comparator != rep_->options.comparator) { + return Status::InvalidArgument("changing comparator while building table"); + } + + // Note that any live BlockBuilders point to rep_->options and therefore + // will automatically pick up the updated options. + rep_->options = options; + rep_->index_block_options = options; + rep_->index_block_options.block_restart_interval = 1; + return Status::OK(); +} + +void TableBuilder::Add(const Slice& key, const Slice& value) { + Rep* r = rep_; + assert(!r->closed); + if (!ok()) return; + if (r->num_entries > 0) { + assert(r->options.comparator->Compare(key, Slice(r->last_key)) > 0); + } + + if (r->pending_index_entry) { + assert(r->data_block.empty()); + r->options.comparator->FindShortestSeparator(&r->last_key, key); + std::string handle_encoding; + r->pending_handle.EncodeTo(&handle_encoding); + r->index_block.Add(r->last_key, Slice(handle_encoding)); + r->pending_index_entry = false; + } + + if (r->filter_block != nullptr) { + r->filter_block->AddKey(key); + } + + r->last_key.assign(key.data(), key.size()); + r->num_entries++; + r->data_block.Add(key, value); + + const size_t estimated_block_size = r->data_block.CurrentSizeEstimate(); + if (estimated_block_size >= r->options.block_size) { + Flush(); + } +} + +void TableBuilder::Flush() { + Rep* r = rep_; + assert(!r->closed); + if (!ok()) return; + if (r->data_block.empty()) return; + assert(!r->pending_index_entry); + WriteBlock(&r->data_block, &r->pending_handle); + if (ok()) { + r->pending_index_entry = true; + r->status = r->file->Flush(); + } + if (r->filter_block != nullptr) { + r->filter_block->StartBlock(r->offset); + } +} + +void TableBuilder::WriteBlock(BlockBuilder* block, BlockHandle* handle) { + // File format contains a sequence of blocks where each block has: + // block_data: uint8[n] + // type: uint8 + // crc: uint32 + assert(ok()); + Rep* r = rep_; + Slice raw = block->Finish(); + + Slice block_contents; + CompressionType type = r->options.compression; + // TODO(postrelease): Support more compression options: zlib? + switch (type) { + case kNoCompression: + block_contents = raw; + break; + + case kSnappyCompression: { + std::string* compressed = &r->compressed_output; + if (port::Snappy_Compress(raw.data(), raw.size(), compressed) && + compressed->size() < raw.size() - (raw.size() / 8u)) { + block_contents = *compressed; + } else { + // Snappy not supported, or compressed less than 12.5%, so just + // store uncompressed form + block_contents = raw; + type = kNoCompression; + } + break; + } + + case kZstdCompression: { + std::string* compressed = &r->compressed_output; + if (port::Zstd_Compress(raw.data(), raw.size(), compressed) && + compressed->size() < raw.size() - (raw.size() / 8u)) { + block_contents = *compressed; + } else { + // Zstd not supported, or compressed less than 12.5%, so just + // store uncompressed form + block_contents = raw; + type = kNoCompression; + } + break; + } + } + WriteRawBlock(block_contents, type, handle); + r->compressed_output.clear(); + block->Reset(); +} + +void TableBuilder::WriteRawBlock(const Slice& block_contents, + CompressionType type, BlockHandle* handle) { + Rep* r = rep_; + handle->set_offset(r->offset); + handle->set_size(block_contents.size()); + r->status = r->file->Append(block_contents); + if (r->status.ok()) { + char trailer[kBlockTrailerSize]; + trailer[0] = type; + uint32_t crc = crc32c::Value(block_contents.data(), block_contents.size()); + crc = crc32c::Extend(crc, trailer, 1); // Extend crc to cover block type + EncodeFixed32(trailer + 1, crc32c::Mask(crc)); + r->status = r->file->Append(Slice(trailer, kBlockTrailerSize)); + if (r->status.ok()) { + r->offset += block_contents.size() + kBlockTrailerSize; + } + } +} + +Status TableBuilder::status() const { return rep_->status; } + +Status TableBuilder::Finish() { + Rep* r = rep_; + Flush(); + assert(!r->closed); + r->closed = true; + + BlockHandle filter_block_handle, metaindex_block_handle, index_block_handle; + + // Write filter block + if (ok() && r->filter_block != nullptr) { + WriteRawBlock(r->filter_block->Finish(), kNoCompression, + &filter_block_handle); + } + + // Write metaindex block + if (ok()) { + BlockBuilder meta_index_block(&r->options); + if (r->filter_block != nullptr) { + // Add mapping from "filter.Name" to location of filter data + std::string key = "filter."; + key.append(r->options.filter_policy->Name()); + std::string handle_encoding; + filter_block_handle.EncodeTo(&handle_encoding); + meta_index_block.Add(key, handle_encoding); + } + + // TODO(postrelease): Add stats and other meta blocks + WriteBlock(&meta_index_block, &metaindex_block_handle); + } + + // Write index block + if (ok()) { + if (r->pending_index_entry) { + r->options.comparator->FindShortSuccessor(&r->last_key); + std::string handle_encoding; + r->pending_handle.EncodeTo(&handle_encoding); + r->index_block.Add(r->last_key, Slice(handle_encoding)); + r->pending_index_entry = false; + } + WriteBlock(&r->index_block, &index_block_handle); + } + + // Write footer + if (ok()) { + Footer footer; + footer.set_metaindex_handle(metaindex_block_handle); + footer.set_index_handle(index_block_handle); + std::string footer_encoding; + footer.EncodeTo(&footer_encoding); + r->status = r->file->Append(footer_encoding); + if (r->status.ok()) { + r->offset += footer_encoding.size(); + } + } + return r->status; +} + +void TableBuilder::Abandon() { + Rep* r = rep_; + assert(!r->closed); + r->closed = true; +} + +uint64_t TableBuilder::NumEntries() const { return rep_->num_entries; } + +uint64_t TableBuilder::FileSize() const { return rep_->offset; } + +} // namespace leveldb diff --git a/leveldb/table/table_test.cc b/leveldb/table/table_test.cc new file mode 100644 index 000000000..b3baf957d --- /dev/null +++ b/leveldb/table/table_test.cc @@ -0,0 +1,842 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/table.h" + +#include +#include + +#include "gtest/gtest.h" +#include "db/dbformat.h" +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" +#include "leveldb/table_builder.h" +#include "table/block.h" +#include "table/block_builder.h" +#include "table/format.h" +#include "util/random.h" +#include "util/testutil.h" + +namespace leveldb { + +// Return reverse of "key". +// Used to test non-lexicographic comparators. +static std::string Reverse(const Slice& key) { + std::string str(key.ToString()); + std::string rev(""); + for (std::string::reverse_iterator rit = str.rbegin(); rit != str.rend(); + ++rit) { + rev.push_back(*rit); + } + return rev; +} + +namespace { +class ReverseKeyComparator : public Comparator { + public: + const char* Name() const override { + return "leveldb.ReverseBytewiseComparator"; + } + + int Compare(const Slice& a, const Slice& b) const override { + return BytewiseComparator()->Compare(Reverse(a), Reverse(b)); + } + + void FindShortestSeparator(std::string* start, + const Slice& limit) const override { + std::string s = Reverse(*start); + std::string l = Reverse(limit); + BytewiseComparator()->FindShortestSeparator(&s, l); + *start = Reverse(s); + } + + void FindShortSuccessor(std::string* key) const override { + std::string s = Reverse(*key); + BytewiseComparator()->FindShortSuccessor(&s); + *key = Reverse(s); + } +}; +} // namespace +static ReverseKeyComparator reverse_key_comparator; + +static void Increment(const Comparator* cmp, std::string* key) { + if (cmp == BytewiseComparator()) { + key->push_back('\0'); + } else { + assert(cmp == &reverse_key_comparator); + std::string rev = Reverse(*key); + rev.push_back('\0'); + *key = Reverse(rev); + } +} + +// An STL comparator that uses a Comparator +namespace { +struct STLLessThan { + const Comparator* cmp; + + STLLessThan() : cmp(BytewiseComparator()) {} + STLLessThan(const Comparator* c) : cmp(c) {} + bool operator()(const std::string& a, const std::string& b) const { + return cmp->Compare(Slice(a), Slice(b)) < 0; + } +}; +} // namespace + +class StringSink : public WritableFile { + public: + ~StringSink() override = default; + + const std::string& contents() const { return contents_; } + + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + + Status Append(const Slice& data) override { + contents_.append(data.data(), data.size()); + return Status::OK(); + } + + private: + std::string contents_; +}; + +class StringSource : public RandomAccessFile { + public: + StringSource(const Slice& contents) + : contents_(contents.data(), contents.size()) {} + + ~StringSource() override = default; + + uint64_t Size() const { return contents_.size(); } + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + if (offset >= contents_.size()) { + return Status::InvalidArgument("invalid Read offset"); + } + if (offset + n > contents_.size()) { + n = contents_.size() - offset; + } + std::memcpy(scratch, &contents_[offset], n); + *result = Slice(scratch, n); + return Status::OK(); + } + + private: + std::string contents_; +}; + +typedef std::map KVMap; + +// Helper class for tests to unify the interface between +// BlockBuilder/TableBuilder and Block/Table. +class Constructor { + public: + explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) {} + virtual ~Constructor() = default; + + void Add(const std::string& key, const Slice& value) { + data_[key] = value.ToString(); + } + + // Finish constructing the data structure with all the keys that have + // been added so far. Returns the keys in sorted order in "*keys" + // and stores the key/value pairs in "*kvmap" + void Finish(const Options& options, std::vector* keys, + KVMap* kvmap) { + *kvmap = data_; + keys->clear(); + for (const auto& kvp : data_) { + keys->push_back(kvp.first); + } + data_.clear(); + Status s = FinishImpl(options, *kvmap); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + // Construct the data structure from the data in "data" + virtual Status FinishImpl(const Options& options, const KVMap& data) = 0; + + virtual Iterator* NewIterator() const = 0; + + const KVMap& data() const { return data_; } + + virtual DB* db() const { return nullptr; } // Overridden in DBConstructor + + private: + KVMap data_; +}; + +class BlockConstructor : public Constructor { + public: + explicit BlockConstructor(const Comparator* cmp) + : Constructor(cmp), comparator_(cmp), block_(nullptr) {} + ~BlockConstructor() override { delete block_; } + Status FinishImpl(const Options& options, const KVMap& data) override { + delete block_; + block_ = nullptr; + BlockBuilder builder(&options); + + for (const auto& kvp : data) { + builder.Add(kvp.first, kvp.second); + } + // Open the block + data_ = builder.Finish().ToString(); + BlockContents contents; + contents.data = data_; + contents.cachable = false; + contents.heap_allocated = false; + block_ = new Block(contents); + return Status::OK(); + } + Iterator* NewIterator() const override { + return block_->NewIterator(comparator_); + } + + private: + const Comparator* const comparator_; + std::string data_; + Block* block_; + + BlockConstructor(); +}; + +class TableConstructor : public Constructor { + public: + TableConstructor(const Comparator* cmp) + : Constructor(cmp), source_(nullptr), table_(nullptr) {} + ~TableConstructor() override { Reset(); } + Status FinishImpl(const Options& options, const KVMap& data) override { + Reset(); + StringSink sink; + TableBuilder builder(options, &sink); + + for (const auto& kvp : data) { + builder.Add(kvp.first, kvp.second); + EXPECT_LEVELDB_OK(builder.status()); + } + Status s = builder.Finish(); + EXPECT_LEVELDB_OK(s); + + EXPECT_EQ(sink.contents().size(), builder.FileSize()); + + // Open the table + source_ = new StringSource(sink.contents()); + Options table_options; + table_options.comparator = options.comparator; + return Table::Open(table_options, source_, sink.contents().size(), &table_); + } + + Iterator* NewIterator() const override { + return table_->NewIterator(ReadOptions()); + } + + uint64_t ApproximateOffsetOf(const Slice& key) const { + return table_->ApproximateOffsetOf(key); + } + + private: + void Reset() { + delete table_; + delete source_; + table_ = nullptr; + source_ = nullptr; + } + + StringSource* source_; + Table* table_; + + TableConstructor(); +}; + +// A helper class that converts internal format keys into user keys +class KeyConvertingIterator : public Iterator { + public: + explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) {} + + KeyConvertingIterator(const KeyConvertingIterator&) = delete; + KeyConvertingIterator& operator=(const KeyConvertingIterator&) = delete; + + ~KeyConvertingIterator() override { delete iter_; } + + bool Valid() const override { return iter_->Valid(); } + void Seek(const Slice& target) override { + ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue); + std::string encoded; + AppendInternalKey(&encoded, ikey); + iter_->Seek(encoded); + } + void SeekToFirst() override { iter_->SeekToFirst(); } + void SeekToLast() override { iter_->SeekToLast(); } + void Next() override { iter_->Next(); } + void Prev() override { iter_->Prev(); } + + Slice key() const override { + assert(Valid()); + ParsedInternalKey key; + if (!ParseInternalKey(iter_->key(), &key)) { + status_ = Status::Corruption("malformed internal key"); + return Slice("corrupted key"); + } + return key.user_key; + } + + Slice value() const override { return iter_->value(); } + Status status() const override { + return status_.ok() ? iter_->status() : status_; + } + + private: + mutable Status status_; + Iterator* iter_; +}; + +class MemTableConstructor : public Constructor { + public: + explicit MemTableConstructor(const Comparator* cmp) + : Constructor(cmp), internal_comparator_(cmp) { + memtable_ = new MemTable(internal_comparator_); + memtable_->Ref(); + } + ~MemTableConstructor() override { memtable_->Unref(); } + Status FinishImpl(const Options& options, const KVMap& data) override { + memtable_->Unref(); + memtable_ = new MemTable(internal_comparator_); + memtable_->Ref(); + int seq = 1; + for (const auto& kvp : data) { + memtable_->Add(seq, kTypeValue, kvp.first, kvp.second); + seq++; + } + return Status::OK(); + } + Iterator* NewIterator() const override { + return new KeyConvertingIterator(memtable_->NewIterator()); + } + + private: + const InternalKeyComparator internal_comparator_; + MemTable* memtable_; +}; + +class DBConstructor : public Constructor { + public: + explicit DBConstructor(const Comparator* cmp) + : Constructor(cmp), comparator_(cmp) { + db_ = nullptr; + NewDB(); + } + ~DBConstructor() override { delete db_; } + Status FinishImpl(const Options& options, const KVMap& data) override { + delete db_; + db_ = nullptr; + NewDB(); + for (const auto& kvp : data) { + WriteBatch batch; + batch.Put(kvp.first, kvp.second); + EXPECT_TRUE(db_->Write(WriteOptions(), &batch).ok()); + } + return Status::OK(); + } + Iterator* NewIterator() const override { + return db_->NewIterator(ReadOptions()); + } + + DB* db() const override { return db_; } + + private: + void NewDB() { + std::string name = testing::TempDir() + "table_testdb"; + + Options options; + options.comparator = comparator_; + Status status = DestroyDB(name, options); + ASSERT_TRUE(status.ok()) << status.ToString(); + + options.create_if_missing = true; + options.error_if_exists = true; + options.write_buffer_size = 10000; // Something small to force merging + status = DB::Open(options, name, &db_); + ASSERT_TRUE(status.ok()) << status.ToString(); + } + + const Comparator* const comparator_; + DB* db_; +}; + +enum TestType { TABLE_TEST, BLOCK_TEST, MEMTABLE_TEST, DB_TEST }; + +struct TestArgs { + TestType type; + bool reverse_compare; + int restart_interval; +}; + +static const TestArgs kTestArgList[] = { + {TABLE_TEST, false, 16}, + {TABLE_TEST, false, 1}, + {TABLE_TEST, false, 1024}, + {TABLE_TEST, true, 16}, + {TABLE_TEST, true, 1}, + {TABLE_TEST, true, 1024}, + + {BLOCK_TEST, false, 16}, + {BLOCK_TEST, false, 1}, + {BLOCK_TEST, false, 1024}, + {BLOCK_TEST, true, 16}, + {BLOCK_TEST, true, 1}, + {BLOCK_TEST, true, 1024}, + + // Restart interval does not matter for memtables + {MEMTABLE_TEST, false, 16}, + {MEMTABLE_TEST, true, 16}, + + // Do not bother with restart interval variations for DB + {DB_TEST, false, 16}, + {DB_TEST, true, 16}, +}; +static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]); + +class Harness : public testing::Test { + public: + Harness() : constructor_(nullptr) {} + + void Init(const TestArgs& args) { + delete constructor_; + constructor_ = nullptr; + options_ = Options(); + + options_.block_restart_interval = args.restart_interval; + // Use shorter block size for tests to exercise block boundary + // conditions more. + options_.block_size = 256; + if (args.reverse_compare) { + options_.comparator = &reverse_key_comparator; + } + switch (args.type) { + case TABLE_TEST: + constructor_ = new TableConstructor(options_.comparator); + break; + case BLOCK_TEST: + constructor_ = new BlockConstructor(options_.comparator); + break; + case MEMTABLE_TEST: + constructor_ = new MemTableConstructor(options_.comparator); + break; + case DB_TEST: + constructor_ = new DBConstructor(options_.comparator); + break; + } + } + + ~Harness() { delete constructor_; } + + void Add(const std::string& key, const std::string& value) { + constructor_->Add(key, value); + } + + void Test(Random* rnd) { + std::vector keys; + KVMap data; + constructor_->Finish(options_, &keys, &data); + + TestForwardScan(keys, data); + TestBackwardScan(keys, data); + TestRandomAccess(rnd, keys, data); + } + + void TestForwardScan(const std::vector& keys, + const KVMap& data) { + Iterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToFirst(); + for (KVMap::const_iterator model_iter = data.begin(); + model_iter != data.end(); ++model_iter) { + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + iter->Next(); + } + ASSERT_TRUE(!iter->Valid()); + delete iter; + } + + void TestBackwardScan(const std::vector& keys, + const KVMap& data) { + Iterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToLast(); + for (KVMap::const_reverse_iterator model_iter = data.rbegin(); + model_iter != data.rend(); ++model_iter) { + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + iter->Prev(); + } + ASSERT_TRUE(!iter->Valid()); + delete iter; + } + + void TestRandomAccess(Random* rnd, const std::vector& keys, + const KVMap& data) { + static const bool kVerbose = false; + Iterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + KVMap::const_iterator model_iter = data.begin(); + if (kVerbose) std::fprintf(stderr, "---\n"); + for (int i = 0; i < 200; i++) { + const int toss = rnd->Uniform(5); + switch (toss) { + case 0: { + if (iter->Valid()) { + if (kVerbose) std::fprintf(stderr, "Next\n"); + iter->Next(); + ++model_iter; + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + } + break; + } + + case 1: { + if (kVerbose) std::fprintf(stderr, "SeekToFirst\n"); + iter->SeekToFirst(); + model_iter = data.begin(); + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + + case 2: { + std::string key = PickRandomKey(rnd, keys); + model_iter = data.lower_bound(key); + if (kVerbose) + std::fprintf(stderr, "Seek '%s'\n", EscapeString(key).c_str()); + iter->Seek(Slice(key)); + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + + case 3: { + if (iter->Valid()) { + if (kVerbose) std::fprintf(stderr, "Prev\n"); + iter->Prev(); + if (model_iter == data.begin()) { + model_iter = data.end(); // Wrap around to invalid value + } else { + --model_iter; + } + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + } + break; + } + + case 4: { + if (kVerbose) std::fprintf(stderr, "SeekToLast\n"); + iter->SeekToLast(); + if (keys.empty()) { + model_iter = data.end(); + } else { + std::string last = data.rbegin()->first; + model_iter = data.lower_bound(last); + } + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + } + } + delete iter; + } + + std::string ToString(const KVMap& data, const KVMap::const_iterator& it) { + if (it == data.end()) { + return "END"; + } else { + return "'" + it->first + "->" + it->second + "'"; + } + } + + std::string ToString(const KVMap& data, + const KVMap::const_reverse_iterator& it) { + if (it == data.rend()) { + return "END"; + } else { + return "'" + it->first + "->" + it->second + "'"; + } + } + + std::string ToString(const Iterator* it) { + if (!it->Valid()) { + return "END"; + } else { + return "'" + it->key().ToString() + "->" + it->value().ToString() + "'"; + } + } + + std::string PickRandomKey(Random* rnd, const std::vector& keys) { + if (keys.empty()) { + return "foo"; + } else { + const int index = rnd->Uniform(keys.size()); + std::string result = keys[index]; + switch (rnd->Uniform(3)) { + case 0: + // Return an existing key + break; + case 1: { + // Attempt to return something smaller than an existing key + if (!result.empty() && result[result.size() - 1] > '\0') { + result[result.size() - 1]--; + } + break; + } + case 2: { + // Return something larger than an existing key + Increment(options_.comparator, &result); + break; + } + } + return result; + } + } + + // Returns nullptr if not running against a DB + DB* db() const { return constructor_->db(); } + + private: + Options options_; + Constructor* constructor_; +}; + +// Test empty table/block. +TEST_F(Harness, Empty) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 1); + Test(&rnd); + } +} + +// Special test for a block with no restart entries. The C++ leveldb +// code never generates such blocks, but the Java version of leveldb +// seems to. +TEST_F(Harness, ZeroRestartPointsInBlock) { + char data[sizeof(uint32_t)]; + memset(data, 0, sizeof(data)); + BlockContents contents; + contents.data = Slice(data, sizeof(data)); + contents.cachable = false; + contents.heap_allocated = false; + Block block(contents); + Iterator* iter = block.NewIterator(BytewiseComparator()); + iter->SeekToFirst(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToLast(); + ASSERT_TRUE(!iter->Valid()); + iter->Seek("foo"); + ASSERT_TRUE(!iter->Valid()); + delete iter; +} + +// Test the empty key +TEST_F(Harness, SimpleEmptyKey) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 1); + Add("", "v"); + Test(&rnd); + } +} + +TEST_F(Harness, SimpleSingle) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 2); + Add("abc", "v"); + Test(&rnd); + } +} + +TEST_F(Harness, SimpleMulti) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 3); + Add("abc", "v"); + Add("abcd", "v"); + Add("ac", "v2"); + Test(&rnd); + } +} + +TEST_F(Harness, SimpleSpecialKey) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 4); + Add("\xff\xff", "v3"); + Test(&rnd); + } +} + +TEST_F(Harness, Randomized) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 5); + for (int num_entries = 0; num_entries < 2000; + num_entries += (num_entries < 50 ? 1 : 200)) { + if ((num_entries % 10) == 0) { + std::fprintf(stderr, "case %d of %d: num_entries = %d\n", (i + 1), + int(kNumTestArgs), num_entries); + } + for (int e = 0; e < num_entries; e++) { + std::string v; + Add(test::RandomKey(&rnd, rnd.Skewed(4)), + test::RandomString(&rnd, rnd.Skewed(5), &v).ToString()); + } + Test(&rnd); + } + } +} + +TEST_F(Harness, RandomizedLongDB) { + Random rnd(test::RandomSeed()); + TestArgs args = {DB_TEST, false, 16}; + Init(args); + int num_entries = 100000; + for (int e = 0; e < num_entries; e++) { + std::string v; + Add(test::RandomKey(&rnd, rnd.Skewed(4)), + test::RandomString(&rnd, rnd.Skewed(5), &v).ToString()); + } + Test(&rnd); + + // We must have created enough data to force merging + int files = 0; + for (int level = 0; level < config::kNumLevels; level++) { + std::string value; + char name[100]; + std::snprintf(name, sizeof(name), "leveldb.num-files-at-level%d", level); + ASSERT_TRUE(db()->GetProperty(name, &value)); + files += atoi(value.c_str()); + } + ASSERT_GT(files, 0); +} + +TEST(MemTableTest, Simple) { + InternalKeyComparator cmp(BytewiseComparator()); + MemTable* memtable = new MemTable(cmp); + memtable->Ref(); + WriteBatch batch; + WriteBatchInternal::SetSequence(&batch, 100); + batch.Put(std::string("k1"), std::string("v1")); + batch.Put(std::string("k2"), std::string("v2")); + batch.Put(std::string("k3"), std::string("v3")); + batch.Put(std::string("largekey"), std::string("vlarge")); + ASSERT_TRUE(WriteBatchInternal::InsertInto(&batch, memtable).ok()); + + Iterator* iter = memtable->NewIterator(); + iter->SeekToFirst(); + while (iter->Valid()) { + std::fprintf(stderr, "key: '%s' -> '%s'\n", iter->key().ToString().c_str(), + iter->value().ToString().c_str()); + iter->Next(); + } + + delete iter; + memtable->Unref(); +} + +static bool Between(uint64_t val, uint64_t low, uint64_t high) { + bool result = (val >= low) && (val <= high); + if (!result) { + std::fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", + (unsigned long long)(val), (unsigned long long)(low), + (unsigned long long)(high)); + } + return result; +} + +TEST(TableTest, ApproximateOffsetOfPlain) { + TableConstructor c(BytewiseComparator()); + c.Add("k01", "hello"); + c.Add("k02", "hello2"); + c.Add("k03", std::string(10000, 'x')); + c.Add("k04", std::string(200000, 'x')); + c.Add("k05", std::string(300000, 'x')); + c.Add("k06", "hello3"); + c.Add("k07", std::string(100000, 'x')); + std::vector keys; + KVMap kvmap; + Options options; + options.block_size = 1024; + options.compression = kNoCompression; + c.Finish(options, &keys, &kvmap); + + ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01a"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 10000, 11000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04a"), 210000, 211000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k05"), 210000, 211000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 612000)); +} + +static bool CompressionSupported(CompressionType type) { + std::string out; + Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + if (type == kSnappyCompression) { + return port::Snappy_Compress(in.data(), in.size(), &out); + } else if (type == kZstdCompression) { + return port::Zstd_Compress(in.data(), in.size(), &out); + } + return false; +} + +class CompressionTableTest + : public ::testing::TestWithParam> {}; + +INSTANTIATE_TEST_SUITE_P(CompressionTests, CompressionTableTest, + ::testing::Values(kSnappyCompression, + kZstdCompression)); + +TEST_P(CompressionTableTest, ApproximateOffsetOfCompressed) { + CompressionType type = ::testing::get<0>(GetParam()); + if (!CompressionSupported(type)) { + GTEST_SKIP() << "skipping compression test: " << type; + } + + Random rnd(301); + TableConstructor c(BytewiseComparator()); + std::string tmp; + c.Add("k01", "hello"); + c.Add("k02", test::CompressibleString(&rnd, 0.25, 10000, &tmp)); + c.Add("k03", "hello3"); + c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp)); + std::vector keys; + KVMap kvmap; + Options options; + options.block_size = 1024; + options.compression = type; + c.Finish(options, &keys, &kvmap); + + // Expected upper and lower bounds of space used by compressible strings. + static const int kSlop = 1000; // Compressor effectiveness varies. + const int expected = 2500; // 10000 * compression ratio (0.25) + const int min_z = expected - kSlop; + const int max_z = expected + kSlop; + + ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, kSlop)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, kSlop)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, kSlop)); + // Have now emitted a large compressible string, so adjust expected offset. + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), min_z, max_z)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), min_z, max_z)); + // Have now emitted two large compressible strings, so adjust expected offset. + ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 2 * min_z, 2 * max_z)); +} + +} // namespace leveldb diff --git a/leveldb/table/two_level_iterator.cc b/leveldb/table/two_level_iterator.cc new file mode 100644 index 000000000..144790dd9 --- /dev/null +++ b/leveldb/table/two_level_iterator.cc @@ -0,0 +1,171 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/two_level_iterator.h" + +#include "leveldb/table.h" +#include "table/block.h" +#include "table/format.h" +#include "table/iterator_wrapper.h" + +namespace leveldb { + +namespace { + +typedef Iterator* (*BlockFunction)(void*, const ReadOptions&, const Slice&); + +class TwoLevelIterator : public Iterator { + public: + TwoLevelIterator(Iterator* index_iter, BlockFunction block_function, + void* arg, const ReadOptions& options); + + ~TwoLevelIterator() override; + + void Seek(const Slice& target) override; + void SeekToFirst() override; + void SeekToLast() override; + void Next() override; + void Prev() override; + + bool Valid() const override { return data_iter_.Valid(); } + Slice key() const override { + assert(Valid()); + return data_iter_.key(); + } + Slice value() const override { + assert(Valid()); + return data_iter_.value(); + } + Status status() const override { + // It'd be nice if status() returned a const Status& instead of a Status + if (!index_iter_.status().ok()) { + return index_iter_.status(); + } else if (data_iter_.iter() != nullptr && !data_iter_.status().ok()) { + return data_iter_.status(); + } else { + return status_; + } + } + + private: + void SaveError(const Status& s) { + if (status_.ok() && !s.ok()) status_ = s; + } + void SkipEmptyDataBlocksForward(); + void SkipEmptyDataBlocksBackward(); + void SetDataIterator(Iterator* data_iter); + void InitDataBlock(); + + BlockFunction block_function_; + void* arg_; + const ReadOptions options_; + Status status_; + IteratorWrapper index_iter_; + IteratorWrapper data_iter_; // May be nullptr + // If data_iter_ is non-null, then "data_block_handle_" holds the + // "index_value" passed to block_function_ to create the data_iter_. + std::string data_block_handle_; +}; + +TwoLevelIterator::TwoLevelIterator(Iterator* index_iter, + BlockFunction block_function, void* arg, + const ReadOptions& options) + : block_function_(block_function), + arg_(arg), + options_(options), + index_iter_(index_iter), + data_iter_(nullptr) {} + +TwoLevelIterator::~TwoLevelIterator() = default; + +void TwoLevelIterator::Seek(const Slice& target) { + index_iter_.Seek(target); + InitDataBlock(); + if (data_iter_.iter() != nullptr) data_iter_.Seek(target); + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIterator::SeekToFirst() { + index_iter_.SeekToFirst(); + InitDataBlock(); + if (data_iter_.iter() != nullptr) data_iter_.SeekToFirst(); + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIterator::SeekToLast() { + index_iter_.SeekToLast(); + InitDataBlock(); + if (data_iter_.iter() != nullptr) data_iter_.SeekToLast(); + SkipEmptyDataBlocksBackward(); +} + +void TwoLevelIterator::Next() { + assert(Valid()); + data_iter_.Next(); + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIterator::Prev() { + assert(Valid()); + data_iter_.Prev(); + SkipEmptyDataBlocksBackward(); +} + +void TwoLevelIterator::SkipEmptyDataBlocksForward() { + while (data_iter_.iter() == nullptr || !data_iter_.Valid()) { + // Move to next block + if (!index_iter_.Valid()) { + SetDataIterator(nullptr); + return; + } + index_iter_.Next(); + InitDataBlock(); + if (data_iter_.iter() != nullptr) data_iter_.SeekToFirst(); + } +} + +void TwoLevelIterator::SkipEmptyDataBlocksBackward() { + while (data_iter_.iter() == nullptr || !data_iter_.Valid()) { + // Move to next block + if (!index_iter_.Valid()) { + SetDataIterator(nullptr); + return; + } + index_iter_.Prev(); + InitDataBlock(); + if (data_iter_.iter() != nullptr) data_iter_.SeekToLast(); + } +} + +void TwoLevelIterator::SetDataIterator(Iterator* data_iter) { + if (data_iter_.iter() != nullptr) SaveError(data_iter_.status()); + data_iter_.Set(data_iter); +} + +void TwoLevelIterator::InitDataBlock() { + if (!index_iter_.Valid()) { + SetDataIterator(nullptr); + } else { + Slice handle = index_iter_.value(); + if (data_iter_.iter() != nullptr && + handle.compare(data_block_handle_) == 0) { + // data_iter_ is already constructed with this iterator, so + // no need to change anything + } else { + Iterator* iter = (*block_function_)(arg_, options_, handle); + data_block_handle_.assign(handle.data(), handle.size()); + SetDataIterator(iter); + } + } +} + +} // namespace + +Iterator* NewTwoLevelIterator(Iterator* index_iter, + BlockFunction block_function, void* arg, + const ReadOptions& options) { + return new TwoLevelIterator(index_iter, block_function, arg, options); +} + +} // namespace leveldb diff --git a/leveldb/table/two_level_iterator.h b/leveldb/table/two_level_iterator.h new file mode 100644 index 000000000..81ffe809a --- /dev/null +++ b/leveldb/table/two_level_iterator.h @@ -0,0 +1,31 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ +#define STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ + +#include "leveldb/iterator.h" + +namespace leveldb { + +struct ReadOptions; + +// Return a new two level iterator. A two-level iterator contains an +// index iterator whose values point to a sequence of blocks where +// each block is itself a sequence of key,value pairs. The returned +// two-level iterator yields the concatenation of all key/value pairs +// in the sequence of blocks. Takes ownership of "index_iter" and +// will delete it when no longer needed. +// +// Uses a supplied function to convert an index_iter value into +// an iterator over the contents of the corresponding block. +Iterator* NewTwoLevelIterator( + Iterator* index_iter, + Iterator* (*block_function)(void* arg, const ReadOptions& options, + const Slice& index_value), + void* arg, const ReadOptions& options); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ diff --git a/leveldb/util/arena.cc b/leveldb/util/arena.cc new file mode 100644 index 000000000..46e3b2eb8 --- /dev/null +++ b/leveldb/util/arena.cc @@ -0,0 +1,66 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/arena.h" + +namespace leveldb { + +static const int kBlockSize = 4096; + +Arena::Arena() + : alloc_ptr_(nullptr), alloc_bytes_remaining_(0), memory_usage_(0) {} + +Arena::~Arena() { + for (size_t i = 0; i < blocks_.size(); i++) { + delete[] blocks_[i]; + } +} + +char* Arena::AllocateFallback(size_t bytes) { + if (bytes > kBlockSize / 4) { + // Object is more than a quarter of our block size. Allocate it separately + // to avoid wasting too much space in leftover bytes. + char* result = AllocateNewBlock(bytes); + return result; + } + + // We waste the remaining space in the current block. + alloc_ptr_ = AllocateNewBlock(kBlockSize); + alloc_bytes_remaining_ = kBlockSize; + + char* result = alloc_ptr_; + alloc_ptr_ += bytes; + alloc_bytes_remaining_ -= bytes; + return result; +} + +char* Arena::AllocateAligned(size_t bytes) { + const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8; + static_assert((align & (align - 1)) == 0, + "Pointer size should be a power of 2"); + size_t current_mod = reinterpret_cast(alloc_ptr_) & (align - 1); + size_t slop = (current_mod == 0 ? 0 : align - current_mod); + size_t needed = bytes + slop; + char* result; + if (needed <= alloc_bytes_remaining_) { + result = alloc_ptr_ + slop; + alloc_ptr_ += needed; + alloc_bytes_remaining_ -= needed; + } else { + // AllocateFallback always returned aligned memory + result = AllocateFallback(bytes); + } + assert((reinterpret_cast(result) & (align - 1)) == 0); + return result; +} + +char* Arena::AllocateNewBlock(size_t block_bytes) { + char* result = new char[block_bytes]; + blocks_.push_back(result); + memory_usage_.fetch_add(block_bytes + sizeof(char*), + std::memory_order_relaxed); + return result; +} + +} // namespace leveldb diff --git a/leveldb/util/arena.h b/leveldb/util/arena.h new file mode 100644 index 000000000..68fc55d4d --- /dev/null +++ b/leveldb/util/arena.h @@ -0,0 +1,71 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_ARENA_H_ +#define STORAGE_LEVELDB_UTIL_ARENA_H_ + +#include +#include +#include +#include +#include + +namespace leveldb { + +class Arena { + public: + Arena(); + + Arena(const Arena&) = delete; + Arena& operator=(const Arena&) = delete; + + ~Arena(); + + // Return a pointer to a newly allocated memory block of "bytes" bytes. + char* Allocate(size_t bytes); + + // Allocate memory with the normal alignment guarantees provided by malloc. + char* AllocateAligned(size_t bytes); + + // Returns an estimate of the total memory usage of data allocated + // by the arena. + size_t MemoryUsage() const { + return memory_usage_.load(std::memory_order_relaxed); + } + + private: + char* AllocateFallback(size_t bytes); + char* AllocateNewBlock(size_t block_bytes); + + // Allocation state + char* alloc_ptr_; + size_t alloc_bytes_remaining_; + + // Array of new[] allocated memory blocks + std::vector blocks_; + + // Total memory usage of the arena. + // + // TODO(costan): This member is accessed via atomics, but the others are + // accessed without any locking. Is this OK? + std::atomic memory_usage_; +}; + +inline char* Arena::Allocate(size_t bytes) { + // The semantics of what to return are a bit messy if we allow + // 0-byte allocations, so we disallow them here (we don't need + // them for our internal use). + assert(bytes > 0); + if (bytes <= alloc_bytes_remaining_) { + char* result = alloc_ptr_; + alloc_ptr_ += bytes; + alloc_bytes_remaining_ -= bytes; + return result; + } + return AllocateFallback(bytes); +} + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_ARENA_H_ diff --git a/leveldb/util/arena_test.cc b/leveldb/util/arena_test.cc new file mode 100644 index 000000000..3e2011eca --- /dev/null +++ b/leveldb/util/arena_test.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/arena.h" + +#include "gtest/gtest.h" +#include "util/random.h" + +namespace leveldb { + +TEST(ArenaTest, Empty) { Arena arena; } + +TEST(ArenaTest, Simple) { + std::vector> allocated; + Arena arena; + const int N = 100000; + size_t bytes = 0; + Random rnd(301); + for (int i = 0; i < N; i++) { + size_t s; + if (i % (N / 10) == 0) { + s = i; + } else { + s = rnd.OneIn(4000) + ? rnd.Uniform(6000) + : (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); + } + if (s == 0) { + // Our arena disallows size 0 allocations. + s = 1; + } + char* r; + if (rnd.OneIn(10)) { + r = arena.AllocateAligned(s); + } else { + r = arena.Allocate(s); + } + + for (size_t b = 0; b < s; b++) { + // Fill the "i"th allocation with a known bit pattern + r[b] = i % 256; + } + bytes += s; + allocated.push_back(std::make_pair(s, r)); + ASSERT_GE(arena.MemoryUsage(), bytes); + if (i > N / 10) { + ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); + } + } + for (size_t i = 0; i < allocated.size(); i++) { + size_t num_bytes = allocated[i].first; + const char* p = allocated[i].second; + for (size_t b = 0; b < num_bytes; b++) { + // Check the "i"th allocation for the known bit pattern + ASSERT_EQ(int(p[b]) & 0xff, i % 256); + } + } +} + +} // namespace leveldb diff --git a/leveldb/util/bloom.cc b/leveldb/util/bloom.cc new file mode 100644 index 000000000..87547a7e6 --- /dev/null +++ b/leveldb/util/bloom.cc @@ -0,0 +1,92 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/filter_policy.h" + +#include "leveldb/slice.h" +#include "util/hash.h" + +namespace leveldb { + +namespace { +static uint32_t BloomHash(const Slice& key) { + return Hash(key.data(), key.size(), 0xbc9f1d34); +} + +class BloomFilterPolicy : public FilterPolicy { + public: + explicit BloomFilterPolicy(int bits_per_key) : bits_per_key_(bits_per_key) { + // We intentionally round down to reduce probing cost a little bit + k_ = static_cast(bits_per_key * 0.69); // 0.69 =~ ln(2) + if (k_ < 1) k_ = 1; + if (k_ > 30) k_ = 30; + } + + const char* Name() const override { return "leveldb.BuiltinBloomFilter2"; } + + void CreateFilter(const Slice* keys, int n, std::string* dst) const override { + // Compute bloom filter size (in both bits and bytes) + size_t bits = n * bits_per_key_; + + // For small n, we can see a very high false positive rate. Fix it + // by enforcing a minimum bloom filter length. + if (bits < 64) bits = 64; + + size_t bytes = (bits + 7) / 8; + bits = bytes * 8; + + const size_t init_size = dst->size(); + dst->resize(init_size + bytes, 0); + dst->push_back(static_cast(k_)); // Remember # of probes in filter + char* array = &(*dst)[init_size]; + for (int i = 0; i < n; i++) { + // Use double-hashing to generate a sequence of hash values. + // See analysis in [Kirsch,Mitzenmacher 2006]. + uint32_t h = BloomHash(keys[i]); + const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits + for (size_t j = 0; j < k_; j++) { + const uint32_t bitpos = h % bits; + array[bitpos / 8] |= (1 << (bitpos % 8)); + h += delta; + } + } + } + + bool KeyMayMatch(const Slice& key, const Slice& bloom_filter) const override { + const size_t len = bloom_filter.size(); + if (len < 2) return false; + + const char* array = bloom_filter.data(); + const size_t bits = (len - 1) * 8; + + // Use the encoded k so that we can read filters generated by + // bloom filters created using different parameters. + const size_t k = array[len - 1]; + if (k > 30) { + // Reserved for potentially new encodings for short bloom filters. + // Consider it a match. + return true; + } + + uint32_t h = BloomHash(key); + const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits + for (size_t j = 0; j < k; j++) { + const uint32_t bitpos = h % bits; + if ((array[bitpos / 8] & (1 << (bitpos % 8))) == 0) return false; + h += delta; + } + return true; + } + + private: + size_t bits_per_key_; + size_t k_; +}; +} // namespace + +const FilterPolicy* NewBloomFilterPolicy(int bits_per_key) { + return new BloomFilterPolicy(bits_per_key); +} + +} // namespace leveldb diff --git a/leveldb/util/bloom_test.cc b/leveldb/util/bloom_test.cc new file mode 100644 index 000000000..9f11108ee --- /dev/null +++ b/leveldb/util/bloom_test.cc @@ -0,0 +1,154 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "gtest/gtest.h" +#include "leveldb/filter_policy.h" +#include "util/coding.h" +#include "util/logging.h" +#include "util/testutil.h" + +namespace leveldb { + +static const int kVerbose = 1; + +static Slice Key(int i, char* buffer) { + EncodeFixed32(buffer, i); + return Slice(buffer, sizeof(uint32_t)); +} + +class BloomTest : public testing::Test { + public: + BloomTest() : policy_(NewBloomFilterPolicy(10)) {} + + ~BloomTest() { delete policy_; } + + void Reset() { + keys_.clear(); + filter_.clear(); + } + + void Add(const Slice& s) { keys_.push_back(s.ToString()); } + + void Build() { + std::vector key_slices; + for (size_t i = 0; i < keys_.size(); i++) { + key_slices.push_back(Slice(keys_[i])); + } + filter_.clear(); + policy_->CreateFilter(&key_slices[0], static_cast(key_slices.size()), + &filter_); + keys_.clear(); + if (kVerbose >= 2) DumpFilter(); + } + + size_t FilterSize() const { return filter_.size(); } + + void DumpFilter() { + std::fprintf(stderr, "F("); + for (size_t i = 0; i + 1 < filter_.size(); i++) { + const unsigned int c = static_cast(filter_[i]); + for (int j = 0; j < 8; j++) { + std::fprintf(stderr, "%c", (c & (1 << j)) ? '1' : '.'); + } + } + std::fprintf(stderr, ")\n"); + } + + bool Matches(const Slice& s) { + if (!keys_.empty()) { + Build(); + } + return policy_->KeyMayMatch(s, filter_); + } + + double FalsePositiveRate() { + char buffer[sizeof(int)]; + int result = 0; + for (int i = 0; i < 10000; i++) { + if (Matches(Key(i + 1000000000, buffer))) { + result++; + } + } + return result / 10000.0; + } + + private: + const FilterPolicy* policy_; + std::string filter_; + std::vector keys_; +}; + +TEST_F(BloomTest, EmptyFilter) { + ASSERT_TRUE(!Matches("hello")); + ASSERT_TRUE(!Matches("world")); +} + +TEST_F(BloomTest, Small) { + Add("hello"); + Add("world"); + ASSERT_TRUE(Matches("hello")); + ASSERT_TRUE(Matches("world")); + ASSERT_TRUE(!Matches("x")); + ASSERT_TRUE(!Matches("foo")); +} + +static int NextLength(int length) { + if (length < 10) { + length += 1; + } else if (length < 100) { + length += 10; + } else if (length < 1000) { + length += 100; + } else { + length += 1000; + } + return length; +} + +TEST_F(BloomTest, VaryingLengths) { + char buffer[sizeof(int)]; + + // Count number of filters that significantly exceed the false positive rate + int mediocre_filters = 0; + int good_filters = 0; + + for (int length = 1; length <= 10000; length = NextLength(length)) { + Reset(); + for (int i = 0; i < length; i++) { + Add(Key(i, buffer)); + } + Build(); + + ASSERT_LE(FilterSize(), static_cast((length * 10 / 8) + 40)) + << length; + + // All added keys must match + for (int i = 0; i < length; i++) { + ASSERT_TRUE(Matches(Key(i, buffer))) + << "Length " << length << "; key " << i; + } + + // Check false positive rate + double rate = FalsePositiveRate(); + if (kVerbose >= 1) { + std::fprintf(stderr, + "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n", + rate * 100.0, length, static_cast(FilterSize())); + } + ASSERT_LE(rate, 0.02); // Must not be over 2% + if (rate > 0.0125) + mediocre_filters++; // Allowed, but not too often + else + good_filters++; + } + if (kVerbose >= 1) { + std::fprintf(stderr, "Filters: %d good, %d mediocre\n", good_filters, + mediocre_filters); + } + ASSERT_LE(mediocre_filters, good_filters / 5); +} + +// Different bits-per-byte + +} // namespace leveldb diff --git a/leveldb/util/cache.cc b/leveldb/util/cache.cc new file mode 100644 index 000000000..fc3d154b8 --- /dev/null +++ b/leveldb/util/cache.cc @@ -0,0 +1,401 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/cache.h" + +#include +#include +#include + +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/hash.h" +#include "util/mutexlock.h" + +namespace leveldb { + +Cache::~Cache() {} + +namespace { + +// LRU cache implementation +// +// Cache entries have an "in_cache" boolean indicating whether the cache has a +// reference on the entry. The only ways that this can become false without the +// entry being passed to its "deleter" are via Erase(), via Insert() when +// an element with a duplicate key is inserted, or on destruction of the cache. +// +// The cache keeps two linked lists of items in the cache. All items in the +// cache are in one list or the other, and never both. Items still referenced +// by clients but erased from the cache are in neither list. The lists are: +// - in-use: contains the items currently referenced by clients, in no +// particular order. (This list is used for invariant checking. If we +// removed the check, elements that would otherwise be on this list could be +// left as disconnected singleton lists.) +// - LRU: contains the items not currently referenced by clients, in LRU order +// Elements are moved between these lists by the Ref() and Unref() methods, +// when they detect an element in the cache acquiring or losing its only +// external reference. + +// An entry is a variable length heap-allocated structure. Entries +// are kept in a circular doubly linked list ordered by access time. +struct LRUHandle { + void* value; + void (*deleter)(const Slice&, void* value); + LRUHandle* next_hash; + LRUHandle* next; + LRUHandle* prev; + size_t charge; // TODO(opt): Only allow uint32_t? + size_t key_length; + bool in_cache; // Whether entry is in the cache. + uint32_t refs; // References, including cache reference, if present. + uint32_t hash; // Hash of key(); used for fast sharding and comparisons + char key_data[1]; // Beginning of key + + Slice key() const { + // next is only equal to this if the LRU handle is the list head of an + // empty list. List heads never have meaningful keys. + assert(next != this); + + return Slice(key_data, key_length); + } +}; + +// We provide our own simple hash table since it removes a whole bunch +// of porting hacks and is also faster than some of the built-in hash +// table implementations in some of the compiler/runtime combinations +// we have tested. E.g., readrandom speeds up by ~5% over the g++ +// 4.4.3's builtin hashtable. +class HandleTable { + public: + HandleTable() : length_(0), elems_(0), list_(nullptr) { Resize(); } + ~HandleTable() { delete[] list_; } + + LRUHandle* Lookup(const Slice& key, uint32_t hash) { + return *FindPointer(key, hash); + } + + LRUHandle* Insert(LRUHandle* h) { + LRUHandle** ptr = FindPointer(h->key(), h->hash); + LRUHandle* old = *ptr; + h->next_hash = (old == nullptr ? nullptr : old->next_hash); + *ptr = h; + if (old == nullptr) { + ++elems_; + if (elems_ > length_) { + // Since each cache entry is fairly large, we aim for a small + // average linked list length (<= 1). + Resize(); + } + } + return old; + } + + LRUHandle* Remove(const Slice& key, uint32_t hash) { + LRUHandle** ptr = FindPointer(key, hash); + LRUHandle* result = *ptr; + if (result != nullptr) { + *ptr = result->next_hash; + --elems_; + } + return result; + } + + private: + // The table consists of an array of buckets where each bucket is + // a linked list of cache entries that hash into the bucket. + uint32_t length_; + uint32_t elems_; + LRUHandle** list_; + + // Return a pointer to slot that points to a cache entry that + // matches key/hash. If there is no such cache entry, return a + // pointer to the trailing slot in the corresponding linked list. + LRUHandle** FindPointer(const Slice& key, uint32_t hash) { + LRUHandle** ptr = &list_[hash & (length_ - 1)]; + while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) { + ptr = &(*ptr)->next_hash; + } + return ptr; + } + + void Resize() { + uint32_t new_length = 4; + while (new_length < elems_) { + new_length *= 2; + } + LRUHandle** new_list = new LRUHandle*[new_length]; + memset(new_list, 0, sizeof(new_list[0]) * new_length); + uint32_t count = 0; + for (uint32_t i = 0; i < length_; i++) { + LRUHandle* h = list_[i]; + while (h != nullptr) { + LRUHandle* next = h->next_hash; + uint32_t hash = h->hash; + LRUHandle** ptr = &new_list[hash & (new_length - 1)]; + h->next_hash = *ptr; + *ptr = h; + h = next; + count++; + } + } + assert(elems_ == count); + delete[] list_; + list_ = new_list; + length_ = new_length; + } +}; + +// A single shard of sharded cache. +class LRUCache { + public: + LRUCache(); + ~LRUCache(); + + // Separate from constructor so caller can easily make an array of LRUCache + void SetCapacity(size_t capacity) { capacity_ = capacity; } + + // Like Cache methods, but with an extra "hash" parameter. + Cache::Handle* Insert(const Slice& key, uint32_t hash, void* value, + size_t charge, + void (*deleter)(const Slice& key, void* value)); + Cache::Handle* Lookup(const Slice& key, uint32_t hash); + void Release(Cache::Handle* handle); + void Erase(const Slice& key, uint32_t hash); + void Prune(); + size_t TotalCharge() const { + MutexLock l(&mutex_); + return usage_; + } + + private: + void LRU_Remove(LRUHandle* e); + void LRU_Append(LRUHandle* list, LRUHandle* e); + void Ref(LRUHandle* e); + void Unref(LRUHandle* e); + bool FinishErase(LRUHandle* e) EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Initialized before use. + size_t capacity_; + + // mutex_ protects the following state. + mutable port::Mutex mutex_; + size_t usage_ GUARDED_BY(mutex_); + + // Dummy head of LRU list. + // lru.prev is newest entry, lru.next is oldest entry. + // Entries have refs==1 and in_cache==true. + LRUHandle lru_ GUARDED_BY(mutex_); + + // Dummy head of in-use list. + // Entries are in use by clients, and have refs >= 2 and in_cache==true. + LRUHandle in_use_ GUARDED_BY(mutex_); + + HandleTable table_ GUARDED_BY(mutex_); +}; + +LRUCache::LRUCache() : capacity_(0), usage_(0) { + // Make empty circular linked lists. + lru_.next = &lru_; + lru_.prev = &lru_; + in_use_.next = &in_use_; + in_use_.prev = &in_use_; +} + +LRUCache::~LRUCache() { + assert(in_use_.next == &in_use_); // Error if caller has an unreleased handle + for (LRUHandle* e = lru_.next; e != &lru_;) { + LRUHandle* next = e->next; + assert(e->in_cache); + e->in_cache = false; + assert(e->refs == 1); // Invariant of lru_ list. + Unref(e); + e = next; + } +} + +void LRUCache::Ref(LRUHandle* e) { + if (e->refs == 1 && e->in_cache) { // If on lru_ list, move to in_use_ list. + LRU_Remove(e); + LRU_Append(&in_use_, e); + } + e->refs++; +} + +void LRUCache::Unref(LRUHandle* e) { + assert(e->refs > 0); + e->refs--; + if (e->refs == 0) { // Deallocate. + assert(!e->in_cache); + (*e->deleter)(e->key(), e->value); + free(e); + } else if (e->in_cache && e->refs == 1) { + // No longer in use; move to lru_ list. + LRU_Remove(e); + LRU_Append(&lru_, e); + } +} + +void LRUCache::LRU_Remove(LRUHandle* e) { + e->next->prev = e->prev; + e->prev->next = e->next; +} + +void LRUCache::LRU_Append(LRUHandle* list, LRUHandle* e) { + // Make "e" newest entry by inserting just before *list + e->next = list; + e->prev = list->prev; + e->prev->next = e; + e->next->prev = e; +} + +Cache::Handle* LRUCache::Lookup(const Slice& key, uint32_t hash) { + MutexLock l(&mutex_); + LRUHandle* e = table_.Lookup(key, hash); + if (e != nullptr) { + Ref(e); + } + return reinterpret_cast(e); +} + +void LRUCache::Release(Cache::Handle* handle) { + MutexLock l(&mutex_); + Unref(reinterpret_cast(handle)); +} + +Cache::Handle* LRUCache::Insert(const Slice& key, uint32_t hash, void* value, + size_t charge, + void (*deleter)(const Slice& key, + void* value)) { + MutexLock l(&mutex_); + + LRUHandle* e = + reinterpret_cast(malloc(sizeof(LRUHandle) - 1 + key.size())); + e->value = value; + e->deleter = deleter; + e->charge = charge; + e->key_length = key.size(); + e->hash = hash; + e->in_cache = false; + e->refs = 1; // for the returned handle. + std::memcpy(e->key_data, key.data(), key.size()); + + if (capacity_ > 0) { + e->refs++; // for the cache's reference. + e->in_cache = true; + LRU_Append(&in_use_, e); + usage_ += charge; + FinishErase(table_.Insert(e)); + } else { // don't cache. (capacity_==0 is supported and turns off caching.) + // next is read by key() in an assert, so it must be initialized + e->next = nullptr; + } + while (usage_ > capacity_ && lru_.next != &lru_) { + LRUHandle* old = lru_.next; + assert(old->refs == 1); + bool erased = FinishErase(table_.Remove(old->key(), old->hash)); + if (!erased) { // to avoid unused variable when compiled NDEBUG + assert(erased); + } + } + + return reinterpret_cast(e); +} + +// If e != nullptr, finish removing *e from the cache; it has already been +// removed from the hash table. Return whether e != nullptr. +bool LRUCache::FinishErase(LRUHandle* e) { + if (e != nullptr) { + assert(e->in_cache); + LRU_Remove(e); + e->in_cache = false; + usage_ -= e->charge; + Unref(e); + } + return e != nullptr; +} + +void LRUCache::Erase(const Slice& key, uint32_t hash) { + MutexLock l(&mutex_); + FinishErase(table_.Remove(key, hash)); +} + +void LRUCache::Prune() { + MutexLock l(&mutex_); + while (lru_.next != &lru_) { + LRUHandle* e = lru_.next; + assert(e->refs == 1); + bool erased = FinishErase(table_.Remove(e->key(), e->hash)); + if (!erased) { // to avoid unused variable when compiled NDEBUG + assert(erased); + } + } +} + +static const int kNumShardBits = 4; +static const int kNumShards = 1 << kNumShardBits; + +class ShardedLRUCache : public Cache { + private: + LRUCache shard_[kNumShards]; + port::Mutex id_mutex_; + uint64_t last_id_; + + static inline uint32_t HashSlice(const Slice& s) { + return Hash(s.data(), s.size(), 0); + } + + static uint32_t Shard(uint32_t hash) { return hash >> (32 - kNumShardBits); } + + public: + explicit ShardedLRUCache(size_t capacity) : last_id_(0) { + const size_t per_shard = (capacity + (kNumShards - 1)) / kNumShards; + for (int s = 0; s < kNumShards; s++) { + shard_[s].SetCapacity(per_shard); + } + } + ~ShardedLRUCache() override {} + Handle* Insert(const Slice& key, void* value, size_t charge, + void (*deleter)(const Slice& key, void* value)) override { + const uint32_t hash = HashSlice(key); + return shard_[Shard(hash)].Insert(key, hash, value, charge, deleter); + } + Handle* Lookup(const Slice& key) override { + const uint32_t hash = HashSlice(key); + return shard_[Shard(hash)].Lookup(key, hash); + } + void Release(Handle* handle) override { + LRUHandle* h = reinterpret_cast(handle); + shard_[Shard(h->hash)].Release(handle); + } + void Erase(const Slice& key) override { + const uint32_t hash = HashSlice(key); + shard_[Shard(hash)].Erase(key, hash); + } + void* Value(Handle* handle) override { + return reinterpret_cast(handle)->value; + } + uint64_t NewId() override { + MutexLock l(&id_mutex_); + return ++(last_id_); + } + void Prune() override { + for (int s = 0; s < kNumShards; s++) { + shard_[s].Prune(); + } + } + size_t TotalCharge() const override { + size_t total = 0; + for (int s = 0; s < kNumShards; s++) { + total += shard_[s].TotalCharge(); + } + return total; + } +}; + +} // end anonymous namespace + +Cache* NewLRUCache(size_t capacity) { return new ShardedLRUCache(capacity); } + +} // namespace leveldb diff --git a/leveldb/util/cache_test.cc b/leveldb/util/cache_test.cc new file mode 100644 index 000000000..e68da34bc --- /dev/null +++ b/leveldb/util/cache_test.cc @@ -0,0 +1,224 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/cache.h" + +#include + +#include "gtest/gtest.h" +#include "util/coding.h" + +namespace leveldb { + +// Conversions between numeric keys/values and the types expected by Cache. +static std::string EncodeKey(int k) { + std::string result; + PutFixed32(&result, k); + return result; +} +static int DecodeKey(const Slice& k) { + assert(k.size() == 4); + return DecodeFixed32(k.data()); +} +static void* EncodeValue(uintptr_t v) { return reinterpret_cast(v); } +static int DecodeValue(void* v) { return reinterpret_cast(v); } + +class CacheTest : public testing::Test { + public: + static void Deleter(const Slice& key, void* v) { + current_->deleted_keys_.push_back(DecodeKey(key)); + current_->deleted_values_.push_back(DecodeValue(v)); + } + + static constexpr int kCacheSize = 1000; + std::vector deleted_keys_; + std::vector deleted_values_; + Cache* cache_; + + CacheTest() : cache_(NewLRUCache(kCacheSize)) { current_ = this; } + + ~CacheTest() { delete cache_; } + + int Lookup(int key) { + Cache::Handle* handle = cache_->Lookup(EncodeKey(key)); + const int r = (handle == nullptr) ? -1 : DecodeValue(cache_->Value(handle)); + if (handle != nullptr) { + cache_->Release(handle); + } + return r; + } + + void Insert(int key, int value, int charge = 1) { + cache_->Release(cache_->Insert(EncodeKey(key), EncodeValue(value), charge, + &CacheTest::Deleter)); + } + + Cache::Handle* InsertAndReturnHandle(int key, int value, int charge = 1) { + return cache_->Insert(EncodeKey(key), EncodeValue(value), charge, + &CacheTest::Deleter); + } + + void Erase(int key) { cache_->Erase(EncodeKey(key)); } + static CacheTest* current_; +}; +CacheTest* CacheTest::current_; + +TEST_F(CacheTest, HitAndMiss) { + ASSERT_EQ(-1, Lookup(100)); + + Insert(100, 101); + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(-1, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + Insert(200, 201); + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + Insert(100, 102); + ASSERT_EQ(102, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + ASSERT_EQ(1, deleted_keys_.size()); + ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(101, deleted_values_[0]); +} + +TEST_F(CacheTest, Erase) { + Erase(200); + ASSERT_EQ(0, deleted_keys_.size()); + + Insert(100, 101); + Insert(200, 201); + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(1, deleted_keys_.size()); + ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(101, deleted_values_[0]); + + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(1, deleted_keys_.size()); +} + +TEST_F(CacheTest, EntriesArePinned) { + Insert(100, 101); + Cache::Handle* h1 = cache_->Lookup(EncodeKey(100)); + ASSERT_EQ(101, DecodeValue(cache_->Value(h1))); + + Insert(100, 102); + Cache::Handle* h2 = cache_->Lookup(EncodeKey(100)); + ASSERT_EQ(102, DecodeValue(cache_->Value(h2))); + ASSERT_EQ(0, deleted_keys_.size()); + + cache_->Release(h1); + ASSERT_EQ(1, deleted_keys_.size()); + ASSERT_EQ(100, deleted_keys_[0]); + ASSERT_EQ(101, deleted_values_[0]); + + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(1, deleted_keys_.size()); + + cache_->Release(h2); + ASSERT_EQ(2, deleted_keys_.size()); + ASSERT_EQ(100, deleted_keys_[1]); + ASSERT_EQ(102, deleted_values_[1]); +} + +TEST_F(CacheTest, EvictionPolicy) { + Insert(100, 101); + Insert(200, 201); + Insert(300, 301); + Cache::Handle* h = cache_->Lookup(EncodeKey(300)); + + // Frequently used entry must be kept around, + // as must things that are still in use. + for (int i = 0; i < kCacheSize + 100; i++) { + Insert(1000 + i, 2000 + i); + ASSERT_EQ(2000 + i, Lookup(1000 + i)); + ASSERT_EQ(101, Lookup(100)); + } + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(-1, Lookup(200)); + ASSERT_EQ(301, Lookup(300)); + cache_->Release(h); +} + +TEST_F(CacheTest, UseExceedsCacheSize) { + // Overfill the cache, keeping handles on all inserted entries. + std::vector h; + for (int i = 0; i < kCacheSize + 100; i++) { + h.push_back(InsertAndReturnHandle(1000 + i, 2000 + i)); + } + + // Check that all the entries can be found in the cache. + for (int i = 0; i < h.size(); i++) { + ASSERT_EQ(2000 + i, Lookup(1000 + i)); + } + + for (int i = 0; i < h.size(); i++) { + cache_->Release(h[i]); + } +} + +TEST_F(CacheTest, HeavyEntries) { + // Add a bunch of light and heavy entries and then count the combined + // size of items still in the cache, which must be approximately the + // same as the total capacity. + const int kLight = 1; + const int kHeavy = 10; + int added = 0; + int index = 0; + while (added < 2 * kCacheSize) { + const int weight = (index & 1) ? kLight : kHeavy; + Insert(index, 1000 + index, weight); + added += weight; + index++; + } + + int cached_weight = 0; + for (int i = 0; i < index; i++) { + const int weight = (i & 1 ? kLight : kHeavy); + int r = Lookup(i); + if (r >= 0) { + cached_weight += weight; + ASSERT_EQ(1000 + i, r); + } + } + ASSERT_LE(cached_weight, kCacheSize + kCacheSize / 10); +} + +TEST_F(CacheTest, NewId) { + uint64_t a = cache_->NewId(); + uint64_t b = cache_->NewId(); + ASSERT_NE(a, b); +} + +TEST_F(CacheTest, Prune) { + Insert(1, 100); + Insert(2, 200); + + Cache::Handle* handle = cache_->Lookup(EncodeKey(1)); + ASSERT_TRUE(handle); + cache_->Prune(); + cache_->Release(handle); + + ASSERT_EQ(100, Lookup(1)); + ASSERT_EQ(-1, Lookup(2)); +} + +TEST_F(CacheTest, ZeroSizeCache) { + delete cache_; + cache_ = NewLRUCache(0); + + Insert(1, 100); + ASSERT_EQ(-1, Lookup(1)); +} + +} // namespace leveldb diff --git a/leveldb/util/coding.cc b/leveldb/util/coding.cc new file mode 100644 index 000000000..a8f8af8be --- /dev/null +++ b/leveldb/util/coding.cc @@ -0,0 +1,156 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/coding.h" + +namespace leveldb { + +void PutFixed32(std::string* dst, uint32_t value) { + char buf[sizeof(value)]; + EncodeFixed32(buf, value); + dst->append(buf, sizeof(buf)); +} + +void PutFixed64(std::string* dst, uint64_t value) { + char buf[sizeof(value)]; + EncodeFixed64(buf, value); + dst->append(buf, sizeof(buf)); +} + +char* EncodeVarint32(char* dst, uint32_t v) { + // Operate on characters as unsigneds + uint8_t* ptr = reinterpret_cast(dst); + static const int B = 128; + if (v < (1 << 7)) { + *(ptr++) = v; + } else if (v < (1 << 14)) { + *(ptr++) = v | B; + *(ptr++) = v >> 7; + } else if (v < (1 << 21)) { + *(ptr++) = v | B; + *(ptr++) = (v >> 7) | B; + *(ptr++) = v >> 14; + } else if (v < (1 << 28)) { + *(ptr++) = v | B; + *(ptr++) = (v >> 7) | B; + *(ptr++) = (v >> 14) | B; + *(ptr++) = v >> 21; + } else { + *(ptr++) = v | B; + *(ptr++) = (v >> 7) | B; + *(ptr++) = (v >> 14) | B; + *(ptr++) = (v >> 21) | B; + *(ptr++) = v >> 28; + } + return reinterpret_cast(ptr); +} + +void PutVarint32(std::string* dst, uint32_t v) { + char buf[5]; + char* ptr = EncodeVarint32(buf, v); + dst->append(buf, ptr - buf); +} + +char* EncodeVarint64(char* dst, uint64_t v) { + static const int B = 128; + uint8_t* ptr = reinterpret_cast(dst); + while (v >= B) { + *(ptr++) = v | B; + v >>= 7; + } + *(ptr++) = static_cast(v); + return reinterpret_cast(ptr); +} + +void PutVarint64(std::string* dst, uint64_t v) { + char buf[10]; + char* ptr = EncodeVarint64(buf, v); + dst->append(buf, ptr - buf); +} + +void PutLengthPrefixedSlice(std::string* dst, const Slice& value) { + PutVarint32(dst, value.size()); + dst->append(value.data(), value.size()); +} + +int VarintLength(uint64_t v) { + int len = 1; + while (v >= 128) { + v >>= 7; + len++; + } + return len; +} + +const char* GetVarint32PtrFallback(const char* p, const char* limit, + uint32_t* value) { + uint32_t result = 0; + for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) { + uint32_t byte = *(reinterpret_cast(p)); + p++; + if (byte & 128) { + // More bytes are present + result |= ((byte & 127) << shift); + } else { + result |= (byte << shift); + *value = result; + return reinterpret_cast(p); + } + } + return nullptr; +} + +bool GetVarint32(Slice* input, uint32_t* value) { + const char* p = input->data(); + const char* limit = p + input->size(); + const char* q = GetVarint32Ptr(p, limit, value); + if (q == nullptr) { + return false; + } else { + *input = Slice(q, limit - q); + return true; + } +} + +const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) { + uint64_t result = 0; + for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) { + uint64_t byte = *(reinterpret_cast(p)); + p++; + if (byte & 128) { + // More bytes are present + result |= ((byte & 127) << shift); + } else { + result |= (byte << shift); + *value = result; + return reinterpret_cast(p); + } + } + return nullptr; +} + +bool GetVarint64(Slice* input, uint64_t* value) { + const char* p = input->data(); + const char* limit = p + input->size(); + const char* q = GetVarint64Ptr(p, limit, value); + if (q == nullptr) { + return false; + } else { + *input = Slice(q, limit - q); + return true; + } +} + +bool GetLengthPrefixedSlice(Slice* input, Slice* result) { + uint32_t len; + if (GetVarint32(input, &len) && input->size() >= len) { + *result = Slice(input->data(), len); + input->remove_prefix(len); + return true; + } else { + return false; + } +} + +} // namespace leveldb diff --git a/leveldb/util/coding.h b/leveldb/util/coding.h new file mode 100644 index 000000000..f0bb57b8e --- /dev/null +++ b/leveldb/util/coding.h @@ -0,0 +1,122 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Endian-neutral encoding: +// * Fixed-length numbers are encoded with least-significant byte first +// * In addition we support variable length "varint" encoding +// * Strings are encoded prefixed by their length in varint format + +#ifndef STORAGE_LEVELDB_UTIL_CODING_H_ +#define STORAGE_LEVELDB_UTIL_CODING_H_ + +#include +#include +#include + +#include "leveldb/slice.h" +#include "port/port.h" + +namespace leveldb { + +// Standard Put... routines append to a string +void PutFixed32(std::string* dst, uint32_t value); +void PutFixed64(std::string* dst, uint64_t value); +void PutVarint32(std::string* dst, uint32_t value); +void PutVarint64(std::string* dst, uint64_t value); +void PutLengthPrefixedSlice(std::string* dst, const Slice& value); + +// Standard Get... routines parse a value from the beginning of a Slice +// and advance the slice past the parsed value. +bool GetVarint32(Slice* input, uint32_t* value); +bool GetVarint64(Slice* input, uint64_t* value); +bool GetLengthPrefixedSlice(Slice* input, Slice* result); + +// Pointer-based variants of GetVarint... These either store a value +// in *v and return a pointer just past the parsed value, or return +// nullptr on error. These routines only look at bytes in the range +// [p..limit-1] +const char* GetVarint32Ptr(const char* p, const char* limit, uint32_t* v); +const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* v); + +// Returns the length of the varint32 or varint64 encoding of "v" +int VarintLength(uint64_t v); + +// Lower-level versions of Put... that write directly into a character buffer +// and return a pointer just past the last byte written. +// REQUIRES: dst has enough space for the value being written +char* EncodeVarint32(char* dst, uint32_t value); +char* EncodeVarint64(char* dst, uint64_t value); + +// Lower-level versions of Put... that write directly into a character buffer +// REQUIRES: dst has enough space for the value being written + +inline void EncodeFixed32(char* dst, uint32_t value) { + uint8_t* const buffer = reinterpret_cast(dst); + + // Recent clang and gcc optimize this to a single mov / str instruction. + buffer[0] = static_cast(value); + buffer[1] = static_cast(value >> 8); + buffer[2] = static_cast(value >> 16); + buffer[3] = static_cast(value >> 24); +} + +inline void EncodeFixed64(char* dst, uint64_t value) { + uint8_t* const buffer = reinterpret_cast(dst); + + // Recent clang and gcc optimize this to a single mov / str instruction. + buffer[0] = static_cast(value); + buffer[1] = static_cast(value >> 8); + buffer[2] = static_cast(value >> 16); + buffer[3] = static_cast(value >> 24); + buffer[4] = static_cast(value >> 32); + buffer[5] = static_cast(value >> 40); + buffer[6] = static_cast(value >> 48); + buffer[7] = static_cast(value >> 56); +} + +// Lower-level versions of Get... that read directly from a character buffer +// without any bounds checking. + +inline uint32_t DecodeFixed32(const char* ptr) { + const uint8_t* const buffer = reinterpret_cast(ptr); + + // Recent clang and gcc optimize this to a single mov / ldr instruction. + return (static_cast(buffer[0])) | + (static_cast(buffer[1]) << 8) | + (static_cast(buffer[2]) << 16) | + (static_cast(buffer[3]) << 24); +} + +inline uint64_t DecodeFixed64(const char* ptr) { + const uint8_t* const buffer = reinterpret_cast(ptr); + + // Recent clang and gcc optimize this to a single mov / ldr instruction. + return (static_cast(buffer[0])) | + (static_cast(buffer[1]) << 8) | + (static_cast(buffer[2]) << 16) | + (static_cast(buffer[3]) << 24) | + (static_cast(buffer[4]) << 32) | + (static_cast(buffer[5]) << 40) | + (static_cast(buffer[6]) << 48) | + (static_cast(buffer[7]) << 56); +} + +// Internal routine for use by fallback path of GetVarint32Ptr +const char* GetVarint32PtrFallback(const char* p, const char* limit, + uint32_t* value); +inline const char* GetVarint32Ptr(const char* p, const char* limit, + uint32_t* value) { + if (p < limit) { + uint32_t result = *(reinterpret_cast(p)); + if ((result & 128) == 0) { + *value = result; + return p + 1; + } + } + return GetVarint32PtrFallback(p, limit, value); +} + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_CODING_H_ diff --git a/leveldb/util/coding_test.cc b/leveldb/util/coding_test.cc new file mode 100644 index 000000000..cceda1471 --- /dev/null +++ b/leveldb/util/coding_test.cc @@ -0,0 +1,193 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/coding.h" + +#include + +#include "gtest/gtest.h" + +namespace leveldb { + +TEST(Coding, Fixed32) { + std::string s; + for (uint32_t v = 0; v < 100000; v++) { + PutFixed32(&s, v); + } + + const char* p = s.data(); + for (uint32_t v = 0; v < 100000; v++) { + uint32_t actual = DecodeFixed32(p); + ASSERT_EQ(v, actual); + p += sizeof(uint32_t); + } +} + +TEST(Coding, Fixed64) { + std::string s; + for (int power = 0; power <= 63; power++) { + uint64_t v = static_cast(1) << power; + PutFixed64(&s, v - 1); + PutFixed64(&s, v + 0); + PutFixed64(&s, v + 1); + } + + const char* p = s.data(); + for (int power = 0; power <= 63; power++) { + uint64_t v = static_cast(1) << power; + uint64_t actual; + actual = DecodeFixed64(p); + ASSERT_EQ(v - 1, actual); + p += sizeof(uint64_t); + + actual = DecodeFixed64(p); + ASSERT_EQ(v + 0, actual); + p += sizeof(uint64_t); + + actual = DecodeFixed64(p); + ASSERT_EQ(v + 1, actual); + p += sizeof(uint64_t); + } +} + +// Test that encoding routines generate little-endian encodings +TEST(Coding, EncodingOutput) { + std::string dst; + PutFixed32(&dst, 0x04030201); + ASSERT_EQ(4, dst.size()); + ASSERT_EQ(0x01, static_cast(dst[0])); + ASSERT_EQ(0x02, static_cast(dst[1])); + ASSERT_EQ(0x03, static_cast(dst[2])); + ASSERT_EQ(0x04, static_cast(dst[3])); + + dst.clear(); + PutFixed64(&dst, 0x0807060504030201ull); + ASSERT_EQ(8, dst.size()); + ASSERT_EQ(0x01, static_cast(dst[0])); + ASSERT_EQ(0x02, static_cast(dst[1])); + ASSERT_EQ(0x03, static_cast(dst[2])); + ASSERT_EQ(0x04, static_cast(dst[3])); + ASSERT_EQ(0x05, static_cast(dst[4])); + ASSERT_EQ(0x06, static_cast(dst[5])); + ASSERT_EQ(0x07, static_cast(dst[6])); + ASSERT_EQ(0x08, static_cast(dst[7])); +} + +TEST(Coding, Varint32) { + std::string s; + for (uint32_t i = 0; i < (32 * 32); i++) { + uint32_t v = (i / 32) << (i % 32); + PutVarint32(&s, v); + } + + const char* p = s.data(); + const char* limit = p + s.size(); + for (uint32_t i = 0; i < (32 * 32); i++) { + uint32_t expected = (i / 32) << (i % 32); + uint32_t actual; + const char* start = p; + p = GetVarint32Ptr(p, limit, &actual); + ASSERT_TRUE(p != nullptr); + ASSERT_EQ(expected, actual); + ASSERT_EQ(VarintLength(actual), p - start); + } + ASSERT_EQ(p, s.data() + s.size()); +} + +TEST(Coding, Varint64) { + // Construct the list of values to check + std::vector values; + // Some special values + values.push_back(0); + values.push_back(100); + values.push_back(~static_cast(0)); + values.push_back(~static_cast(0) - 1); + for (uint32_t k = 0; k < 64; k++) { + // Test values near powers of two + const uint64_t power = 1ull << k; + values.push_back(power); + values.push_back(power - 1); + values.push_back(power + 1); + } + + std::string s; + for (size_t i = 0; i < values.size(); i++) { + PutVarint64(&s, values[i]); + } + + const char* p = s.data(); + const char* limit = p + s.size(); + for (size_t i = 0; i < values.size(); i++) { + ASSERT_TRUE(p < limit); + uint64_t actual; + const char* start = p; + p = GetVarint64Ptr(p, limit, &actual); + ASSERT_TRUE(p != nullptr); + ASSERT_EQ(values[i], actual); + ASSERT_EQ(VarintLength(actual), p - start); + } + ASSERT_EQ(p, limit); +} + +TEST(Coding, Varint32Overflow) { + uint32_t result; + std::string input("\x81\x82\x83\x84\x85\x11"); + ASSERT_TRUE(GetVarint32Ptr(input.data(), input.data() + input.size(), + &result) == nullptr); +} + +TEST(Coding, Varint32Truncation) { + uint32_t large_value = (1u << 31) + 100; + std::string s; + PutVarint32(&s, large_value); + uint32_t result; + for (size_t len = 0; len < s.size() - 1; len++) { + ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + len, &result) == nullptr); + } + ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + s.size(), &result) != + nullptr); + ASSERT_EQ(large_value, result); +} + +TEST(Coding, Varint64Overflow) { + uint64_t result; + std::string input("\x81\x82\x83\x84\x85\x81\x82\x83\x84\x85\x11"); + ASSERT_TRUE(GetVarint64Ptr(input.data(), input.data() + input.size(), + &result) == nullptr); +} + +TEST(Coding, Varint64Truncation) { + uint64_t large_value = (1ull << 63) + 100ull; + std::string s; + PutVarint64(&s, large_value); + uint64_t result; + for (size_t len = 0; len < s.size() - 1; len++) { + ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + len, &result) == nullptr); + } + ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + s.size(), &result) != + nullptr); + ASSERT_EQ(large_value, result); +} + +TEST(Coding, Strings) { + std::string s; + PutLengthPrefixedSlice(&s, Slice("")); + PutLengthPrefixedSlice(&s, Slice("foo")); + PutLengthPrefixedSlice(&s, Slice("bar")); + PutLengthPrefixedSlice(&s, Slice(std::string(200, 'x'))); + + Slice input(s); + Slice v; + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("foo", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("bar", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ(std::string(200, 'x'), v.ToString()); + ASSERT_EQ("", input.ToString()); +} + +} // namespace leveldb diff --git a/leveldb/util/comparator.cc b/leveldb/util/comparator.cc new file mode 100644 index 000000000..c5766e946 --- /dev/null +++ b/leveldb/util/comparator.cc @@ -0,0 +1,75 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/comparator.h" + +#include +#include +#include +#include + +#include "leveldb/slice.h" +#include "util/logging.h" +#include "util/no_destructor.h" + +namespace leveldb { + +Comparator::~Comparator() = default; + +namespace { +class BytewiseComparatorImpl : public Comparator { + public: + BytewiseComparatorImpl() = default; + + const char* Name() const override { return "leveldb.BytewiseComparator"; } + + int Compare(const Slice& a, const Slice& b) const override { + return a.compare(b); + } + + void FindShortestSeparator(std::string* start, + const Slice& limit) const override { + // Find length of common prefix + size_t min_length = std::min(start->size(), limit.size()); + size_t diff_index = 0; + while ((diff_index < min_length) && + ((*start)[diff_index] == limit[diff_index])) { + diff_index++; + } + + if (diff_index >= min_length) { + // Do not shorten if one string is a prefix of the other + } else { + uint8_t diff_byte = static_cast((*start)[diff_index]); + if (diff_byte < static_cast(0xff) && + diff_byte + 1 < static_cast(limit[diff_index])) { + (*start)[diff_index]++; + start->resize(diff_index + 1); + assert(Compare(*start, limit) < 0); + } + } + } + + void FindShortSuccessor(std::string* key) const override { + // Find first character that can be incremented + size_t n = key->size(); + for (size_t i = 0; i < n; i++) { + const uint8_t byte = (*key)[i]; + if (byte != static_cast(0xff)) { + (*key)[i] = byte + 1; + key->resize(i + 1); + return; + } + } + // *key is a run of 0xffs. Leave it alone. + } +}; +} // namespace + +const Comparator* BytewiseComparator() { + static NoDestructor singleton; + return singleton.get(); +} + +} // namespace leveldb diff --git a/leveldb/util/crc32c.cc b/leveldb/util/crc32c.cc new file mode 100644 index 000000000..3f18908c4 --- /dev/null +++ b/leveldb/util/crc32c.cc @@ -0,0 +1,380 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A portable implementation of crc32c. + +#include "util/crc32c.h" + +#include +#include + +#include "port/port.h" +#include "util/coding.h" + +namespace leveldb { +namespace crc32c { + +namespace { + +const uint32_t kByteExtensionTable[256] = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, + 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, + 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, + 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, + 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, + 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, + 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, + 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, + 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, + 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, + 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, + 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, + 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, + 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, + 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, + 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, + 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, + 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, + 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, + 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, + 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, + 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351}; + +const uint32_t kStrideExtensionTable0[256] = { + 0x00000000, 0x30d23865, 0x61a470ca, 0x517648af, 0xc348e194, 0xf39ad9f1, + 0xa2ec915e, 0x923ea93b, 0x837db5d9, 0xb3af8dbc, 0xe2d9c513, 0xd20bfd76, + 0x4035544d, 0x70e76c28, 0x21912487, 0x11431ce2, 0x03171d43, 0x33c52526, + 0x62b36d89, 0x526155ec, 0xc05ffcd7, 0xf08dc4b2, 0xa1fb8c1d, 0x9129b478, + 0x806aa89a, 0xb0b890ff, 0xe1ced850, 0xd11ce035, 0x4322490e, 0x73f0716b, + 0x228639c4, 0x125401a1, 0x062e3a86, 0x36fc02e3, 0x678a4a4c, 0x57587229, + 0xc566db12, 0xf5b4e377, 0xa4c2abd8, 0x941093bd, 0x85538f5f, 0xb581b73a, + 0xe4f7ff95, 0xd425c7f0, 0x461b6ecb, 0x76c956ae, 0x27bf1e01, 0x176d2664, + 0x053927c5, 0x35eb1fa0, 0x649d570f, 0x544f6f6a, 0xc671c651, 0xf6a3fe34, + 0xa7d5b69b, 0x97078efe, 0x8644921c, 0xb696aa79, 0xe7e0e2d6, 0xd732dab3, + 0x450c7388, 0x75de4bed, 0x24a80342, 0x147a3b27, 0x0c5c750c, 0x3c8e4d69, + 0x6df805c6, 0x5d2a3da3, 0xcf149498, 0xffc6acfd, 0xaeb0e452, 0x9e62dc37, + 0x8f21c0d5, 0xbff3f8b0, 0xee85b01f, 0xde57887a, 0x4c692141, 0x7cbb1924, + 0x2dcd518b, 0x1d1f69ee, 0x0f4b684f, 0x3f99502a, 0x6eef1885, 0x5e3d20e0, + 0xcc0389db, 0xfcd1b1be, 0xada7f911, 0x9d75c174, 0x8c36dd96, 0xbce4e5f3, + 0xed92ad5c, 0xdd409539, 0x4f7e3c02, 0x7fac0467, 0x2eda4cc8, 0x1e0874ad, + 0x0a724f8a, 0x3aa077ef, 0x6bd63f40, 0x5b040725, 0xc93aae1e, 0xf9e8967b, + 0xa89eded4, 0x984ce6b1, 0x890ffa53, 0xb9ddc236, 0xe8ab8a99, 0xd879b2fc, + 0x4a471bc7, 0x7a9523a2, 0x2be36b0d, 0x1b315368, 0x096552c9, 0x39b76aac, + 0x68c12203, 0x58131a66, 0xca2db35d, 0xfaff8b38, 0xab89c397, 0x9b5bfbf2, + 0x8a18e710, 0xbacadf75, 0xebbc97da, 0xdb6eafbf, 0x49500684, 0x79823ee1, + 0x28f4764e, 0x18264e2b, 0x18b8ea18, 0x286ad27d, 0x791c9ad2, 0x49cea2b7, + 0xdbf00b8c, 0xeb2233e9, 0xba547b46, 0x8a864323, 0x9bc55fc1, 0xab1767a4, + 0xfa612f0b, 0xcab3176e, 0x588dbe55, 0x685f8630, 0x3929ce9f, 0x09fbf6fa, + 0x1baff75b, 0x2b7dcf3e, 0x7a0b8791, 0x4ad9bff4, 0xd8e716cf, 0xe8352eaa, + 0xb9436605, 0x89915e60, 0x98d24282, 0xa8007ae7, 0xf9763248, 0xc9a40a2d, + 0x5b9aa316, 0x6b489b73, 0x3a3ed3dc, 0x0aecebb9, 0x1e96d09e, 0x2e44e8fb, + 0x7f32a054, 0x4fe09831, 0xddde310a, 0xed0c096f, 0xbc7a41c0, 0x8ca879a5, + 0x9deb6547, 0xad395d22, 0xfc4f158d, 0xcc9d2de8, 0x5ea384d3, 0x6e71bcb6, + 0x3f07f419, 0x0fd5cc7c, 0x1d81cddd, 0x2d53f5b8, 0x7c25bd17, 0x4cf78572, + 0xdec92c49, 0xee1b142c, 0xbf6d5c83, 0x8fbf64e6, 0x9efc7804, 0xae2e4061, + 0xff5808ce, 0xcf8a30ab, 0x5db49990, 0x6d66a1f5, 0x3c10e95a, 0x0cc2d13f, + 0x14e49f14, 0x2436a771, 0x7540efde, 0x4592d7bb, 0xd7ac7e80, 0xe77e46e5, + 0xb6080e4a, 0x86da362f, 0x97992acd, 0xa74b12a8, 0xf63d5a07, 0xc6ef6262, + 0x54d1cb59, 0x6403f33c, 0x3575bb93, 0x05a783f6, 0x17f38257, 0x2721ba32, + 0x7657f29d, 0x4685caf8, 0xd4bb63c3, 0xe4695ba6, 0xb51f1309, 0x85cd2b6c, + 0x948e378e, 0xa45c0feb, 0xf52a4744, 0xc5f87f21, 0x57c6d61a, 0x6714ee7f, + 0x3662a6d0, 0x06b09eb5, 0x12caa592, 0x22189df7, 0x736ed558, 0x43bced3d, + 0xd1824406, 0xe1507c63, 0xb02634cc, 0x80f40ca9, 0x91b7104b, 0xa165282e, + 0xf0136081, 0xc0c158e4, 0x52fff1df, 0x622dc9ba, 0x335b8115, 0x0389b970, + 0x11ddb8d1, 0x210f80b4, 0x7079c81b, 0x40abf07e, 0xd2955945, 0xe2476120, + 0xb331298f, 0x83e311ea, 0x92a00d08, 0xa272356d, 0xf3047dc2, 0xc3d645a7, + 0x51e8ec9c, 0x613ad4f9, 0x304c9c56, 0x009ea433}; + +const uint32_t kStrideExtensionTable1[256] = { + 0x00000000, 0x54075546, 0xa80eaa8c, 0xfc09ffca, 0x55f123e9, 0x01f676af, + 0xfdff8965, 0xa9f8dc23, 0xabe247d2, 0xffe51294, 0x03eced5e, 0x57ebb818, + 0xfe13643b, 0xaa14317d, 0x561dceb7, 0x021a9bf1, 0x5228f955, 0x062fac13, + 0xfa2653d9, 0xae21069f, 0x07d9dabc, 0x53de8ffa, 0xafd77030, 0xfbd02576, + 0xf9cabe87, 0xadcdebc1, 0x51c4140b, 0x05c3414d, 0xac3b9d6e, 0xf83cc828, + 0x043537e2, 0x503262a4, 0xa451f2aa, 0xf056a7ec, 0x0c5f5826, 0x58580d60, + 0xf1a0d143, 0xa5a78405, 0x59ae7bcf, 0x0da92e89, 0x0fb3b578, 0x5bb4e03e, + 0xa7bd1ff4, 0xf3ba4ab2, 0x5a429691, 0x0e45c3d7, 0xf24c3c1d, 0xa64b695b, + 0xf6790bff, 0xa27e5eb9, 0x5e77a173, 0x0a70f435, 0xa3882816, 0xf78f7d50, + 0x0b86829a, 0x5f81d7dc, 0x5d9b4c2d, 0x099c196b, 0xf595e6a1, 0xa192b3e7, + 0x086a6fc4, 0x5c6d3a82, 0xa064c548, 0xf463900e, 0x4d4f93a5, 0x1948c6e3, + 0xe5413929, 0xb1466c6f, 0x18beb04c, 0x4cb9e50a, 0xb0b01ac0, 0xe4b74f86, + 0xe6add477, 0xb2aa8131, 0x4ea37efb, 0x1aa42bbd, 0xb35cf79e, 0xe75ba2d8, + 0x1b525d12, 0x4f550854, 0x1f676af0, 0x4b603fb6, 0xb769c07c, 0xe36e953a, + 0x4a964919, 0x1e911c5f, 0xe298e395, 0xb69fb6d3, 0xb4852d22, 0xe0827864, + 0x1c8b87ae, 0x488cd2e8, 0xe1740ecb, 0xb5735b8d, 0x497aa447, 0x1d7df101, + 0xe91e610f, 0xbd193449, 0x4110cb83, 0x15179ec5, 0xbcef42e6, 0xe8e817a0, + 0x14e1e86a, 0x40e6bd2c, 0x42fc26dd, 0x16fb739b, 0xeaf28c51, 0xbef5d917, + 0x170d0534, 0x430a5072, 0xbf03afb8, 0xeb04fafe, 0xbb36985a, 0xef31cd1c, + 0x133832d6, 0x473f6790, 0xeec7bbb3, 0xbac0eef5, 0x46c9113f, 0x12ce4479, + 0x10d4df88, 0x44d38ace, 0xb8da7504, 0xecdd2042, 0x4525fc61, 0x1122a927, + 0xed2b56ed, 0xb92c03ab, 0x9a9f274a, 0xce98720c, 0x32918dc6, 0x6696d880, + 0xcf6e04a3, 0x9b6951e5, 0x6760ae2f, 0x3367fb69, 0x317d6098, 0x657a35de, + 0x9973ca14, 0xcd749f52, 0x648c4371, 0x308b1637, 0xcc82e9fd, 0x9885bcbb, + 0xc8b7de1f, 0x9cb08b59, 0x60b97493, 0x34be21d5, 0x9d46fdf6, 0xc941a8b0, + 0x3548577a, 0x614f023c, 0x635599cd, 0x3752cc8b, 0xcb5b3341, 0x9f5c6607, + 0x36a4ba24, 0x62a3ef62, 0x9eaa10a8, 0xcaad45ee, 0x3eced5e0, 0x6ac980a6, + 0x96c07f6c, 0xc2c72a2a, 0x6b3ff609, 0x3f38a34f, 0xc3315c85, 0x973609c3, + 0x952c9232, 0xc12bc774, 0x3d2238be, 0x69256df8, 0xc0ddb1db, 0x94dae49d, + 0x68d31b57, 0x3cd44e11, 0x6ce62cb5, 0x38e179f3, 0xc4e88639, 0x90efd37f, + 0x39170f5c, 0x6d105a1a, 0x9119a5d0, 0xc51ef096, 0xc7046b67, 0x93033e21, + 0x6f0ac1eb, 0x3b0d94ad, 0x92f5488e, 0xc6f21dc8, 0x3afbe202, 0x6efcb744, + 0xd7d0b4ef, 0x83d7e1a9, 0x7fde1e63, 0x2bd94b25, 0x82219706, 0xd626c240, + 0x2a2f3d8a, 0x7e2868cc, 0x7c32f33d, 0x2835a67b, 0xd43c59b1, 0x803b0cf7, + 0x29c3d0d4, 0x7dc48592, 0x81cd7a58, 0xd5ca2f1e, 0x85f84dba, 0xd1ff18fc, + 0x2df6e736, 0x79f1b270, 0xd0096e53, 0x840e3b15, 0x7807c4df, 0x2c009199, + 0x2e1a0a68, 0x7a1d5f2e, 0x8614a0e4, 0xd213f5a2, 0x7beb2981, 0x2fec7cc7, + 0xd3e5830d, 0x87e2d64b, 0x73814645, 0x27861303, 0xdb8fecc9, 0x8f88b98f, + 0x267065ac, 0x727730ea, 0x8e7ecf20, 0xda799a66, 0xd8630197, 0x8c6454d1, + 0x706dab1b, 0x246afe5d, 0x8d92227e, 0xd9957738, 0x259c88f2, 0x719bddb4, + 0x21a9bf10, 0x75aeea56, 0x89a7159c, 0xdda040da, 0x74589cf9, 0x205fc9bf, + 0xdc563675, 0x88516333, 0x8a4bf8c2, 0xde4cad84, 0x2245524e, 0x76420708, + 0xdfbadb2b, 0x8bbd8e6d, 0x77b471a7, 0x23b324e1}; + +const uint32_t kStrideExtensionTable2[256] = { + 0x00000000, 0x678efd01, 0xcf1dfa02, 0xa8930703, 0x9bd782f5, 0xfc597ff4, + 0x54ca78f7, 0x334485f6, 0x3243731b, 0x55cd8e1a, 0xfd5e8919, 0x9ad07418, + 0xa994f1ee, 0xce1a0cef, 0x66890bec, 0x0107f6ed, 0x6486e636, 0x03081b37, + 0xab9b1c34, 0xcc15e135, 0xff5164c3, 0x98df99c2, 0x304c9ec1, 0x57c263c0, + 0x56c5952d, 0x314b682c, 0x99d86f2f, 0xfe56922e, 0xcd1217d8, 0xaa9cead9, + 0x020fedda, 0x658110db, 0xc90dcc6c, 0xae83316d, 0x0610366e, 0x619ecb6f, + 0x52da4e99, 0x3554b398, 0x9dc7b49b, 0xfa49499a, 0xfb4ebf77, 0x9cc04276, + 0x34534575, 0x53ddb874, 0x60993d82, 0x0717c083, 0xaf84c780, 0xc80a3a81, + 0xad8b2a5a, 0xca05d75b, 0x6296d058, 0x05182d59, 0x365ca8af, 0x51d255ae, + 0xf94152ad, 0x9ecfafac, 0x9fc85941, 0xf846a440, 0x50d5a343, 0x375b5e42, + 0x041fdbb4, 0x639126b5, 0xcb0221b6, 0xac8cdcb7, 0x97f7ee29, 0xf0791328, + 0x58ea142b, 0x3f64e92a, 0x0c206cdc, 0x6bae91dd, 0xc33d96de, 0xa4b36bdf, + 0xa5b49d32, 0xc23a6033, 0x6aa96730, 0x0d279a31, 0x3e631fc7, 0x59ede2c6, + 0xf17ee5c5, 0x96f018c4, 0xf371081f, 0x94fff51e, 0x3c6cf21d, 0x5be20f1c, + 0x68a68aea, 0x0f2877eb, 0xa7bb70e8, 0xc0358de9, 0xc1327b04, 0xa6bc8605, + 0x0e2f8106, 0x69a17c07, 0x5ae5f9f1, 0x3d6b04f0, 0x95f803f3, 0xf276fef2, + 0x5efa2245, 0x3974df44, 0x91e7d847, 0xf6692546, 0xc52da0b0, 0xa2a35db1, + 0x0a305ab2, 0x6dbea7b3, 0x6cb9515e, 0x0b37ac5f, 0xa3a4ab5c, 0xc42a565d, + 0xf76ed3ab, 0x90e02eaa, 0x387329a9, 0x5ffdd4a8, 0x3a7cc473, 0x5df23972, + 0xf5613e71, 0x92efc370, 0xa1ab4686, 0xc625bb87, 0x6eb6bc84, 0x09384185, + 0x083fb768, 0x6fb14a69, 0xc7224d6a, 0xa0acb06b, 0x93e8359d, 0xf466c89c, + 0x5cf5cf9f, 0x3b7b329e, 0x2a03aaa3, 0x4d8d57a2, 0xe51e50a1, 0x8290ada0, + 0xb1d42856, 0xd65ad557, 0x7ec9d254, 0x19472f55, 0x1840d9b8, 0x7fce24b9, + 0xd75d23ba, 0xb0d3debb, 0x83975b4d, 0xe419a64c, 0x4c8aa14f, 0x2b045c4e, + 0x4e854c95, 0x290bb194, 0x8198b697, 0xe6164b96, 0xd552ce60, 0xb2dc3361, + 0x1a4f3462, 0x7dc1c963, 0x7cc63f8e, 0x1b48c28f, 0xb3dbc58c, 0xd455388d, + 0xe711bd7b, 0x809f407a, 0x280c4779, 0x4f82ba78, 0xe30e66cf, 0x84809bce, + 0x2c139ccd, 0x4b9d61cc, 0x78d9e43a, 0x1f57193b, 0xb7c41e38, 0xd04ae339, + 0xd14d15d4, 0xb6c3e8d5, 0x1e50efd6, 0x79de12d7, 0x4a9a9721, 0x2d146a20, + 0x85876d23, 0xe2099022, 0x878880f9, 0xe0067df8, 0x48957afb, 0x2f1b87fa, + 0x1c5f020c, 0x7bd1ff0d, 0xd342f80e, 0xb4cc050f, 0xb5cbf3e2, 0xd2450ee3, + 0x7ad609e0, 0x1d58f4e1, 0x2e1c7117, 0x49928c16, 0xe1018b15, 0x868f7614, + 0xbdf4448a, 0xda7ab98b, 0x72e9be88, 0x15674389, 0x2623c67f, 0x41ad3b7e, + 0xe93e3c7d, 0x8eb0c17c, 0x8fb73791, 0xe839ca90, 0x40aacd93, 0x27243092, + 0x1460b564, 0x73ee4865, 0xdb7d4f66, 0xbcf3b267, 0xd972a2bc, 0xbefc5fbd, + 0x166f58be, 0x71e1a5bf, 0x42a52049, 0x252bdd48, 0x8db8da4b, 0xea36274a, + 0xeb31d1a7, 0x8cbf2ca6, 0x242c2ba5, 0x43a2d6a4, 0x70e65352, 0x1768ae53, + 0xbffba950, 0xd8755451, 0x74f988e6, 0x137775e7, 0xbbe472e4, 0xdc6a8fe5, + 0xef2e0a13, 0x88a0f712, 0x2033f011, 0x47bd0d10, 0x46bafbfd, 0x213406fc, + 0x89a701ff, 0xee29fcfe, 0xdd6d7908, 0xbae38409, 0x1270830a, 0x75fe7e0b, + 0x107f6ed0, 0x77f193d1, 0xdf6294d2, 0xb8ec69d3, 0x8ba8ec25, 0xec261124, + 0x44b51627, 0x233beb26, 0x223c1dcb, 0x45b2e0ca, 0xed21e7c9, 0x8aaf1ac8, + 0xb9eb9f3e, 0xde65623f, 0x76f6653c, 0x1178983d}; + +const uint32_t kStrideExtensionTable3[256] = { + 0x00000000, 0xf20c0dfe, 0xe1f46d0d, 0x13f860f3, 0xc604aceb, 0x3408a115, + 0x27f0c1e6, 0xd5fccc18, 0x89e52f27, 0x7be922d9, 0x6811422a, 0x9a1d4fd4, + 0x4fe183cc, 0xbded8e32, 0xae15eec1, 0x5c19e33f, 0x162628bf, 0xe42a2541, + 0xf7d245b2, 0x05de484c, 0xd0228454, 0x222e89aa, 0x31d6e959, 0xc3dae4a7, + 0x9fc30798, 0x6dcf0a66, 0x7e376a95, 0x8c3b676b, 0x59c7ab73, 0xabcba68d, + 0xb833c67e, 0x4a3fcb80, 0x2c4c517e, 0xde405c80, 0xcdb83c73, 0x3fb4318d, + 0xea48fd95, 0x1844f06b, 0x0bbc9098, 0xf9b09d66, 0xa5a97e59, 0x57a573a7, + 0x445d1354, 0xb6511eaa, 0x63add2b2, 0x91a1df4c, 0x8259bfbf, 0x7055b241, + 0x3a6a79c1, 0xc866743f, 0xdb9e14cc, 0x29921932, 0xfc6ed52a, 0x0e62d8d4, + 0x1d9ab827, 0xef96b5d9, 0xb38f56e6, 0x41835b18, 0x527b3beb, 0xa0773615, + 0x758bfa0d, 0x8787f7f3, 0x947f9700, 0x66739afe, 0x5898a2fc, 0xaa94af02, + 0xb96ccff1, 0x4b60c20f, 0x9e9c0e17, 0x6c9003e9, 0x7f68631a, 0x8d646ee4, + 0xd17d8ddb, 0x23718025, 0x3089e0d6, 0xc285ed28, 0x17792130, 0xe5752cce, + 0xf68d4c3d, 0x048141c3, 0x4ebe8a43, 0xbcb287bd, 0xaf4ae74e, 0x5d46eab0, + 0x88ba26a8, 0x7ab62b56, 0x694e4ba5, 0x9b42465b, 0xc75ba564, 0x3557a89a, + 0x26afc869, 0xd4a3c597, 0x015f098f, 0xf3530471, 0xe0ab6482, 0x12a7697c, + 0x74d4f382, 0x86d8fe7c, 0x95209e8f, 0x672c9371, 0xb2d05f69, 0x40dc5297, + 0x53243264, 0xa1283f9a, 0xfd31dca5, 0x0f3dd15b, 0x1cc5b1a8, 0xeec9bc56, + 0x3b35704e, 0xc9397db0, 0xdac11d43, 0x28cd10bd, 0x62f2db3d, 0x90fed6c3, + 0x8306b630, 0x710abbce, 0xa4f677d6, 0x56fa7a28, 0x45021adb, 0xb70e1725, + 0xeb17f41a, 0x191bf9e4, 0x0ae39917, 0xf8ef94e9, 0x2d1358f1, 0xdf1f550f, + 0xcce735fc, 0x3eeb3802, 0xb13145f8, 0x433d4806, 0x50c528f5, 0xa2c9250b, + 0x7735e913, 0x8539e4ed, 0x96c1841e, 0x64cd89e0, 0x38d46adf, 0xcad86721, + 0xd92007d2, 0x2b2c0a2c, 0xfed0c634, 0x0cdccbca, 0x1f24ab39, 0xed28a6c7, + 0xa7176d47, 0x551b60b9, 0x46e3004a, 0xb4ef0db4, 0x6113c1ac, 0x931fcc52, + 0x80e7aca1, 0x72eba15f, 0x2ef24260, 0xdcfe4f9e, 0xcf062f6d, 0x3d0a2293, + 0xe8f6ee8b, 0x1afae375, 0x09028386, 0xfb0e8e78, 0x9d7d1486, 0x6f711978, + 0x7c89798b, 0x8e857475, 0x5b79b86d, 0xa975b593, 0xba8dd560, 0x4881d89e, + 0x14983ba1, 0xe694365f, 0xf56c56ac, 0x07605b52, 0xd29c974a, 0x20909ab4, + 0x3368fa47, 0xc164f7b9, 0x8b5b3c39, 0x795731c7, 0x6aaf5134, 0x98a35cca, + 0x4d5f90d2, 0xbf539d2c, 0xacabfddf, 0x5ea7f021, 0x02be131e, 0xf0b21ee0, + 0xe34a7e13, 0x114673ed, 0xc4babff5, 0x36b6b20b, 0x254ed2f8, 0xd742df06, + 0xe9a9e704, 0x1ba5eafa, 0x085d8a09, 0xfa5187f7, 0x2fad4bef, 0xdda14611, + 0xce5926e2, 0x3c552b1c, 0x604cc823, 0x9240c5dd, 0x81b8a52e, 0x73b4a8d0, + 0xa64864c8, 0x54446936, 0x47bc09c5, 0xb5b0043b, 0xff8fcfbb, 0x0d83c245, + 0x1e7ba2b6, 0xec77af48, 0x398b6350, 0xcb876eae, 0xd87f0e5d, 0x2a7303a3, + 0x766ae09c, 0x8466ed62, 0x979e8d91, 0x6592806f, 0xb06e4c77, 0x42624189, + 0x519a217a, 0xa3962c84, 0xc5e5b67a, 0x37e9bb84, 0x2411db77, 0xd61dd689, + 0x03e11a91, 0xf1ed176f, 0xe215779c, 0x10197a62, 0x4c00995d, 0xbe0c94a3, + 0xadf4f450, 0x5ff8f9ae, 0x8a0435b6, 0x78083848, 0x6bf058bb, 0x99fc5545, + 0xd3c39ec5, 0x21cf933b, 0x3237f3c8, 0xc03bfe36, 0x15c7322e, 0xe7cb3fd0, + 0xf4335f23, 0x063f52dd, 0x5a26b1e2, 0xa82abc1c, 0xbbd2dcef, 0x49ded111, + 0x9c221d09, 0x6e2e10f7, 0x7dd67004, 0x8fda7dfa}; + +// CRCs are pre- and post- conditioned by xoring with all ones. +static constexpr const uint32_t kCRC32Xor = static_cast(0xffffffffU); + +// Reads a little-endian 32-bit integer from a 32-bit-aligned buffer. +inline uint32_t ReadUint32LE(const uint8_t* buffer) { + return DecodeFixed32(reinterpret_cast(buffer)); +} + +// Returns the smallest address >= the given address that is aligned to N bytes. +// +// N must be a power of two. +template +constexpr inline const uint8_t* RoundUp(const uint8_t* pointer) { + return reinterpret_cast( + (reinterpret_cast(pointer) + (N - 1)) & + ~static_cast(N - 1)); +} + +} // namespace + +// Determine if the CPU running this program can accelerate the CRC32C +// calculation. +static bool CanAccelerateCRC32C() { + // port::AcceleretedCRC32C returns zero when unable to accelerate. + static const char kTestCRCBuffer[] = "TestCRCBuffer"; + static const char kBufSize = sizeof(kTestCRCBuffer) - 1; + static const uint32_t kTestCRCValue = 0xdcbc59fa; + + return port::AcceleratedCRC32C(0, kTestCRCBuffer, kBufSize) == kTestCRCValue; +} + +uint32_t Extend(uint32_t crc, const char* data, size_t n) { + static bool accelerate = CanAccelerateCRC32C(); + if (accelerate) { + return port::AcceleratedCRC32C(crc, data, n); + } + + const uint8_t* p = reinterpret_cast(data); + const uint8_t* e = p + n; + uint32_t l = crc ^ kCRC32Xor; + +// Process one byte at a time. +#define STEP1 \ + do { \ + int c = (l & 0xff) ^ *p++; \ + l = kByteExtensionTable[c] ^ (l >> 8); \ + } while (0) + +// Process one of the 4 strides of 4-byte data. +#define STEP4(s) \ + do { \ + crc##s = ReadUint32LE(p + s * 4) ^ kStrideExtensionTable3[crc##s & 0xff] ^ \ + kStrideExtensionTable2[(crc##s >> 8) & 0xff] ^ \ + kStrideExtensionTable1[(crc##s >> 16) & 0xff] ^ \ + kStrideExtensionTable0[crc##s >> 24]; \ + } while (0) + +// Process a 16-byte swath of 4 strides, each of which has 4 bytes of data. +#define STEP16 \ + do { \ + STEP4(0); \ + STEP4(1); \ + STEP4(2); \ + STEP4(3); \ + p += 16; \ + } while (0) + +// Process 4 bytes that were already loaded into a word. +#define STEP4W(w) \ + do { \ + w ^= l; \ + for (size_t i = 0; i < 4; ++i) { \ + w = (w >> 8) ^ kByteExtensionTable[w & 0xff]; \ + } \ + l = w; \ + } while (0) + + // Point x at first 4-byte aligned byte in the buffer. This might be past the + // end of the buffer. + const uint8_t* x = RoundUp<4>(p); + if (x <= e) { + // Process bytes p is 4-byte aligned. + while (p != x) { + STEP1; + } + } + + if ((e - p) >= 16) { + // Load a 16-byte swath into the stride partial results. + uint32_t crc0 = ReadUint32LE(p + 0 * 4) ^ l; + uint32_t crc1 = ReadUint32LE(p + 1 * 4); + uint32_t crc2 = ReadUint32LE(p + 2 * 4); + uint32_t crc3 = ReadUint32LE(p + 3 * 4); + p += 16; + + // It is possible to get better speeds (at least on x86) by interleaving + // prefetching 256 bytes ahead with processing 64 bytes at a time. See the + // portable implementation in https://github.com/google/crc32c/. + + // Process one 16-byte swath at a time. + while ((e - p) >= 16) { + STEP16; + } + + // Advance one word at a time as far as possible. + while ((e - p) >= 4) { + STEP4(0); + uint32_t tmp = crc0; + crc0 = crc1; + crc1 = crc2; + crc2 = crc3; + crc3 = tmp; + p += 4; + } + + // Combine the 4 partial stride results. + l = 0; + STEP4W(crc0); + STEP4W(crc1); + STEP4W(crc2); + STEP4W(crc3); + } + + // Process the last few bytes. + while (p != e) { + STEP1; + } +#undef STEP4W +#undef STEP16 +#undef STEP4 +#undef STEP1 + return l ^ kCRC32Xor; +} + +} // namespace crc32c +} // namespace leveldb diff --git a/leveldb/util/crc32c.h b/leveldb/util/crc32c.h new file mode 100644 index 000000000..b420b5f7c --- /dev/null +++ b/leveldb/util/crc32c.h @@ -0,0 +1,43 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_CRC32C_H_ +#define STORAGE_LEVELDB_UTIL_CRC32C_H_ + +#include +#include + +namespace leveldb { +namespace crc32c { + +// Return the crc32c of concat(A, data[0,n-1]) where init_crc is the +// crc32c of some string A. Extend() is often used to maintain the +// crc32c of a stream of data. +uint32_t Extend(uint32_t init_crc, const char* data, size_t n); + +// Return the crc32c of data[0,n-1] +inline uint32_t Value(const char* data, size_t n) { return Extend(0, data, n); } + +static const uint32_t kMaskDelta = 0xa282ead8ul; + +// Return a masked representation of crc. +// +// Motivation: it is problematic to compute the CRC of a string that +// contains embedded CRCs. Therefore we recommend that CRCs stored +// somewhere (e.g., in files) should be masked before being stored. +inline uint32_t Mask(uint32_t crc) { + // Rotate right by 15 bits and add a constant. + return ((crc >> 15) | (crc << 17)) + kMaskDelta; +} + +// Return the crc whose masked representation is masked_crc. +inline uint32_t Unmask(uint32_t masked_crc) { + uint32_t rot = masked_crc - kMaskDelta; + return ((rot >> 17) | (rot << 15)); +} + +} // namespace crc32c +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_CRC32C_H_ diff --git a/leveldb/util/crc32c_test.cc b/leveldb/util/crc32c_test.cc new file mode 100644 index 000000000..2fe1c415e --- /dev/null +++ b/leveldb/util/crc32c_test.cc @@ -0,0 +1,56 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/crc32c.h" + +#include "gtest/gtest.h" + +namespace leveldb { +namespace crc32c { + +TEST(CRC, StandardResults) { + // From rfc3720 section B.4. + char buf[32]; + + memset(buf, 0, sizeof(buf)); + ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf))); + + memset(buf, 0xff, sizeof(buf)); + ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf))); + + for (int i = 0; i < 32; i++) { + buf[i] = i; + } + ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf))); + + for (int i = 0; i < 32; i++) { + buf[i] = 31 - i; + } + ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf))); + + uint8_t data[48] = { + 0x01, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x28, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + ASSERT_EQ(0xd9963a56, Value(reinterpret_cast(data), sizeof(data))); +} + +TEST(CRC, Values) { ASSERT_NE(Value("a", 1), Value("foo", 3)); } + +TEST(CRC, Extend) { + ASSERT_EQ(Value("hello world", 11), Extend(Value("hello ", 6), "world", 5)); +} + +TEST(CRC, Mask) { + uint32_t crc = Value("foo", 3); + ASSERT_NE(crc, Mask(crc)); + ASSERT_NE(crc, Mask(Mask(crc))); + ASSERT_EQ(crc, Unmask(Mask(crc))); + ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc))))); +} + +} // namespace crc32c +} // namespace leveldb diff --git a/leveldb/util/env.cc b/leveldb/util/env.cc new file mode 100644 index 000000000..a53b230a9 --- /dev/null +++ b/leveldb/util/env.cc @@ -0,0 +1,108 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/env.h" + +#include + +// This workaround can be removed when leveldb::Env::DeleteFile is removed. +// See env.h for justification. +#if defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED) +#undef DeleteFile +#endif + +namespace leveldb { + +Env::Env() = default; + +Env::~Env() = default; + +Status Env::NewAppendableFile(const std::string& fname, WritableFile** result) { + return Status::NotSupported("NewAppendableFile", fname); +} + +Status Env::RemoveDir(const std::string& dirname) { return DeleteDir(dirname); } +Status Env::DeleteDir(const std::string& dirname) { return RemoveDir(dirname); } + +Status Env::RemoveFile(const std::string& fname) { return DeleteFile(fname); } +Status Env::DeleteFile(const std::string& fname) { return RemoveFile(fname); } + +SequentialFile::~SequentialFile() = default; + +RandomAccessFile::~RandomAccessFile() = default; + +WritableFile::~WritableFile() = default; + +Logger::~Logger() = default; + +FileLock::~FileLock() = default; + +void Log(Logger* info_log, const char* format, ...) { + if (info_log != nullptr) { + std::va_list ap; + va_start(ap, format); + info_log->Logv(format, ap); + va_end(ap); + } +} + +static Status DoWriteStringToFile(Env* env, const Slice& data, + const std::string& fname, bool should_sync) { + WritableFile* file; + Status s = env->NewWritableFile(fname, &file); + if (!s.ok()) { + return s; + } + s = file->Append(data); + if (s.ok() && should_sync) { + s = file->Sync(); + } + if (s.ok()) { + s = file->Close(); + } + delete file; // Will auto-close if we did not close above + if (!s.ok()) { + env->RemoveFile(fname); + } + return s; +} + +Status WriteStringToFile(Env* env, const Slice& data, + const std::string& fname) { + return DoWriteStringToFile(env, data, fname, false); +} + +Status WriteStringToFileSync(Env* env, const Slice& data, + const std::string& fname) { + return DoWriteStringToFile(env, data, fname, true); +} + +Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { + data->clear(); + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + return s; + } + static const int kBufferSize = 8192; + char* space = new char[kBufferSize]; + while (true) { + Slice fragment; + s = file->Read(kBufferSize, &fragment, space); + if (!s.ok()) { + break; + } + data->append(fragment.data(), fragment.size()); + if (fragment.empty()) { + break; + } + } + delete[] space; + delete file; + return s; +} + +EnvWrapper::~EnvWrapper() {} + +} // namespace leveldb diff --git a/leveldb/util/env_posix.cc b/leveldb/util/env_posix.cc new file mode 100644 index 000000000..ffd06c403 --- /dev/null +++ b/leveldb/util/env_posix.cc @@ -0,0 +1,926 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#ifndef __Fuchsia__ +#include +#endif +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "leveldb/env.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/env_posix_test_helper.h" +#include "util/posix_logger.h" + +namespace leveldb { + +namespace { + +// Set by EnvPosixTestHelper::SetReadOnlyMMapLimit() and MaxOpenFiles(). +int g_open_read_only_file_limit = -1; + +// Up to 1000 mmap regions for 64-bit binaries; none for 32-bit. +constexpr const int kDefaultMmapLimit = (sizeof(void*) >= 8) ? 1000 : 0; + +// Can be set using EnvPosixTestHelper::SetReadOnlyMMapLimit(). +int g_mmap_limit = kDefaultMmapLimit; + +// Common flags defined for all posix open operations +#if defined(HAVE_O_CLOEXEC) +constexpr const int kOpenBaseFlags = O_CLOEXEC; +#else +constexpr const int kOpenBaseFlags = 0; +#endif // defined(HAVE_O_CLOEXEC) + +constexpr const size_t kWritableFileBufferSize = 65536; + +Status PosixError(const std::string& context, int error_number) { + if (error_number == ENOENT) { + return Status::NotFound(context, std::strerror(error_number)); + } else { + return Status::IOError(context, std::strerror(error_number)); + } +} + +// Helper class to limit resource usage to avoid exhaustion. +// Currently used to limit read-only file descriptors and mmap file usage +// so that we do not run out of file descriptors or virtual memory, or run into +// kernel performance problems for very large databases. +class Limiter { + public: + // Limit maximum number of resources to |max_acquires|. + Limiter(int max_acquires) + : +#if !defined(NDEBUG) + max_acquires_(max_acquires), +#endif // !defined(NDEBUG) + acquires_allowed_(max_acquires) { + assert(max_acquires >= 0); + } + + Limiter(const Limiter&) = delete; + Limiter operator=(const Limiter&) = delete; + + // If another resource is available, acquire it and return true. + // Else return false. + bool Acquire() { + int old_acquires_allowed = + acquires_allowed_.fetch_sub(1, std::memory_order_relaxed); + + if (old_acquires_allowed > 0) return true; + + int pre_increment_acquires_allowed = + acquires_allowed_.fetch_add(1, std::memory_order_relaxed); + + // Silence compiler warnings about unused arguments when NDEBUG is defined. + (void)pre_increment_acquires_allowed; + // If the check below fails, Release() was called more times than acquire. + assert(pre_increment_acquires_allowed < max_acquires_); + + return false; + } + + // Release a resource acquired by a previous call to Acquire() that returned + // true. + void Release() { + int old_acquires_allowed = + acquires_allowed_.fetch_add(1, std::memory_order_relaxed); + + // Silence compiler warnings about unused arguments when NDEBUG is defined. + (void)old_acquires_allowed; + // If the check below fails, Release() was called more times than acquire. + assert(old_acquires_allowed < max_acquires_); + } + + private: +#if !defined(NDEBUG) + // Catches an excessive number of Release() calls. + const int max_acquires_; +#endif // !defined(NDEBUG) + + // The number of available resources. + // + // This is a counter and is not tied to the invariants of any other class, so + // it can be operated on safely using std::memory_order_relaxed. + std::atomic acquires_allowed_; +}; + +// Implements sequential read access in a file using read(). +// +// Instances of this class are thread-friendly but not thread-safe, as required +// by the SequentialFile API. +class PosixSequentialFile final : public SequentialFile { + public: + PosixSequentialFile(std::string filename, int fd) + : fd_(fd), filename_(std::move(filename)) {} + ~PosixSequentialFile() override { close(fd_); } + + Status Read(size_t n, Slice* result, char* scratch) override { + Status status; + while (true) { + ::ssize_t read_size = ::read(fd_, scratch, n); + if (read_size < 0) { // Read error. + if (errno == EINTR) { + continue; // Retry + } + status = PosixError(filename_, errno); + break; + } + *result = Slice(scratch, read_size); + break; + } + return status; + } + + Status Skip(uint64_t n) override { + if (::lseek(fd_, n, SEEK_CUR) == static_cast(-1)) { + return PosixError(filename_, errno); + } + return Status::OK(); + } + + private: + const int fd_; + const std::string filename_; +}; + +// Implements random read access in a file using pread(). +// +// Instances of this class are thread-safe, as required by the RandomAccessFile +// API. Instances are immutable and Read() only calls thread-safe library +// functions. +class PosixRandomAccessFile final : public RandomAccessFile { + public: + // The new instance takes ownership of |fd|. |fd_limiter| must outlive this + // instance, and will be used to determine if . + PosixRandomAccessFile(std::string filename, int fd, Limiter* fd_limiter) + : has_permanent_fd_(fd_limiter->Acquire()), + fd_(has_permanent_fd_ ? fd : -1), + fd_limiter_(fd_limiter), + filename_(std::move(filename)) { + if (!has_permanent_fd_) { + assert(fd_ == -1); + ::close(fd); // The file will be opened on every read. + } + } + + ~PosixRandomAccessFile() override { + if (has_permanent_fd_) { + assert(fd_ != -1); + ::close(fd_); + fd_limiter_->Release(); + } + } + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + int fd = fd_; + if (!has_permanent_fd_) { + fd = ::open(filename_.c_str(), O_RDONLY | kOpenBaseFlags); + if (fd < 0) { + return PosixError(filename_, errno); + } + } + + assert(fd != -1); + + Status status; + ssize_t read_size = ::pread(fd, scratch, n, static_cast(offset)); + *result = Slice(scratch, (read_size < 0) ? 0 : read_size); + if (read_size < 0) { + // An error: return a non-ok status. + status = PosixError(filename_, errno); + } + if (!has_permanent_fd_) { + // Close the temporary file descriptor opened earlier. + assert(fd != fd_); + ::close(fd); + } + return status; + } + + private: + const bool has_permanent_fd_; // If false, the file is opened on every read. + const int fd_; // -1 if has_permanent_fd_ is false. + Limiter* const fd_limiter_; + const std::string filename_; +}; + +// Implements random read access in a file using mmap(). +// +// Instances of this class are thread-safe, as required by the RandomAccessFile +// API. Instances are immutable and Read() only calls thread-safe library +// functions. +class PosixMmapReadableFile final : public RandomAccessFile { + public: + // mmap_base[0, length-1] points to the memory-mapped contents of the file. It + // must be the result of a successful call to mmap(). This instances takes + // over the ownership of the region. + // + // |mmap_limiter| must outlive this instance. The caller must have already + // acquired the right to use one mmap region, which will be released when this + // instance is destroyed. + PosixMmapReadableFile(std::string filename, char* mmap_base, size_t length, + Limiter* mmap_limiter) + : mmap_base_(mmap_base), + length_(length), + mmap_limiter_(mmap_limiter), + filename_(std::move(filename)) {} + + ~PosixMmapReadableFile() override { + ::munmap(static_cast(mmap_base_), length_); + mmap_limiter_->Release(); + } + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + if (offset + n > length_) { + *result = Slice(); + return PosixError(filename_, EINVAL); + } + + *result = Slice(mmap_base_ + offset, n); + return Status::OK(); + } + + private: + char* const mmap_base_; + const size_t length_; + Limiter* const mmap_limiter_; + const std::string filename_; +}; + +class PosixWritableFile final : public WritableFile { + public: + PosixWritableFile(std::string filename, int fd) + : pos_(0), + fd_(fd), + is_manifest_(IsManifest(filename)), + filename_(std::move(filename)), + dirname_(Dirname(filename_)) {} + + ~PosixWritableFile() override { + if (fd_ >= 0) { + // Ignoring any potential errors + Close(); + } + } + + Status Append(const Slice& data) override { + size_t write_size = data.size(); + const char* write_data = data.data(); + + // Fit as much as possible into buffer. + size_t copy_size = std::min(write_size, kWritableFileBufferSize - pos_); + std::memcpy(buf_ + pos_, write_data, copy_size); + write_data += copy_size; + write_size -= copy_size; + pos_ += copy_size; + if (write_size == 0) { + return Status::OK(); + } + + // Can't fit in buffer, so need to do at least one write. + Status status = FlushBuffer(); + if (!status.ok()) { + return status; + } + + // Small writes go to buffer, large writes are written directly. + if (write_size < kWritableFileBufferSize) { + std::memcpy(buf_, write_data, write_size); + pos_ = write_size; + return Status::OK(); + } + return WriteUnbuffered(write_data, write_size); + } + + Status Close() override { + Status status = FlushBuffer(); + const int close_result = ::close(fd_); + if (close_result < 0 && status.ok()) { + status = PosixError(filename_, errno); + } + fd_ = -1; + return status; + } + + Status Flush() override { return FlushBuffer(); } + + Status Sync() override { + // Ensure new files referred to by the manifest are in the filesystem. + // + // This needs to happen before the manifest file is flushed to disk, to + // avoid crashing in a state where the manifest refers to files that are not + // yet on disk. + Status status = SyncDirIfManifest(); + if (!status.ok()) { + return status; + } + + status = FlushBuffer(); + if (!status.ok()) { + return status; + } + + return SyncFd(fd_, filename_); + } + + private: + Status FlushBuffer() { + Status status = WriteUnbuffered(buf_, pos_); + pos_ = 0; + return status; + } + + Status WriteUnbuffered(const char* data, size_t size) { + while (size > 0) { + ssize_t write_result = ::write(fd_, data, size); + if (write_result < 0) { + if (errno == EINTR) { + continue; // Retry + } + return PosixError(filename_, errno); + } + data += write_result; + size -= write_result; + } + return Status::OK(); + } + + Status SyncDirIfManifest() { + Status status; + if (!is_manifest_) { + return status; + } + + int fd = ::open(dirname_.c_str(), O_RDONLY | kOpenBaseFlags); + if (fd < 0) { + status = PosixError(dirname_, errno); + } else { + status = SyncFd(fd, dirname_); + ::close(fd); + } + return status; + } + + // Ensures that all the caches associated with the given file descriptor's + // data are flushed all the way to durable media, and can withstand power + // failures. + // + // The path argument is only used to populate the description string in the + // returned Status if an error occurs. + static Status SyncFd(int fd, const std::string& fd_path) { +#if HAVE_FULLFSYNC + // On macOS and iOS, fsync() doesn't guarantee durability past power + // failures. fcntl(F_FULLFSYNC) is required for that purpose. Some + // filesystems don't support fcntl(F_FULLFSYNC), and require a fallback to + // fsync(). + if (::fcntl(fd, F_FULLFSYNC) == 0) { + return Status::OK(); + } +#endif // HAVE_FULLFSYNC + +#if HAVE_FDATASYNC + bool sync_success = ::fdatasync(fd) == 0; +#else + bool sync_success = ::fsync(fd) == 0; +#endif // HAVE_FDATASYNC + + if (sync_success) { + return Status::OK(); + } + return PosixError(fd_path, errno); + } + + // Returns the directory name in a path pointing to a file. + // + // Returns "." if the path does not contain any directory separator. + static std::string Dirname(const std::string& filename) { + std::string::size_type separator_pos = filename.rfind('/'); + if (separator_pos == std::string::npos) { + return std::string("."); + } + // The filename component should not contain a path separator. If it does, + // the splitting was done incorrectly. + assert(filename.find('/', separator_pos + 1) == std::string::npos); + + return filename.substr(0, separator_pos); + } + + // Extracts the file name from a path pointing to a file. + // + // The returned Slice points to |filename|'s data buffer, so it is only valid + // while |filename| is alive and unchanged. + static Slice Basename(const std::string& filename) { + std::string::size_type separator_pos = filename.rfind('/'); + if (separator_pos == std::string::npos) { + return Slice(filename); + } + // The filename component should not contain a path separator. If it does, + // the splitting was done incorrectly. + assert(filename.find('/', separator_pos + 1) == std::string::npos); + + return Slice(filename.data() + separator_pos + 1, + filename.length() - separator_pos - 1); + } + + // True if the given file is a manifest file. + static bool IsManifest(const std::string& filename) { + return Basename(filename).starts_with("MANIFEST"); + } + + // buf_[0, pos_ - 1] contains data to be written to fd_. + char buf_[kWritableFileBufferSize]; + size_t pos_; + int fd_; + + const bool is_manifest_; // True if the file's name starts with MANIFEST. + const std::string filename_; + const std::string dirname_; // The directory of filename_. +}; + +int LockOrUnlock(int fd, bool lock) { + errno = 0; + struct ::flock file_lock_info; + std::memset(&file_lock_info, 0, sizeof(file_lock_info)); + file_lock_info.l_type = (lock ? F_WRLCK : F_UNLCK); + file_lock_info.l_whence = SEEK_SET; + file_lock_info.l_start = 0; + file_lock_info.l_len = 0; // Lock/unlock entire file. + return ::fcntl(fd, F_SETLK, &file_lock_info); +} + +// Instances are thread-safe because they are immutable. +class PosixFileLock : public FileLock { + public: + PosixFileLock(int fd, std::string filename) + : fd_(fd), filename_(std::move(filename)) {} + + int fd() const { return fd_; } + const std::string& filename() const { return filename_; } + + private: + const int fd_; + const std::string filename_; +}; + +// Tracks the files locked by PosixEnv::LockFile(). +// +// We maintain a separate set instead of relying on fcntl(F_SETLK) because +// fcntl(F_SETLK) does not provide any protection against multiple uses from the +// same process. +// +// Instances are thread-safe because all member data is guarded by a mutex. +class PosixLockTable { + public: + bool Insert(const std::string& fname) LOCKS_EXCLUDED(mu_) { + mu_.Lock(); + bool succeeded = locked_files_.insert(fname).second; + mu_.Unlock(); + return succeeded; + } + void Remove(const std::string& fname) LOCKS_EXCLUDED(mu_) { + mu_.Lock(); + locked_files_.erase(fname); + mu_.Unlock(); + } + + private: + port::Mutex mu_; + std::set locked_files_ GUARDED_BY(mu_); +}; + +class PosixEnv : public Env { + public: + PosixEnv(); + ~PosixEnv() override { + static const char msg[] = + "PosixEnv singleton destroyed. Unsupported behavior!\n"; + std::fwrite(msg, 1, sizeof(msg), stderr); + std::abort(); + } + + Status NewSequentialFile(const std::string& filename, + SequentialFile** result) override { + int fd = ::open(filename.c_str(), O_RDONLY | kOpenBaseFlags); + if (fd < 0) { + *result = nullptr; + return PosixError(filename, errno); + } + + *result = new PosixSequentialFile(filename, fd); + return Status::OK(); + } + + Status NewRandomAccessFile(const std::string& filename, + RandomAccessFile** result) override { + *result = nullptr; + int fd = ::open(filename.c_str(), O_RDONLY | kOpenBaseFlags); + if (fd < 0) { + return PosixError(filename, errno); + } + + if (!mmap_limiter_.Acquire()) { + *result = new PosixRandomAccessFile(filename, fd, &fd_limiter_); + return Status::OK(); + } + + uint64_t file_size; + Status status = GetFileSize(filename, &file_size); + if (status.ok()) { + void* mmap_base = + ::mmap(/*addr=*/nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); + if (mmap_base != MAP_FAILED) { + *result = new PosixMmapReadableFile(filename, + reinterpret_cast(mmap_base), + file_size, &mmap_limiter_); + } else { + status = PosixError(filename, errno); + } + } + ::close(fd); + if (!status.ok()) { + mmap_limiter_.Release(); + } + return status; + } + + Status NewWritableFile(const std::string& filename, + WritableFile** result) override { + int fd = ::open(filename.c_str(), + O_TRUNC | O_WRONLY | O_CREAT | kOpenBaseFlags, 0644); + if (fd < 0) { + *result = nullptr; + return PosixError(filename, errno); + } + + *result = new PosixWritableFile(filename, fd); + return Status::OK(); + } + + Status NewAppendableFile(const std::string& filename, + WritableFile** result) override { + int fd = ::open(filename.c_str(), + O_APPEND | O_WRONLY | O_CREAT | kOpenBaseFlags, 0644); + if (fd < 0) { + *result = nullptr; + return PosixError(filename, errno); + } + + *result = new PosixWritableFile(filename, fd); + return Status::OK(); + } + + bool FileExists(const std::string& filename) override { + return ::access(filename.c_str(), F_OK) == 0; + } + + Status GetChildren(const std::string& directory_path, + std::vector* result) override { + result->clear(); + ::DIR* dir = ::opendir(directory_path.c_str()); + if (dir == nullptr) { + return PosixError(directory_path, errno); + } + struct ::dirent* entry; + while ((entry = ::readdir(dir)) != nullptr) { + result->emplace_back(entry->d_name); + } + ::closedir(dir); + return Status::OK(); + } + + Status RemoveFile(const std::string& filename) override { + if (::unlink(filename.c_str()) != 0) { + return PosixError(filename, errno); + } + return Status::OK(); + } + + Status CreateDir(const std::string& dirname) override { + if (::mkdir(dirname.c_str(), 0755) != 0) { + return PosixError(dirname, errno); + } + return Status::OK(); + } + + Status RemoveDir(const std::string& dirname) override { + if (::rmdir(dirname.c_str()) != 0) { + return PosixError(dirname, errno); + } + return Status::OK(); + } + + Status GetFileSize(const std::string& filename, uint64_t* size) override { + struct ::stat file_stat; + if (::stat(filename.c_str(), &file_stat) != 0) { + *size = 0; + return PosixError(filename, errno); + } + *size = file_stat.st_size; + return Status::OK(); + } + + Status RenameFile(const std::string& from, const std::string& to) override { + if (std::rename(from.c_str(), to.c_str()) != 0) { + return PosixError(from, errno); + } + return Status::OK(); + } + + Status LockFile(const std::string& filename, FileLock** lock) override { + *lock = nullptr; + + int fd = ::open(filename.c_str(), O_RDWR | O_CREAT | kOpenBaseFlags, 0644); + if (fd < 0) { + return PosixError(filename, errno); + } + + if (!locks_.Insert(filename)) { + ::close(fd); + return Status::IOError("lock " + filename, "already held by process"); + } + + if (LockOrUnlock(fd, true) == -1) { + int lock_errno = errno; + ::close(fd); + locks_.Remove(filename); + return PosixError("lock " + filename, lock_errno); + } + + *lock = new PosixFileLock(fd, filename); + return Status::OK(); + } + + Status UnlockFile(FileLock* lock) override { + PosixFileLock* posix_file_lock = static_cast(lock); + if (LockOrUnlock(posix_file_lock->fd(), false) == -1) { + return PosixError("unlock " + posix_file_lock->filename(), errno); + } + locks_.Remove(posix_file_lock->filename()); + ::close(posix_file_lock->fd()); + delete posix_file_lock; + return Status::OK(); + } + + void Schedule(void (*background_work_function)(void* background_work_arg), + void* background_work_arg) override; + + void StartThread(void (*thread_main)(void* thread_main_arg), + void* thread_main_arg) override { + std::thread new_thread(thread_main, thread_main_arg); + new_thread.detach(); + } + + Status GetTestDirectory(std::string* result) override { + const char* env = std::getenv("TEST_TMPDIR"); + if (env && env[0] != '\0') { + *result = env; + } else { + char buf[100]; + std::snprintf(buf, sizeof(buf), "/tmp/leveldbtest-%d", + static_cast(::geteuid())); + *result = buf; + } + + // The CreateDir status is ignored because the directory may already exist. + CreateDir(*result); + + return Status::OK(); + } + + Status NewLogger(const std::string& filename, Logger** result) override { + int fd = ::open(filename.c_str(), + O_APPEND | O_WRONLY | O_CREAT | kOpenBaseFlags, 0644); + if (fd < 0) { + *result = nullptr; + return PosixError(filename, errno); + } + + std::FILE* fp = ::fdopen(fd, "w"); + if (fp == nullptr) { + ::close(fd); + *result = nullptr; + return PosixError(filename, errno); + } else { + *result = new PosixLogger(fp); + return Status::OK(); + } + } + + uint64_t NowMicros() override { + static constexpr uint64_t kUsecondsPerSecond = 1000000; + struct ::timeval tv; + ::gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * kUsecondsPerSecond + tv.tv_usec; + } + + void SleepForMicroseconds(int micros) override { + std::this_thread::sleep_for(std::chrono::microseconds(micros)); + } + + private: + void BackgroundThreadMain(); + + static void BackgroundThreadEntryPoint(PosixEnv* env) { + env->BackgroundThreadMain(); + } + + // Stores the work item data in a Schedule() call. + // + // Instances are constructed on the thread calling Schedule() and used on the + // background thread. + // + // This structure is thread-safe because it is immutable. + struct BackgroundWorkItem { + explicit BackgroundWorkItem(void (*function)(void* arg), void* arg) + : function(function), arg(arg) {} + + void (*const function)(void*); + void* const arg; + }; + + port::Mutex background_work_mutex_; + port::CondVar background_work_cv_ GUARDED_BY(background_work_mutex_); + bool started_background_thread_ GUARDED_BY(background_work_mutex_); + + std::queue background_work_queue_ + GUARDED_BY(background_work_mutex_); + + PosixLockTable locks_; // Thread-safe. + Limiter mmap_limiter_; // Thread-safe. + Limiter fd_limiter_; // Thread-safe. +}; + +// Return the maximum number of concurrent mmaps. +int MaxMmaps() { return g_mmap_limit; } + +// Return the maximum number of read-only files to keep open. +int MaxOpenFiles() { + if (g_open_read_only_file_limit >= 0) { + return g_open_read_only_file_limit; + } +#ifdef __Fuchsia__ + // Fuchsia doesn't implement getrlimit. + g_open_read_only_file_limit = 50; +#else + struct ::rlimit rlim; + if (::getrlimit(RLIMIT_NOFILE, &rlim)) { + // getrlimit failed, fallback to hard-coded default. + g_open_read_only_file_limit = 50; + } else if (rlim.rlim_cur == RLIM_INFINITY) { + g_open_read_only_file_limit = std::numeric_limits::max(); + } else { + // Allow use of 20% of available file descriptors for read-only files. + g_open_read_only_file_limit = rlim.rlim_cur / 5; + } +#endif + return g_open_read_only_file_limit; +} + +} // namespace + +PosixEnv::PosixEnv() + : background_work_cv_(&background_work_mutex_), + started_background_thread_(false), + mmap_limiter_(MaxMmaps()), + fd_limiter_(MaxOpenFiles()) {} + +void PosixEnv::Schedule( + void (*background_work_function)(void* background_work_arg), + void* background_work_arg) { + background_work_mutex_.Lock(); + + // Start the background thread, if we haven't done so already. + if (!started_background_thread_) { + started_background_thread_ = true; + std::thread background_thread(PosixEnv::BackgroundThreadEntryPoint, this); + background_thread.detach(); + } + + // If the queue is empty, the background thread may be waiting for work. + if (background_work_queue_.empty()) { + background_work_cv_.Signal(); + } + + background_work_queue_.emplace(background_work_function, background_work_arg); + background_work_mutex_.Unlock(); +} + +void PosixEnv::BackgroundThreadMain() { + while (true) { + background_work_mutex_.Lock(); + + // Wait until there is work to be done. + while (background_work_queue_.empty()) { + background_work_cv_.Wait(); + } + + assert(!background_work_queue_.empty()); + auto background_work_function = background_work_queue_.front().function; + void* background_work_arg = background_work_queue_.front().arg; + background_work_queue_.pop(); + + background_work_mutex_.Unlock(); + background_work_function(background_work_arg); + } +} + +namespace { + +// Wraps an Env instance whose destructor is never created. +// +// Intended usage: +// using PlatformSingletonEnv = SingletonEnv; +// void ConfigurePosixEnv(int param) { +// PlatformSingletonEnv::AssertEnvNotInitialized(); +// // set global configuration flags. +// } +// Env* Env::Default() { +// static PlatformSingletonEnv default_env; +// return default_env.env(); +// } +template +class SingletonEnv { + public: + SingletonEnv() { +#if !defined(NDEBUG) + env_initialized_.store(true, std::memory_order_relaxed); +#endif // !defined(NDEBUG) + static_assert(sizeof(env_storage_) >= sizeof(EnvType), + "env_storage_ will not fit the Env"); + static_assert(alignof(decltype(env_storage_)) >= alignof(EnvType), + "env_storage_ does not meet the Env's alignment needs"); + new (&env_storage_) EnvType(); + } + ~SingletonEnv() = default; + + SingletonEnv(const SingletonEnv&) = delete; + SingletonEnv& operator=(const SingletonEnv&) = delete; + + Env* env() { return reinterpret_cast(&env_storage_); } + + static void AssertEnvNotInitialized() { +#if !defined(NDEBUG) + assert(!env_initialized_.load(std::memory_order_relaxed)); +#endif // !defined(NDEBUG) + } + + private: + typename std::aligned_storage::type + env_storage_; +#if !defined(NDEBUG) + static std::atomic env_initialized_; +#endif // !defined(NDEBUG) +}; + +#if !defined(NDEBUG) +template +std::atomic SingletonEnv::env_initialized_; +#endif // !defined(NDEBUG) + +using PosixDefaultEnv = SingletonEnv; + +} // namespace + +void EnvPosixTestHelper::SetReadOnlyFDLimit(int limit) { + PosixDefaultEnv::AssertEnvNotInitialized(); + g_open_read_only_file_limit = limit; +} + +void EnvPosixTestHelper::SetReadOnlyMMapLimit(int limit) { + PosixDefaultEnv::AssertEnvNotInitialized(); + g_mmap_limit = limit; +} + +Env* Env::Default() { + static PosixDefaultEnv env_container; + return env_container.env(); +} + +} // namespace leveldb diff --git a/leveldb/util/env_posix_test.cc b/leveldb/util/env_posix_test.cc new file mode 100644 index 000000000..34bda62c3 --- /dev/null +++ b/leveldb/util/env_posix_test.cc @@ -0,0 +1,353 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "leveldb/env.h" +#include "port/port.h" +#include "util/env_posix_test_helper.h" +#include "util/testutil.h" + +#if HAVE_O_CLOEXEC + +namespace { + +// Exit codes for the helper process spawned by TestCloseOnExec* tests. +// Useful for debugging test failures. +constexpr int kTextCloseOnExecHelperExecFailedCode = 61; +constexpr int kTextCloseOnExecHelperDup2FailedCode = 62; +constexpr int kTextCloseOnExecHelperFoundOpenFdCode = 63; + +// Global set by main() and read in TestCloseOnExec. +// +// The argv[0] value is stored in a std::vector instead of a std::string because +// std::string does not return a mutable pointer to its buffer until C++17. +// +// The vector stores the string pointed to by argv[0], plus the trailing null. +std::vector* GetArgvZero() { + static std::vector program_name; + return &program_name; +} + +// Command-line switch used to run this test as the CloseOnExecSwitch helper. +static const char kTestCloseOnExecSwitch[] = "--test-close-on-exec-helper"; + +// Executed in a separate process by TestCloseOnExec* tests. +// +// main() delegates to this function when the test executable is launched with +// a special command-line switch. TestCloseOnExec* tests fork()+exec() the test +// executable and pass the special command-line switch. +// + +// main() delegates to this function when the test executable is launched with +// a special command-line switch. TestCloseOnExec* tests fork()+exec() the test +// executable and pass the special command-line switch. +// +// When main() delegates to this function, the process probes whether a given +// file descriptor is open, and communicates the result via its exit code. +int TestCloseOnExecHelperMain(char* pid_arg) { + int fd = std::atoi(pid_arg); + // When given the same file descriptor twice, dup2() returns -1 if the + // file descriptor is closed, or the given file descriptor if it is open. + if (::dup2(fd, fd) == fd) { + std::fprintf(stderr, "Unexpected open fd %d\n", fd); + return kTextCloseOnExecHelperFoundOpenFdCode; + } + // Double-check that dup2() is saying the file descriptor is closed. + if (errno != EBADF) { + std::fprintf(stderr, "Unexpected errno after calling dup2 on fd %d: %s\n", + fd, std::strerror(errno)); + return kTextCloseOnExecHelperDup2FailedCode; + } + return 0; +} + +// File descriptors are small non-negative integers. +// +// Returns void so the implementation can use ASSERT_EQ. +void GetMaxFileDescriptor(int* result_fd) { + // Get the maximum file descriptor number. + ::rlimit fd_rlimit; + ASSERT_EQ(0, ::getrlimit(RLIMIT_NOFILE, &fd_rlimit)); + *result_fd = fd_rlimit.rlim_cur; +} + +// Iterates through all possible FDs and returns the currently open ones. +// +// Returns void so the implementation can use ASSERT_EQ. +void GetOpenFileDescriptors(std::unordered_set* open_fds) { + int max_fd = 0; + GetMaxFileDescriptor(&max_fd); + + for (int fd = 0; fd < max_fd; ++fd) { + if (::dup2(fd, fd) != fd) { + // When given the same file descriptor twice, dup2() returns -1 if the + // file descriptor is closed, or the given file descriptor if it is open. + // + // Double-check that dup2() is saying the fd is closed. + ASSERT_EQ(EBADF, errno) + << "dup2() should set errno to EBADF on closed file descriptors"; + continue; + } + open_fds->insert(fd); + } +} + +// Finds an FD open since a previous call to GetOpenFileDescriptors(). +// +// |baseline_open_fds| is the result of a previous GetOpenFileDescriptors() +// call. Assumes that exactly one FD was opened since that call. +// +// Returns void so the implementation can use ASSERT_EQ. +void GetNewlyOpenedFileDescriptor( + const std::unordered_set& baseline_open_fds, int* result_fd) { + std::unordered_set open_fds; + GetOpenFileDescriptors(&open_fds); + for (int fd : baseline_open_fds) { + ASSERT_EQ(1, open_fds.count(fd)) + << "Previously opened file descriptor was closed during test setup"; + open_fds.erase(fd); + } + ASSERT_EQ(1, open_fds.size()) + << "Expected exactly one newly opened file descriptor during test setup"; + *result_fd = *open_fds.begin(); +} + +// Check that a fork()+exec()-ed child process does not have an extra open FD. +void CheckCloseOnExecDoesNotLeakFDs( + const std::unordered_set& baseline_open_fds) { + // Prepare the argument list for the child process. + // execv() wants mutable buffers. + char switch_buffer[sizeof(kTestCloseOnExecSwitch)]; + std::memcpy(switch_buffer, kTestCloseOnExecSwitch, + sizeof(kTestCloseOnExecSwitch)); + + int probed_fd; + GetNewlyOpenedFileDescriptor(baseline_open_fds, &probed_fd); + std::string fd_string = std::to_string(probed_fd); + std::vector fd_buffer(fd_string.begin(), fd_string.end()); + fd_buffer.emplace_back('\0'); + + // The helper process is launched with the command below. + // env_posix_tests --test-close-on-exec-helper 3 + char* child_argv[] = {GetArgvZero()->data(), switch_buffer, fd_buffer.data(), + nullptr}; + + constexpr int kForkInChildProcessReturnValue = 0; + int child_pid = fork(); + if (child_pid == kForkInChildProcessReturnValue) { + ::execv(child_argv[0], child_argv); + std::fprintf(stderr, "Error spawning child process: %s\n", strerror(errno)); + std::exit(kTextCloseOnExecHelperExecFailedCode); + } + + int child_status = 0; + ASSERT_EQ(child_pid, ::waitpid(child_pid, &child_status, 0)); + ASSERT_TRUE(WIFEXITED(child_status)) + << "The helper process did not exit with an exit code"; + ASSERT_EQ(0, WEXITSTATUS(child_status)) + << "The helper process encountered an error"; +} + +} // namespace + +#endif // HAVE_O_CLOEXEC + +namespace leveldb { + +static const int kReadOnlyFileLimit = 4; +static const int kMMapLimit = 4; + +class EnvPosixTest : public testing::Test { + public: + static void SetFileLimits(int read_only_file_limit, int mmap_limit) { + EnvPosixTestHelper::SetReadOnlyFDLimit(read_only_file_limit); + EnvPosixTestHelper::SetReadOnlyMMapLimit(mmap_limit); + } + + EnvPosixTest() : env_(Env::Default()) {} + + Env* env_; +}; + +TEST_F(EnvPosixTest, TestOpenOnRead) { + // Write some test data to a single file that will be opened |n| times. + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string test_file = test_dir + "/open_on_read.txt"; + + FILE* f = std::fopen(test_file.c_str(), "we"); + ASSERT_TRUE(f != nullptr); + const char kFileData[] = "abcdefghijklmnopqrstuvwxyz"; + fputs(kFileData, f); + std::fclose(f); + + // Open test file some number above the sum of the two limits to force + // open-on-read behavior of POSIX Env leveldb::RandomAccessFile. + const int kNumFiles = kReadOnlyFileLimit + kMMapLimit + 5; + leveldb::RandomAccessFile* files[kNumFiles] = {0}; + for (int i = 0; i < kNumFiles; i++) { + ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(test_file, &files[i])); + } + char scratch; + Slice read_result; + for (int i = 0; i < kNumFiles; i++) { + ASSERT_LEVELDB_OK(files[i]->Read(i, 1, &read_result, &scratch)); + ASSERT_EQ(kFileData[i], read_result[0]); + } + for (int i = 0; i < kNumFiles; i++) { + delete files[i]; + } + ASSERT_LEVELDB_OK(env_->RemoveFile(test_file)); +} + +#if HAVE_O_CLOEXEC + +TEST_F(EnvPosixTest, TestCloseOnExecSequentialFile) { + std::unordered_set open_fds; + GetOpenFileDescriptors(&open_fds); + + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string file_path = test_dir + "/close_on_exec_sequential.txt"; + ASSERT_LEVELDB_OK(WriteStringToFile(env_, "0123456789", file_path)); + + leveldb::SequentialFile* file = nullptr; + ASSERT_LEVELDB_OK(env_->NewSequentialFile(file_path, &file)); + CheckCloseOnExecDoesNotLeakFDs(open_fds); + delete file; + + ASSERT_LEVELDB_OK(env_->RemoveFile(file_path)); +} + +TEST_F(EnvPosixTest, TestCloseOnExecRandomAccessFile) { + std::unordered_set open_fds; + GetOpenFileDescriptors(&open_fds); + + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string file_path = test_dir + "/close_on_exec_random_access.txt"; + ASSERT_LEVELDB_OK(WriteStringToFile(env_, "0123456789", file_path)); + + // Exhaust the RandomAccessFile mmap limit. This way, the test + // RandomAccessFile instance below is backed by a file descriptor, not by an + // mmap region. + leveldb::RandomAccessFile* mmapped_files[kMMapLimit]; + for (int i = 0; i < kMMapLimit; i++) { + ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(file_path, &mmapped_files[i])); + } + + leveldb::RandomAccessFile* file = nullptr; + ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(file_path, &file)); + CheckCloseOnExecDoesNotLeakFDs(open_fds); + delete file; + + for (int i = 0; i < kMMapLimit; i++) { + delete mmapped_files[i]; + } + ASSERT_LEVELDB_OK(env_->RemoveFile(file_path)); +} + +TEST_F(EnvPosixTest, TestCloseOnExecWritableFile) { + std::unordered_set open_fds; + GetOpenFileDescriptors(&open_fds); + + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string file_path = test_dir + "/close_on_exec_writable.txt"; + ASSERT_LEVELDB_OK(WriteStringToFile(env_, "0123456789", file_path)); + + leveldb::WritableFile* file = nullptr; + ASSERT_LEVELDB_OK(env_->NewWritableFile(file_path, &file)); + CheckCloseOnExecDoesNotLeakFDs(open_fds); + delete file; + + ASSERT_LEVELDB_OK(env_->RemoveFile(file_path)); +} + +TEST_F(EnvPosixTest, TestCloseOnExecAppendableFile) { + std::unordered_set open_fds; + GetOpenFileDescriptors(&open_fds); + + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string file_path = test_dir + "/close_on_exec_appendable.txt"; + ASSERT_LEVELDB_OK(WriteStringToFile(env_, "0123456789", file_path)); + + leveldb::WritableFile* file = nullptr; + ASSERT_LEVELDB_OK(env_->NewAppendableFile(file_path, &file)); + CheckCloseOnExecDoesNotLeakFDs(open_fds); + delete file; + + ASSERT_LEVELDB_OK(env_->RemoveFile(file_path)); +} + +TEST_F(EnvPosixTest, TestCloseOnExecLockFile) { + std::unordered_set open_fds; + GetOpenFileDescriptors(&open_fds); + + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string file_path = test_dir + "/close_on_exec_lock.txt"; + ASSERT_LEVELDB_OK(WriteStringToFile(env_, "0123456789", file_path)); + + leveldb::FileLock* lock = nullptr; + ASSERT_LEVELDB_OK(env_->LockFile(file_path, &lock)); + CheckCloseOnExecDoesNotLeakFDs(open_fds); + ASSERT_LEVELDB_OK(env_->UnlockFile(lock)); + + ASSERT_LEVELDB_OK(env_->RemoveFile(file_path)); +} + +TEST_F(EnvPosixTest, TestCloseOnExecLogger) { + std::unordered_set open_fds; + GetOpenFileDescriptors(&open_fds); + + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string file_path = test_dir + "/close_on_exec_logger.txt"; + ASSERT_LEVELDB_OK(WriteStringToFile(env_, "0123456789", file_path)); + + leveldb::Logger* file = nullptr; + ASSERT_LEVELDB_OK(env_->NewLogger(file_path, &file)); + CheckCloseOnExecDoesNotLeakFDs(open_fds); + delete file; + + ASSERT_LEVELDB_OK(env_->RemoveFile(file_path)); +} + +#endif // HAVE_O_CLOEXEC + +} // namespace leveldb + +int main(int argc, char** argv) { +#if HAVE_O_CLOEXEC + // Check if we're invoked as a helper program, or as the test suite. + for (int i = 1; i < argc; ++i) { + if (!std::strcmp(argv[i], kTestCloseOnExecSwitch)) { + return TestCloseOnExecHelperMain(argv[i + 1]); + } + } + + // Save argv[0] early, because googletest may modify argv. + GetArgvZero()->assign(argv[0], argv[0] + std::strlen(argv[0]) + 1); +#endif // HAVE_O_CLOEXEC + + // All tests currently run with the same read-only file limits. + leveldb::EnvPosixTest::SetFileLimits(leveldb::kReadOnlyFileLimit, + leveldb::kMMapLimit); + + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/leveldb/util/env_posix_test_helper.h b/leveldb/util/env_posix_test_helper.h new file mode 100644 index 000000000..038696059 --- /dev/null +++ b/leveldb/util/env_posix_test_helper.h @@ -0,0 +1,28 @@ +// Copyright 2017 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ +#define STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ + +namespace leveldb { + +class EnvPosixTest; + +// A helper for the POSIX Env to facilitate testing. +class EnvPosixTestHelper { + private: + friend class EnvPosixTest; + + // Set the maximum number of read-only files that will be opened. + // Must be called before creating an Env. + static void SetReadOnlyFDLimit(int limit); + + // Set the maximum number of read-only files that will be mapped via mmap. + // Must be called before creating an Env. + static void SetReadOnlyMMapLimit(int limit); +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ diff --git a/leveldb/util/env_test.cc b/leveldb/util/env_test.cc new file mode 100644 index 000000000..e5c30fec9 --- /dev/null +++ b/leveldb/util/env_test.cc @@ -0,0 +1,248 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/env.h" + +#include + +#include "gtest/gtest.h" +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/mutexlock.h" +#include "util/testutil.h" + +namespace leveldb { + +class EnvTest : public testing::Test { + public: + EnvTest() : env_(Env::Default()) {} + + Env* env_; +}; + +TEST_F(EnvTest, ReadWrite) { + Random rnd(test::RandomSeed()); + + // Get file to use for testing. + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string test_file_name = test_dir + "/open_on_read.txt"; + WritableFile* writable_file; + ASSERT_LEVELDB_OK(env_->NewWritableFile(test_file_name, &writable_file)); + + // Fill a file with data generated via a sequence of randomly sized writes. + static const size_t kDataSize = 10 * 1048576; + std::string data; + while (data.size() < kDataSize) { + int len = rnd.Skewed(18); // Up to 2^18 - 1, but typically much smaller + std::string r; + test::RandomString(&rnd, len, &r); + ASSERT_LEVELDB_OK(writable_file->Append(r)); + data += r; + if (rnd.OneIn(10)) { + ASSERT_LEVELDB_OK(writable_file->Flush()); + } + } + ASSERT_LEVELDB_OK(writable_file->Sync()); + ASSERT_LEVELDB_OK(writable_file->Close()); + delete writable_file; + + // Read all data using a sequence of randomly sized reads. + SequentialFile* sequential_file; + ASSERT_LEVELDB_OK(env_->NewSequentialFile(test_file_name, &sequential_file)); + std::string read_result; + std::string scratch; + while (read_result.size() < data.size()) { + int len = std::min(rnd.Skewed(18), data.size() - read_result.size()); + scratch.resize(std::max(len, 1)); // at least 1 so &scratch[0] is legal + Slice read; + ASSERT_LEVELDB_OK(sequential_file->Read(len, &read, &scratch[0])); + if (len > 0) { + ASSERT_GT(read.size(), 0); + } + ASSERT_LE(read.size(), len); + read_result.append(read.data(), read.size()); + } + ASSERT_EQ(read_result, data); + delete sequential_file; +} + +TEST_F(EnvTest, RunImmediately) { + struct RunState { + port::Mutex mu; + port::CondVar cvar{&mu}; + bool called = false; + + static void Run(void* arg) { + RunState* state = reinterpret_cast(arg); + MutexLock l(&state->mu); + ASSERT_EQ(state->called, false); + state->called = true; + state->cvar.Signal(); + } + }; + + RunState state; + env_->Schedule(&RunState::Run, &state); + + MutexLock l(&state.mu); + while (!state.called) { + state.cvar.Wait(); + } +} + +TEST_F(EnvTest, RunMany) { + struct RunState { + port::Mutex mu; + port::CondVar cvar{&mu}; + int run_count = 0; + }; + + struct Callback { + RunState* const state_; // Pointer to shared state. + bool run = false; + + Callback(RunState* s) : state_(s) {} + + static void Run(void* arg) { + Callback* callback = reinterpret_cast(arg); + RunState* state = callback->state_; + + MutexLock l(&state->mu); + state->run_count++; + callback->run = true; + state->cvar.Signal(); + } + }; + + RunState state; + Callback callback1(&state); + Callback callback2(&state); + Callback callback3(&state); + Callback callback4(&state); + env_->Schedule(&Callback::Run, &callback1); + env_->Schedule(&Callback::Run, &callback2); + env_->Schedule(&Callback::Run, &callback3); + env_->Schedule(&Callback::Run, &callback4); + + MutexLock l(&state.mu); + while (state.run_count != 4) { + state.cvar.Wait(); + } + + ASSERT_TRUE(callback1.run); + ASSERT_TRUE(callback2.run); + ASSERT_TRUE(callback3.run); + ASSERT_TRUE(callback4.run); +} + +struct State { + port::Mutex mu; + port::CondVar cvar{&mu}; + + int val GUARDED_BY(mu); + int num_running GUARDED_BY(mu); + + State(int val, int num_running) : val(val), num_running(num_running) {} +}; + +static void ThreadBody(void* arg) { + State* s = reinterpret_cast(arg); + s->mu.Lock(); + s->val += 1; + s->num_running -= 1; + s->cvar.Signal(); + s->mu.Unlock(); +} + +TEST_F(EnvTest, StartThread) { + State state(0, 3); + for (int i = 0; i < 3; i++) { + env_->StartThread(&ThreadBody, &state); + } + + MutexLock l(&state.mu); + while (state.num_running != 0) { + state.cvar.Wait(); + } + ASSERT_EQ(state.val, 3); +} + +TEST_F(EnvTest, TestOpenNonExistentFile) { + // Write some test data to a single file that will be opened |n| times. + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + + std::string non_existent_file = test_dir + "/non_existent_file"; + ASSERT_TRUE(!env_->FileExists(non_existent_file)); + + RandomAccessFile* random_access_file; + Status status = + env_->NewRandomAccessFile(non_existent_file, &random_access_file); +#if defined(LEVELDB_PLATFORM_CHROMIUM) + // TODO(crbug.com/760362): See comment in MakeIOError() from env_chromium.cc. + ASSERT_TRUE(status.IsIOError()); +#else + ASSERT_TRUE(status.IsNotFound()); +#endif // defined(LEVELDB_PLATFORM_CHROMIUM) + + SequentialFile* sequential_file; + status = env_->NewSequentialFile(non_existent_file, &sequential_file); +#if defined(LEVELDB_PLATFORM_CHROMIUM) + // TODO(crbug.com/760362): See comment in MakeIOError() from env_chromium.cc. + ASSERT_TRUE(status.IsIOError()); +#else + ASSERT_TRUE(status.IsNotFound()); +#endif // defined(LEVELDB_PLATFORM_CHROMIUM) +} + +TEST_F(EnvTest, ReopenWritableFile) { + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string test_file_name = test_dir + "/reopen_writable_file.txt"; + env_->RemoveFile(test_file_name); + + WritableFile* writable_file; + ASSERT_LEVELDB_OK(env_->NewWritableFile(test_file_name, &writable_file)); + std::string data("hello world!"); + ASSERT_LEVELDB_OK(writable_file->Append(data)); + ASSERT_LEVELDB_OK(writable_file->Close()); + delete writable_file; + + ASSERT_LEVELDB_OK(env_->NewWritableFile(test_file_name, &writable_file)); + data = "42"; + ASSERT_LEVELDB_OK(writable_file->Append(data)); + ASSERT_LEVELDB_OK(writable_file->Close()); + delete writable_file; + + ASSERT_LEVELDB_OK(ReadFileToString(env_, test_file_name, &data)); + ASSERT_EQ(std::string("42"), data); + env_->RemoveFile(test_file_name); +} + +TEST_F(EnvTest, ReopenAppendableFile) { + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string test_file_name = test_dir + "/reopen_appendable_file.txt"; + env_->RemoveFile(test_file_name); + + WritableFile* appendable_file; + ASSERT_LEVELDB_OK(env_->NewAppendableFile(test_file_name, &appendable_file)); + std::string data("hello world!"); + ASSERT_LEVELDB_OK(appendable_file->Append(data)); + ASSERT_LEVELDB_OK(appendable_file->Close()); + delete appendable_file; + + ASSERT_LEVELDB_OK(env_->NewAppendableFile(test_file_name, &appendable_file)); + data = "42"; + ASSERT_LEVELDB_OK(appendable_file->Append(data)); + ASSERT_LEVELDB_OK(appendable_file->Close()); + delete appendable_file; + + ASSERT_LEVELDB_OK(ReadFileToString(env_, test_file_name, &data)); + ASSERT_EQ(std::string("hello world!42"), data); + env_->RemoveFile(test_file_name); +} + +} // namespace leveldb diff --git a/leveldb/util/env_windows.cc b/leveldb/util/env_windows.cc new file mode 100644 index 000000000..1c74b02a7 --- /dev/null +++ b/leveldb/util/env_windows.cc @@ -0,0 +1,816 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Prevent Windows headers from defining min/max macros and instead +// use STL. +#ifndef NOMINMAX +#define NOMINMAX +#endif // ifndef NOMINMAX +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "leveldb/env.h" +#include "leveldb/slice.h" +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/env_windows_test_helper.h" +#include "util/logging.h" +#include "util/mutexlock.h" +#include "util/windows_logger.h" + +namespace leveldb { + +namespace { + +constexpr const size_t kWritableFileBufferSize = 65536; + +// Up to 1000 mmaps for 64-bit binaries; none for 32-bit. +constexpr int kDefaultMmapLimit = (sizeof(void*) >= 8) ? 1000 : 0; + +// Can be set by by EnvWindowsTestHelper::SetReadOnlyMMapLimit(). +int g_mmap_limit = kDefaultMmapLimit; + +std::string GetWindowsErrorMessage(DWORD error_code) { + std::string message; + char* error_text = nullptr; + // Use MBCS version of FormatMessage to match return value. + size_t error_text_size = ::FormatMessageA( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_IGNORE_INSERTS, + nullptr, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + reinterpret_cast(&error_text), 0, nullptr); + if (!error_text) { + return message; + } + message.assign(error_text, error_text_size); + ::LocalFree(error_text); + return message; +} + +Status WindowsError(const std::string& context, DWORD error_code) { + if (error_code == ERROR_FILE_NOT_FOUND || error_code == ERROR_PATH_NOT_FOUND) + return Status::NotFound(context, GetWindowsErrorMessage(error_code)); + return Status::IOError(context, GetWindowsErrorMessage(error_code)); +} + +class ScopedHandle { + public: + ScopedHandle(HANDLE handle) : handle_(handle) {} + ScopedHandle(const ScopedHandle&) = delete; + ScopedHandle(ScopedHandle&& other) noexcept : handle_(other.Release()) {} + ~ScopedHandle() { Close(); } + + ScopedHandle& operator=(const ScopedHandle&) = delete; + + ScopedHandle& operator=(ScopedHandle&& rhs) noexcept { + if (this != &rhs) handle_ = rhs.Release(); + return *this; + } + + bool Close() { + if (!is_valid()) { + return true; + } + HANDLE h = handle_; + handle_ = INVALID_HANDLE_VALUE; + return ::CloseHandle(h); + } + + bool is_valid() const { + return handle_ != INVALID_HANDLE_VALUE && handle_ != nullptr; + } + + HANDLE get() const { return handle_; } + + HANDLE Release() { + HANDLE h = handle_; + handle_ = INVALID_HANDLE_VALUE; + return h; + } + + private: + HANDLE handle_; +}; + +// Helper class to limit resource usage to avoid exhaustion. +// Currently used to limit read-only file descriptors and mmap file usage +// so that we do not run out of file descriptors or virtual memory, or run into +// kernel performance problems for very large databases. +class Limiter { + public: + // Limit maximum number of resources to |max_acquires|. + Limiter(int max_acquires) + : +#if !defined(NDEBUG) + max_acquires_(max_acquires), +#endif // !defined(NDEBUG) + acquires_allowed_(max_acquires) { + assert(max_acquires >= 0); + } + + Limiter(const Limiter&) = delete; + Limiter operator=(const Limiter&) = delete; + + // If another resource is available, acquire it and return true. + // Else return false. + bool Acquire() { + int old_acquires_allowed = + acquires_allowed_.fetch_sub(1, std::memory_order_relaxed); + + if (old_acquires_allowed > 0) return true; + + acquires_allowed_.fetch_add(1, std::memory_order_relaxed); + return false; + } + + // Release a resource acquired by a previous call to Acquire() that returned + // true. + void Release() { + int old_acquires_allowed = + acquires_allowed_.fetch_add(1, std::memory_order_relaxed); + + // Silence compiler warnings about unused arguments when NDEBUG is defined. + (void)old_acquires_allowed; + // If the check below fails, Release() was called more times than acquire. + assert(old_acquires_allowed < max_acquires_); + } + + private: +#if !defined(NDEBUG) + // Catches an excessive number of Release() calls. + const int max_acquires_; +#endif // !defined(NDEBUG) + + // The number of available resources. + // + // This is a counter and is not tied to the invariants of any other class, so + // it can be operated on safely using std::memory_order_relaxed. + std::atomic acquires_allowed_; +}; + +class WindowsSequentialFile : public SequentialFile { + public: + WindowsSequentialFile(std::string filename, ScopedHandle handle) + : handle_(std::move(handle)), filename_(std::move(filename)) {} + ~WindowsSequentialFile() override {} + + Status Read(size_t n, Slice* result, char* scratch) override { + DWORD bytes_read; + // DWORD is 32-bit, but size_t could technically be larger. However leveldb + // files are limited to leveldb::Options::max_file_size which is clamped to + // 1<<30 or 1 GiB. + assert(n <= std::numeric_limits::max()); + if (!::ReadFile(handle_.get(), scratch, static_cast(n), &bytes_read, + nullptr)) { + return WindowsError(filename_, ::GetLastError()); + } + + *result = Slice(scratch, bytes_read); + return Status::OK(); + } + + Status Skip(uint64_t n) override { + LARGE_INTEGER distance; + distance.QuadPart = n; + if (!::SetFilePointerEx(handle_.get(), distance, nullptr, FILE_CURRENT)) { + return WindowsError(filename_, ::GetLastError()); + } + return Status::OK(); + } + + private: + const ScopedHandle handle_; + const std::string filename_; +}; + +class WindowsRandomAccessFile : public RandomAccessFile { + public: + WindowsRandomAccessFile(std::string filename, ScopedHandle handle) + : handle_(std::move(handle)), filename_(std::move(filename)) {} + + ~WindowsRandomAccessFile() override = default; + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + DWORD bytes_read = 0; + OVERLAPPED overlapped = {0}; + + overlapped.OffsetHigh = static_cast(offset >> 32); + overlapped.Offset = static_cast(offset); + if (!::ReadFile(handle_.get(), scratch, static_cast(n), &bytes_read, + &overlapped)) { + DWORD error_code = ::GetLastError(); + if (error_code != ERROR_HANDLE_EOF) { + *result = Slice(scratch, 0); + return Status::IOError(filename_, GetWindowsErrorMessage(error_code)); + } + } + + *result = Slice(scratch, bytes_read); + return Status::OK(); + } + + private: + const ScopedHandle handle_; + const std::string filename_; +}; + +class WindowsMmapReadableFile : public RandomAccessFile { + public: + // base[0,length-1] contains the mmapped contents of the file. + WindowsMmapReadableFile(std::string filename, char* mmap_base, size_t length, + Limiter* mmap_limiter) + : mmap_base_(mmap_base), + length_(length), + mmap_limiter_(mmap_limiter), + filename_(std::move(filename)) {} + + ~WindowsMmapReadableFile() override { + ::UnmapViewOfFile(mmap_base_); + mmap_limiter_->Release(); + } + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + if (offset + n > length_) { + *result = Slice(); + return WindowsError(filename_, ERROR_INVALID_PARAMETER); + } + + *result = Slice(mmap_base_ + offset, n); + return Status::OK(); + } + + private: + char* const mmap_base_; + const size_t length_; + Limiter* const mmap_limiter_; + const std::string filename_; +}; + +class WindowsWritableFile : public WritableFile { + public: + WindowsWritableFile(std::string filename, ScopedHandle handle) + : pos_(0), handle_(std::move(handle)), filename_(std::move(filename)) {} + + ~WindowsWritableFile() override = default; + + Status Append(const Slice& data) override { + size_t write_size = data.size(); + const char* write_data = data.data(); + + // Fit as much as possible into buffer. + size_t copy_size = std::min(write_size, kWritableFileBufferSize - pos_); + std::memcpy(buf_ + pos_, write_data, copy_size); + write_data += copy_size; + write_size -= copy_size; + pos_ += copy_size; + if (write_size == 0) { + return Status::OK(); + } + + // Can't fit in buffer, so need to do at least one write. + Status status = FlushBuffer(); + if (!status.ok()) { + return status; + } + + // Small writes go to buffer, large writes are written directly. + if (write_size < kWritableFileBufferSize) { + std::memcpy(buf_, write_data, write_size); + pos_ = write_size; + return Status::OK(); + } + return WriteUnbuffered(write_data, write_size); + } + + Status Close() override { + Status status = FlushBuffer(); + if (!handle_.Close() && status.ok()) { + status = WindowsError(filename_, ::GetLastError()); + } + return status; + } + + Status Flush() override { return FlushBuffer(); } + + Status Sync() override { + // On Windows no need to sync parent directory. Its metadata will be updated + // via the creation of the new file, without an explicit sync. + + Status status = FlushBuffer(); + if (!status.ok()) { + return status; + } + + if (!::FlushFileBuffers(handle_.get())) { + return Status::IOError(filename_, + GetWindowsErrorMessage(::GetLastError())); + } + return Status::OK(); + } + + private: + Status FlushBuffer() { + Status status = WriteUnbuffered(buf_, pos_); + pos_ = 0; + return status; + } + + Status WriteUnbuffered(const char* data, size_t size) { + DWORD bytes_written; + if (!::WriteFile(handle_.get(), data, static_cast(size), + &bytes_written, nullptr)) { + return Status::IOError(filename_, + GetWindowsErrorMessage(::GetLastError())); + } + return Status::OK(); + } + + // buf_[0, pos_-1] contains data to be written to handle_. + char buf_[kWritableFileBufferSize]; + size_t pos_; + + ScopedHandle handle_; + const std::string filename_; +}; + +// Lock or unlock the entire file as specified by |lock|. Returns true +// when successful, false upon failure. Caller should call ::GetLastError() +// to determine cause of failure +bool LockOrUnlock(HANDLE handle, bool lock) { + if (lock) { + return ::LockFile(handle, + /*dwFileOffsetLow=*/0, /*dwFileOffsetHigh=*/0, + /*nNumberOfBytesToLockLow=*/MAXDWORD, + /*nNumberOfBytesToLockHigh=*/MAXDWORD); + } else { + return ::UnlockFile(handle, + /*dwFileOffsetLow=*/0, /*dwFileOffsetHigh=*/0, + /*nNumberOfBytesToLockLow=*/MAXDWORD, + /*nNumberOfBytesToLockHigh=*/MAXDWORD); + } +} + +class WindowsFileLock : public FileLock { + public: + WindowsFileLock(ScopedHandle handle, std::string filename) + : handle_(std::move(handle)), filename_(std::move(filename)) {} + + const ScopedHandle& handle() const { return handle_; } + const std::string& filename() const { return filename_; } + + private: + const ScopedHandle handle_; + const std::string filename_; +}; + +class WindowsEnv : public Env { + public: + WindowsEnv(); + ~WindowsEnv() override { + static const char msg[] = + "WindowsEnv singleton destroyed. Unsupported behavior!\n"; + std::fwrite(msg, 1, sizeof(msg), stderr); + std::abort(); + } + + Status NewSequentialFile(const std::string& filename, + SequentialFile** result) override { + *result = nullptr; + DWORD desired_access = GENERIC_READ; + DWORD share_mode = FILE_SHARE_READ; + ScopedHandle handle = ::CreateFileA( + filename.c_str(), desired_access, share_mode, + /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/nullptr); + if (!handle.is_valid()) { + return WindowsError(filename, ::GetLastError()); + } + + *result = new WindowsSequentialFile(filename, std::move(handle)); + return Status::OK(); + } + + Status NewRandomAccessFile(const std::string& filename, + RandomAccessFile** result) override { + *result = nullptr; + DWORD desired_access = GENERIC_READ; + DWORD share_mode = FILE_SHARE_READ; + ScopedHandle handle = + ::CreateFileA(filename.c_str(), desired_access, share_mode, + /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_READONLY, + /*hTemplateFile=*/nullptr); + if (!handle.is_valid()) { + return WindowsError(filename, ::GetLastError()); + } + if (!mmap_limiter_.Acquire()) { + *result = new WindowsRandomAccessFile(filename, std::move(handle)); + return Status::OK(); + } + + LARGE_INTEGER file_size; + Status status; + if (!::GetFileSizeEx(handle.get(), &file_size)) { + mmap_limiter_.Release(); + return WindowsError(filename, ::GetLastError()); + } + + ScopedHandle mapping = + ::CreateFileMappingA(handle.get(), + /*security attributes=*/nullptr, PAGE_READONLY, + /*dwMaximumSizeHigh=*/0, + /*dwMaximumSizeLow=*/0, + /*lpName=*/nullptr); + if (mapping.is_valid()) { + void* mmap_base = ::MapViewOfFile(mapping.get(), FILE_MAP_READ, + /*dwFileOffsetHigh=*/0, + /*dwFileOffsetLow=*/0, + /*dwNumberOfBytesToMap=*/0); + if (mmap_base) { + *result = new WindowsMmapReadableFile( + filename, reinterpret_cast(mmap_base), + static_cast(file_size.QuadPart), &mmap_limiter_); + return Status::OK(); + } + } + mmap_limiter_.Release(); + return WindowsError(filename, ::GetLastError()); + } + + Status NewWritableFile(const std::string& filename, + WritableFile** result) override { + DWORD desired_access = GENERIC_WRITE; + DWORD share_mode = 0; // Exclusive access. + ScopedHandle handle = ::CreateFileA( + filename.c_str(), desired_access, share_mode, + /*lpSecurityAttributes=*/nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/nullptr); + if (!handle.is_valid()) { + *result = nullptr; + return WindowsError(filename, ::GetLastError()); + } + + *result = new WindowsWritableFile(filename, std::move(handle)); + return Status::OK(); + } + + Status NewAppendableFile(const std::string& filename, + WritableFile** result) override { + DWORD desired_access = FILE_APPEND_DATA; + DWORD share_mode = 0; // Exclusive access. + ScopedHandle handle = ::CreateFileA( + filename.c_str(), desired_access, share_mode, + /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/nullptr); + if (!handle.is_valid()) { + *result = nullptr; + return WindowsError(filename, ::GetLastError()); + } + + *result = new WindowsWritableFile(filename, std::move(handle)); + return Status::OK(); + } + + bool FileExists(const std::string& filename) override { + return GetFileAttributesA(filename.c_str()) != INVALID_FILE_ATTRIBUTES; + } + + Status GetChildren(const std::string& directory_path, + std::vector* result) override { + const std::string find_pattern = directory_path + "\\*"; + WIN32_FIND_DATAA find_data; + HANDLE dir_handle = ::FindFirstFileA(find_pattern.c_str(), &find_data); + if (dir_handle == INVALID_HANDLE_VALUE) { + DWORD last_error = ::GetLastError(); + if (last_error == ERROR_FILE_NOT_FOUND) { + return Status::OK(); + } + return WindowsError(directory_path, last_error); + } + do { + char base_name[_MAX_FNAME]; + char ext[_MAX_EXT]; + + if (!_splitpath_s(find_data.cFileName, nullptr, 0, nullptr, 0, base_name, + ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) { + result->emplace_back(std::string(base_name) + ext); + } + } while (::FindNextFileA(dir_handle, &find_data)); + DWORD last_error = ::GetLastError(); + ::FindClose(dir_handle); + if (last_error != ERROR_NO_MORE_FILES) { + return WindowsError(directory_path, last_error); + } + return Status::OK(); + } + + Status RemoveFile(const std::string& filename) override { + if (!::DeleteFileA(filename.c_str())) { + return WindowsError(filename, ::GetLastError()); + } + return Status::OK(); + } + + Status CreateDir(const std::string& dirname) override { + if (!::CreateDirectoryA(dirname.c_str(), nullptr)) { + return WindowsError(dirname, ::GetLastError()); + } + return Status::OK(); + } + + Status RemoveDir(const std::string& dirname) override { + if (!::RemoveDirectoryA(dirname.c_str())) { + return WindowsError(dirname, ::GetLastError()); + } + return Status::OK(); + } + + Status GetFileSize(const std::string& filename, uint64_t* size) override { + WIN32_FILE_ATTRIBUTE_DATA file_attributes; + if (!::GetFileAttributesExA(filename.c_str(), GetFileExInfoStandard, + &file_attributes)) { + return WindowsError(filename, ::GetLastError()); + } + ULARGE_INTEGER file_size; + file_size.HighPart = file_attributes.nFileSizeHigh; + file_size.LowPart = file_attributes.nFileSizeLow; + *size = file_size.QuadPart; + return Status::OK(); + } + + Status RenameFile(const std::string& from, const std::string& to) override { + // Try a simple move first. It will only succeed when |to| doesn't already + // exist. + if (::MoveFileA(from.c_str(), to.c_str())) { + return Status::OK(); + } + DWORD move_error = ::GetLastError(); + + // Try the full-blown replace if the move fails, as ReplaceFile will only + // succeed when |to| does exist. When writing to a network share, we may not + // be able to change the ACLs. Ignore ACL errors then + // (REPLACEFILE_IGNORE_MERGE_ERRORS). + if (::ReplaceFileA(to.c_str(), from.c_str(), /*lpBackupFileName=*/nullptr, + REPLACEFILE_IGNORE_MERGE_ERRORS, + /*lpExclude=*/nullptr, /*lpReserved=*/nullptr)) { + return Status::OK(); + } + DWORD replace_error = ::GetLastError(); + // In the case of FILE_ERROR_NOT_FOUND from ReplaceFile, it is likely that + // |to| does not exist. In this case, the more relevant error comes from the + // call to MoveFile. + if (replace_error == ERROR_FILE_NOT_FOUND || + replace_error == ERROR_PATH_NOT_FOUND) { + return WindowsError(from, move_error); + } else { + return WindowsError(from, replace_error); + } + } + + Status LockFile(const std::string& filename, FileLock** lock) override { + *lock = nullptr; + Status result; + ScopedHandle handle = ::CreateFileA( + filename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, + /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, + nullptr); + if (!handle.is_valid()) { + result = WindowsError(filename, ::GetLastError()); + } else if (!LockOrUnlock(handle.get(), true)) { + result = WindowsError("lock " + filename, ::GetLastError()); + } else { + *lock = new WindowsFileLock(std::move(handle), filename); + } + return result; + } + + Status UnlockFile(FileLock* lock) override { + WindowsFileLock* windows_file_lock = + reinterpret_cast(lock); + if (!LockOrUnlock(windows_file_lock->handle().get(), false)) { + return WindowsError("unlock " + windows_file_lock->filename(), + ::GetLastError()); + } + delete windows_file_lock; + return Status::OK(); + } + + void Schedule(void (*background_work_function)(void* background_work_arg), + void* background_work_arg) override; + + void StartThread(void (*thread_main)(void* thread_main_arg), + void* thread_main_arg) override { + std::thread new_thread(thread_main, thread_main_arg); + new_thread.detach(); + } + + Status GetTestDirectory(std::string* result) override { + const char* env = getenv("TEST_TMPDIR"); + if (env && env[0] != '\0') { + *result = env; + return Status::OK(); + } + + char tmp_path[MAX_PATH]; + if (!GetTempPathA(ARRAYSIZE(tmp_path), tmp_path)) { + return WindowsError("GetTempPath", ::GetLastError()); + } + std::stringstream ss; + ss << tmp_path << "leveldbtest-" << std::this_thread::get_id(); + *result = ss.str(); + + // Directory may already exist + CreateDir(*result); + return Status::OK(); + } + + Status NewLogger(const std::string& filename, Logger** result) override { + std::FILE* fp = std::fopen(filename.c_str(), "wN"); + if (fp == nullptr) { + *result = nullptr; + return WindowsError(filename, ::GetLastError()); + } else { + *result = new WindowsLogger(fp); + return Status::OK(); + } + } + + uint64_t NowMicros() override { + // GetSystemTimeAsFileTime typically has a resolution of 10-20 msec. + // TODO(cmumford): Switch to GetSystemTimePreciseAsFileTime which is + // available in Windows 8 and later. + FILETIME ft; + ::GetSystemTimeAsFileTime(&ft); + // Each tick represents a 100-nanosecond intervals since January 1, 1601 + // (UTC). + uint64_t num_ticks = + (static_cast(ft.dwHighDateTime) << 32) + ft.dwLowDateTime; + return num_ticks / 10; + } + + void SleepForMicroseconds(int micros) override { + std::this_thread::sleep_for(std::chrono::microseconds(micros)); + } + + private: + void BackgroundThreadMain(); + + static void BackgroundThreadEntryPoint(WindowsEnv* env) { + env->BackgroundThreadMain(); + } + + // Stores the work item data in a Schedule() call. + // + // Instances are constructed on the thread calling Schedule() and used on the + // background thread. + // + // This structure is thread-safe because it is immutable. + struct BackgroundWorkItem { + explicit BackgroundWorkItem(void (*function)(void* arg), void* arg) + : function(function), arg(arg) {} + + void (*const function)(void*); + void* const arg; + }; + + port::Mutex background_work_mutex_; + port::CondVar background_work_cv_ GUARDED_BY(background_work_mutex_); + bool started_background_thread_ GUARDED_BY(background_work_mutex_); + + std::queue background_work_queue_ + GUARDED_BY(background_work_mutex_); + + Limiter mmap_limiter_; // Thread-safe. +}; + +// Return the maximum number of concurrent mmaps. +int MaxMmaps() { return g_mmap_limit; } + +WindowsEnv::WindowsEnv() + : background_work_cv_(&background_work_mutex_), + started_background_thread_(false), + mmap_limiter_(MaxMmaps()) {} + +void WindowsEnv::Schedule( + void (*background_work_function)(void* background_work_arg), + void* background_work_arg) { + background_work_mutex_.Lock(); + + // Start the background thread, if we haven't done so already. + if (!started_background_thread_) { + started_background_thread_ = true; + std::thread background_thread(WindowsEnv::BackgroundThreadEntryPoint, this); + background_thread.detach(); + } + + // If the queue is empty, the background thread may be waiting for work. + if (background_work_queue_.empty()) { + background_work_cv_.Signal(); + } + + background_work_queue_.emplace(background_work_function, background_work_arg); + background_work_mutex_.Unlock(); +} + +void WindowsEnv::BackgroundThreadMain() { + while (true) { + background_work_mutex_.Lock(); + + // Wait until there is work to be done. + while (background_work_queue_.empty()) { + background_work_cv_.Wait(); + } + + assert(!background_work_queue_.empty()); + auto background_work_function = background_work_queue_.front().function; + void* background_work_arg = background_work_queue_.front().arg; + background_work_queue_.pop(); + + background_work_mutex_.Unlock(); + background_work_function(background_work_arg); + } +} + +// Wraps an Env instance whose destructor is never created. +// +// Intended usage: +// using PlatformSingletonEnv = SingletonEnv; +// void ConfigurePosixEnv(int param) { +// PlatformSingletonEnv::AssertEnvNotInitialized(); +// // set global configuration flags. +// } +// Env* Env::Default() { +// static PlatformSingletonEnv default_env; +// return default_env.env(); +// } +template +class SingletonEnv { + public: + SingletonEnv() { +#if !defined(NDEBUG) + env_initialized_.store(true, std::memory_order_relaxed); +#endif // !defined(NDEBUG) + static_assert(sizeof(env_storage_) >= sizeof(EnvType), + "env_storage_ will not fit the Env"); + static_assert(alignof(decltype(env_storage_)) >= alignof(EnvType), + "env_storage_ does not meet the Env's alignment needs"); + new (&env_storage_) EnvType(); + } + ~SingletonEnv() = default; + + SingletonEnv(const SingletonEnv&) = delete; + SingletonEnv& operator=(const SingletonEnv&) = delete; + + Env* env() { return reinterpret_cast(&env_storage_); } + + static void AssertEnvNotInitialized() { +#if !defined(NDEBUG) + assert(!env_initialized_.load(std::memory_order_relaxed)); +#endif // !defined(NDEBUG) + } + + private: + typename std::aligned_storage::type + env_storage_; +#if !defined(NDEBUG) + static std::atomic env_initialized_; +#endif // !defined(NDEBUG) +}; + +#if !defined(NDEBUG) +template +std::atomic SingletonEnv::env_initialized_; +#endif // !defined(NDEBUG) + +using WindowsDefaultEnv = SingletonEnv; + +} // namespace + +void EnvWindowsTestHelper::SetReadOnlyMMapLimit(int limit) { + WindowsDefaultEnv::AssertEnvNotInitialized(); + g_mmap_limit = limit; +} + +Env* Env::Default() { + static WindowsDefaultEnv env_container; + return env_container.env(); +} + +} // namespace leveldb diff --git a/leveldb/util/env_windows_test.cc b/leveldb/util/env_windows_test.cc new file mode 100644 index 000000000..d6822d26a --- /dev/null +++ b/leveldb/util/env_windows_test.cc @@ -0,0 +1,65 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "gtest/gtest.h" +#include "leveldb/env.h" +#include "port/port.h" +#include "util/env_windows_test_helper.h" +#include "util/testutil.h" + +namespace leveldb { + +static const int kMMapLimit = 4; + +class EnvWindowsTest : public testing::Test { + public: + static void SetFileLimits(int mmap_limit) { + EnvWindowsTestHelper::SetReadOnlyMMapLimit(mmap_limit); + } + + EnvWindowsTest() : env_(Env::Default()) {} + + Env* env_; +}; + +TEST_F(EnvWindowsTest, TestOpenOnRead) { + // Write some test data to a single file that will be opened |n| times. + std::string test_dir; + ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); + std::string test_file = test_dir + "/open_on_read.txt"; + + FILE* f = std::fopen(test_file.c_str(), "w"); + ASSERT_TRUE(f != nullptr); + const char kFileData[] = "abcdefghijklmnopqrstuvwxyz"; + fputs(kFileData, f); + std::fclose(f); + + // Open test file some number above the sum of the two limits to force + // leveldb::WindowsEnv to switch from mapping the file into memory + // to basic file reading. + const int kNumFiles = kMMapLimit + 5; + leveldb::RandomAccessFile* files[kNumFiles] = {0}; + for (int i = 0; i < kNumFiles; i++) { + ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(test_file, &files[i])); + } + char scratch; + Slice read_result; + for (int i = 0; i < kNumFiles; i++) { + ASSERT_LEVELDB_OK(files[i]->Read(i, 1, &read_result, &scratch)); + ASSERT_EQ(kFileData[i], read_result[0]); + } + for (int i = 0; i < kNumFiles; i++) { + delete files[i]; + } + ASSERT_LEVELDB_OK(env_->RemoveFile(test_file)); +} + +} // namespace leveldb + +int main(int argc, char** argv) { + // All tests currently run with the same read-only file limits. + leveldb::EnvWindowsTest::SetFileLimits(leveldb::kMMapLimit); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/leveldb/util/env_windows_test_helper.h b/leveldb/util/env_windows_test_helper.h new file mode 100644 index 000000000..e6f602056 --- /dev/null +++ b/leveldb/util/env_windows_test_helper.h @@ -0,0 +1,25 @@ +// Copyright 2018 (c) The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ +#define STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ + +namespace leveldb { + +class EnvWindowsTest; + +// A helper for the Windows Env to facilitate testing. +class EnvWindowsTestHelper { + private: + friend class CorruptionTest; + friend class EnvWindowsTest; + + // Set the maximum number of read-only files that will be mapped via mmap. + // Must be called before creating an Env. + static void SetReadOnlyMMapLimit(int limit); +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ diff --git a/leveldb/util/filter_policy.cc b/leveldb/util/filter_policy.cc new file mode 100644 index 000000000..90fd754d6 --- /dev/null +++ b/leveldb/util/filter_policy.cc @@ -0,0 +1,11 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/filter_policy.h" + +namespace leveldb { + +FilterPolicy::~FilterPolicy() {} + +} // namespace leveldb diff --git a/leveldb/util/hash.cc b/leveldb/util/hash.cc new file mode 100644 index 000000000..8122fa834 --- /dev/null +++ b/leveldb/util/hash.cc @@ -0,0 +1,55 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/hash.h" + +#include + +#include "util/coding.h" + +// The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through +// between switch labels. The real definition should be provided externally. +// This one is a fallback version for unsupported compilers. +#ifndef FALLTHROUGH_INTENDED +#define FALLTHROUGH_INTENDED \ + do { \ + } while (0) +#endif + +namespace leveldb { + +uint32_t Hash(const char* data, size_t n, uint32_t seed) { + // Similar to murmur hash + const uint32_t m = 0xc6a4a793; + const uint32_t r = 24; + const char* limit = data + n; + uint32_t h = seed ^ (n * m); + + // Pick up four bytes at a time + while (data + 4 <= limit) { + uint32_t w = DecodeFixed32(data); + data += 4; + h += w; + h *= m; + h ^= (h >> 16); + } + + // Pick up remaining bytes + switch (limit - data) { + case 3: + h += static_cast(data[2]) << 16; + FALLTHROUGH_INTENDED; + case 2: + h += static_cast(data[1]) << 8; + FALLTHROUGH_INTENDED; + case 1: + h += static_cast(data[0]); + h *= m; + h ^= (h >> r); + break; + } + return h; +} + +} // namespace leveldb diff --git a/leveldb/util/hash.h b/leveldb/util/hash.h new file mode 100644 index 000000000..87ab27990 --- /dev/null +++ b/leveldb/util/hash.h @@ -0,0 +1,19 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Simple hash function used for internal data structures + +#ifndef STORAGE_LEVELDB_UTIL_HASH_H_ +#define STORAGE_LEVELDB_UTIL_HASH_H_ + +#include +#include + +namespace leveldb { + +uint32_t Hash(const char* data, size_t n, uint32_t seed); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_HASH_H_ diff --git a/leveldb/util/hash_test.cc b/leveldb/util/hash_test.cc new file mode 100644 index 000000000..0ea597702 --- /dev/null +++ b/leveldb/util/hash_test.cc @@ -0,0 +1,41 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/hash.h" + +#include "gtest/gtest.h" + +namespace leveldb { + +TEST(HASH, SignedUnsignedIssue) { + const uint8_t data1[1] = {0x62}; + const uint8_t data2[2] = {0xc3, 0x97}; + const uint8_t data3[3] = {0xe2, 0x99, 0xa5}; + const uint8_t data4[4] = {0xe1, 0x80, 0xb9, 0x32}; + const uint8_t data5[48] = { + 0x01, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x28, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + + ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34); + ASSERT_EQ( + Hash(reinterpret_cast(data1), sizeof(data1), 0xbc9f1d34), + 0xef1345c4); + ASSERT_EQ( + Hash(reinterpret_cast(data2), sizeof(data2), 0xbc9f1d34), + 0x5b663814); + ASSERT_EQ( + Hash(reinterpret_cast(data3), sizeof(data3), 0xbc9f1d34), + 0x323c078f); + ASSERT_EQ( + Hash(reinterpret_cast(data4), sizeof(data4), 0xbc9f1d34), + 0xed21633a); + ASSERT_EQ( + Hash(reinterpret_cast(data5), sizeof(data5), 0x12345678), + 0xf333dabb); +} + +} // namespace leveldb diff --git a/leveldb/util/histogram.cc b/leveldb/util/histogram.cc new file mode 100644 index 000000000..7af40309e --- /dev/null +++ b/leveldb/util/histogram.cc @@ -0,0 +1,272 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/histogram.h" + +#include +#include + +#include "port/port.h" + +namespace leveldb { + +const double Histogram::kBucketLimit[kNumBuckets] = { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 12, + 14, + 16, + 18, + 20, + 25, + 30, + 35, + 40, + 45, + 50, + 60, + 70, + 80, + 90, + 100, + 120, + 140, + 160, + 180, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 600, + 700, + 800, + 900, + 1000, + 1200, + 1400, + 1600, + 1800, + 2000, + 2500, + 3000, + 3500, + 4000, + 4500, + 5000, + 6000, + 7000, + 8000, + 9000, + 10000, + 12000, + 14000, + 16000, + 18000, + 20000, + 25000, + 30000, + 35000, + 40000, + 45000, + 50000, + 60000, + 70000, + 80000, + 90000, + 100000, + 120000, + 140000, + 160000, + 180000, + 200000, + 250000, + 300000, + 350000, + 400000, + 450000, + 500000, + 600000, + 700000, + 800000, + 900000, + 1000000, + 1200000, + 1400000, + 1600000, + 1800000, + 2000000, + 2500000, + 3000000, + 3500000, + 4000000, + 4500000, + 5000000, + 6000000, + 7000000, + 8000000, + 9000000, + 10000000, + 12000000, + 14000000, + 16000000, + 18000000, + 20000000, + 25000000, + 30000000, + 35000000, + 40000000, + 45000000, + 50000000, + 60000000, + 70000000, + 80000000, + 90000000, + 100000000, + 120000000, + 140000000, + 160000000, + 180000000, + 200000000, + 250000000, + 300000000, + 350000000, + 400000000, + 450000000, + 500000000, + 600000000, + 700000000, + 800000000, + 900000000, + 1000000000, + 1200000000, + 1400000000, + 1600000000, + 1800000000, + 2000000000, + 2500000000.0, + 3000000000.0, + 3500000000.0, + 4000000000.0, + 4500000000.0, + 5000000000.0, + 6000000000.0, + 7000000000.0, + 8000000000.0, + 9000000000.0, + 1e200, +}; + +void Histogram::Clear() { + min_ = kBucketLimit[kNumBuckets - 1]; + max_ = 0; + num_ = 0; + sum_ = 0; + sum_squares_ = 0; + for (int i = 0; i < kNumBuckets; i++) { + buckets_[i] = 0; + } +} + +void Histogram::Add(double value) { + // Linear search is fast enough for our usage in db_bench + int b = 0; + while (b < kNumBuckets - 1 && kBucketLimit[b] <= value) { + b++; + } + buckets_[b] += 1.0; + if (min_ > value) min_ = value; + if (max_ < value) max_ = value; + num_++; + sum_ += value; + sum_squares_ += (value * value); +} + +void Histogram::Merge(const Histogram& other) { + if (other.min_ < min_) min_ = other.min_; + if (other.max_ > max_) max_ = other.max_; + num_ += other.num_; + sum_ += other.sum_; + sum_squares_ += other.sum_squares_; + for (int b = 0; b < kNumBuckets; b++) { + buckets_[b] += other.buckets_[b]; + } +} + +double Histogram::Median() const { return Percentile(50.0); } + +double Histogram::Percentile(double p) const { + double threshold = num_ * (p / 100.0); + double sum = 0; + for (int b = 0; b < kNumBuckets; b++) { + sum += buckets_[b]; + if (sum >= threshold) { + // Scale linearly within this bucket + double left_point = (b == 0) ? 0 : kBucketLimit[b - 1]; + double right_point = kBucketLimit[b]; + double left_sum = sum - buckets_[b]; + double right_sum = sum; + double pos = (threshold - left_sum) / (right_sum - left_sum); + double r = left_point + (right_point - left_point) * pos; + if (r < min_) r = min_; + if (r > max_) r = max_; + return r; + } + } + return max_; +} + +double Histogram::Average() const { + if (num_ == 0.0) return 0; + return sum_ / num_; +} + +double Histogram::StandardDeviation() const { + if (num_ == 0.0) return 0; + double variance = (sum_squares_ * num_ - sum_ * sum_) / (num_ * num_); + return sqrt(variance); +} + +std::string Histogram::ToString() const { + std::string r; + char buf[200]; + std::snprintf(buf, sizeof(buf), "Count: %.0f Average: %.4f StdDev: %.2f\n", + num_, Average(), StandardDeviation()); + r.append(buf); + std::snprintf(buf, sizeof(buf), "Min: %.4f Median: %.4f Max: %.4f\n", + (num_ == 0.0 ? 0.0 : min_), Median(), max_); + r.append(buf); + r.append("------------------------------------------------------\n"); + const double mult = 100.0 / num_; + double sum = 0; + for (int b = 0; b < kNumBuckets; b++) { + if (buckets_[b] <= 0.0) continue; + sum += buckets_[b]; + std::snprintf(buf, sizeof(buf), "[ %7.0f, %7.0f ) %7.0f %7.3f%% %7.3f%% ", + ((b == 0) ? 0.0 : kBucketLimit[b - 1]), // left + kBucketLimit[b], // right + buckets_[b], // count + mult * buckets_[b], // percentage + mult * sum); // cumulative percentage + r.append(buf); + + // Add hash marks based on percentage; 20 marks for 100%. + int marks = static_cast(20 * (buckets_[b] / num_) + 0.5); + r.append(marks, '#'); + r.push_back('\n'); + } + return r; +} + +} // namespace leveldb diff --git a/leveldb/util/histogram.h b/leveldb/util/histogram.h new file mode 100644 index 000000000..4da60fba4 --- /dev/null +++ b/leveldb/util/histogram.h @@ -0,0 +1,44 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ +#define STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ + +#include + +namespace leveldb { + +class Histogram { + public: + Histogram() {} + ~Histogram() {} + + void Clear(); + void Add(double value); + void Merge(const Histogram& other); + + std::string ToString() const; + + private: + enum { kNumBuckets = 154 }; + + double Median() const; + double Percentile(double p) const; + double Average() const; + double StandardDeviation() const; + + static const double kBucketLimit[kNumBuckets]; + + double min_; + double max_; + double num_; + double sum_; + double sum_squares_; + + double buckets_[kNumBuckets]; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ diff --git a/leveldb/util/logging.cc b/leveldb/util/logging.cc new file mode 100644 index 000000000..8d6fb5b0e --- /dev/null +++ b/leveldb/util/logging.cc @@ -0,0 +1,82 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/logging.h" + +#include +#include +#include +#include + +#include "leveldb/env.h" +#include "leveldb/slice.h" + +namespace leveldb { + +void AppendNumberTo(std::string* str, uint64_t num) { + char buf[30]; + std::snprintf(buf, sizeof(buf), "%llu", static_cast(num)); + str->append(buf); +} + +void AppendEscapedStringTo(std::string* str, const Slice& value) { + for (size_t i = 0; i < value.size(); i++) { + char c = value[i]; + if (c >= ' ' && c <= '~') { + str->push_back(c); + } else { + char buf[10]; + std::snprintf(buf, sizeof(buf), "\\x%02x", + static_cast(c) & 0xff); + str->append(buf); + } + } +} + +std::string NumberToString(uint64_t num) { + std::string r; + AppendNumberTo(&r, num); + return r; +} + +std::string EscapeString(const Slice& value) { + std::string r; + AppendEscapedStringTo(&r, value); + return r; +} + +bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { + // Constants that will be optimized away. + constexpr const uint64_t kMaxUint64 = std::numeric_limits::max(); + constexpr const char kLastDigitOfMaxUint64 = + '0' + static_cast(kMaxUint64 % 10); + + uint64_t value = 0; + + // reinterpret_cast-ing from char* to uint8_t* to avoid signedness. + const uint8_t* start = reinterpret_cast(in->data()); + + const uint8_t* end = start + in->size(); + const uint8_t* current = start; + for (; current != end; ++current) { + const uint8_t ch = *current; + if (ch < '0' || ch > '9') break; + + // Overflow check. + // kMaxUint64 / 10 is also constant and will be optimized away. + if (value > kMaxUint64 / 10 || + (value == kMaxUint64 / 10 && ch > kLastDigitOfMaxUint64)) { + return false; + } + + value = (value * 10) + (ch - '0'); + } + + *val = value; + const size_t digits_consumed = current - start; + in->remove_prefix(digits_consumed); + return digits_consumed != 0; +} + +} // namespace leveldb diff --git a/leveldb/util/logging.h b/leveldb/util/logging.h new file mode 100644 index 000000000..a0394b2c8 --- /dev/null +++ b/leveldb/util/logging.h @@ -0,0 +1,44 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Must not be included from any .h files to avoid polluting the namespace +// with macros. + +#ifndef STORAGE_LEVELDB_UTIL_LOGGING_H_ +#define STORAGE_LEVELDB_UTIL_LOGGING_H_ + +#include +#include +#include + +#include "port/port.h" + +namespace leveldb { + +class Slice; +class WritableFile; + +// Append a human-readable printout of "num" to *str +void AppendNumberTo(std::string* str, uint64_t num); + +// Append a human-readable printout of "value" to *str. +// Escapes any non-printable characters found in "value". +void AppendEscapedStringTo(std::string* str, const Slice& value); + +// Return a human-readable printout of "num" +std::string NumberToString(uint64_t num); + +// Return a human-readable version of "value". +// Escapes any non-printable characters found in "value". +std::string EscapeString(const Slice& value); + +// Parse a human-readable number from "*in" into *value. On success, +// advances "*in" past the consumed number and sets "*val" to the +// numeric value. Otherwise, returns false and leaves *in in an +// unspecified state. +bool ConsumeDecimalNumber(Slice* in, uint64_t* val); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_LOGGING_H_ diff --git a/leveldb/util/logging_test.cc b/leveldb/util/logging_test.cc new file mode 100644 index 000000000..1746c57d4 --- /dev/null +++ b/leveldb/util/logging_test.cc @@ -0,0 +1,140 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/logging.h" + +#include +#include + +#include "gtest/gtest.h" +#include "leveldb/slice.h" + +namespace leveldb { + +TEST(Logging, NumberToString) { + ASSERT_EQ("0", NumberToString(0)); + ASSERT_EQ("1", NumberToString(1)); + ASSERT_EQ("9", NumberToString(9)); + + ASSERT_EQ("10", NumberToString(10)); + ASSERT_EQ("11", NumberToString(11)); + ASSERT_EQ("19", NumberToString(19)); + ASSERT_EQ("99", NumberToString(99)); + + ASSERT_EQ("100", NumberToString(100)); + ASSERT_EQ("109", NumberToString(109)); + ASSERT_EQ("190", NumberToString(190)); + ASSERT_EQ("123", NumberToString(123)); + ASSERT_EQ("12345678", NumberToString(12345678)); + + static_assert(std::numeric_limits::max() == 18446744073709551615U, + "Test consistency check"); + ASSERT_EQ("18446744073709551000", NumberToString(18446744073709551000U)); + ASSERT_EQ("18446744073709551600", NumberToString(18446744073709551600U)); + ASSERT_EQ("18446744073709551610", NumberToString(18446744073709551610U)); + ASSERT_EQ("18446744073709551614", NumberToString(18446744073709551614U)); + ASSERT_EQ("18446744073709551615", NumberToString(18446744073709551615U)); +} + +void ConsumeDecimalNumberRoundtripTest(uint64_t number, + const std::string& padding = "") { + std::string decimal_number = NumberToString(number); + std::string input_string = decimal_number + padding; + Slice input(input_string); + Slice output = input; + uint64_t result; + ASSERT_TRUE(ConsumeDecimalNumber(&output, &result)); + ASSERT_EQ(number, result); + ASSERT_EQ(decimal_number.size(), output.data() - input.data()); + ASSERT_EQ(padding.size(), output.size()); +} + +TEST(Logging, ConsumeDecimalNumberRoundtrip) { + ConsumeDecimalNumberRoundtripTest(0); + ConsumeDecimalNumberRoundtripTest(1); + ConsumeDecimalNumberRoundtripTest(9); + + ConsumeDecimalNumberRoundtripTest(10); + ConsumeDecimalNumberRoundtripTest(11); + ConsumeDecimalNumberRoundtripTest(19); + ConsumeDecimalNumberRoundtripTest(99); + + ConsumeDecimalNumberRoundtripTest(100); + ConsumeDecimalNumberRoundtripTest(109); + ConsumeDecimalNumberRoundtripTest(190); + ConsumeDecimalNumberRoundtripTest(123); + ASSERT_EQ("12345678", NumberToString(12345678)); + + for (uint64_t i = 0; i < 100; ++i) { + uint64_t large_number = std::numeric_limits::max() - i; + ConsumeDecimalNumberRoundtripTest(large_number); + } +} + +TEST(Logging, ConsumeDecimalNumberRoundtripWithPadding) { + ConsumeDecimalNumberRoundtripTest(0, " "); + ConsumeDecimalNumberRoundtripTest(1, "abc"); + ConsumeDecimalNumberRoundtripTest(9, "x"); + + ConsumeDecimalNumberRoundtripTest(10, "_"); + ConsumeDecimalNumberRoundtripTest(11, std::string("\0\0\0", 3)); + ConsumeDecimalNumberRoundtripTest(19, "abc"); + ConsumeDecimalNumberRoundtripTest(99, "padding"); + + ConsumeDecimalNumberRoundtripTest(100, " "); + + for (uint64_t i = 0; i < 100; ++i) { + uint64_t large_number = std::numeric_limits::max() - i; + ConsumeDecimalNumberRoundtripTest(large_number, "pad"); + } +} + +void ConsumeDecimalNumberOverflowTest(const std::string& input_string) { + Slice input(input_string); + Slice output = input; + uint64_t result; + ASSERT_EQ(false, ConsumeDecimalNumber(&output, &result)); +} + +TEST(Logging, ConsumeDecimalNumberOverflow) { + static_assert(std::numeric_limits::max() == 18446744073709551615U, + "Test consistency check"); + ConsumeDecimalNumberOverflowTest("18446744073709551616"); + ConsumeDecimalNumberOverflowTest("18446744073709551617"); + ConsumeDecimalNumberOverflowTest("18446744073709551618"); + ConsumeDecimalNumberOverflowTest("18446744073709551619"); + ConsumeDecimalNumberOverflowTest("18446744073709551620"); + ConsumeDecimalNumberOverflowTest("18446744073709551621"); + ConsumeDecimalNumberOverflowTest("18446744073709551622"); + ConsumeDecimalNumberOverflowTest("18446744073709551623"); + ConsumeDecimalNumberOverflowTest("18446744073709551624"); + ConsumeDecimalNumberOverflowTest("18446744073709551625"); + ConsumeDecimalNumberOverflowTest("18446744073709551626"); + + ConsumeDecimalNumberOverflowTest("18446744073709551700"); + + ConsumeDecimalNumberOverflowTest("99999999999999999999"); +} + +void ConsumeDecimalNumberNoDigitsTest(const std::string& input_string) { + Slice input(input_string); + Slice output = input; + uint64_t result; + ASSERT_EQ(false, ConsumeDecimalNumber(&output, &result)); + ASSERT_EQ(input.data(), output.data()); + ASSERT_EQ(input.size(), output.size()); +} + +TEST(Logging, ConsumeDecimalNumberNoDigits) { + ConsumeDecimalNumberNoDigitsTest(""); + ConsumeDecimalNumberNoDigitsTest(" "); + ConsumeDecimalNumberNoDigitsTest("a"); + ConsumeDecimalNumberNoDigitsTest(" 123"); + ConsumeDecimalNumberNoDigitsTest("a123"); + ConsumeDecimalNumberNoDigitsTest(std::string("\000123", 4)); + ConsumeDecimalNumberNoDigitsTest(std::string("\177123", 4)); + ConsumeDecimalNumberNoDigitsTest(std::string("\377123", 4)); +} + +} // namespace leveldb diff --git a/leveldb/util/mutexlock.h b/leveldb/util/mutexlock.h new file mode 100644 index 000000000..0cb2e250f --- /dev/null +++ b/leveldb/util/mutexlock.h @@ -0,0 +1,39 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ +#define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ + +#include "port/port.h" +#include "port/thread_annotations.h" + +namespace leveldb { + +// Helper class that locks a mutex on construction and unlocks the mutex when +// the destructor of the MutexLock object is invoked. +// +// Typical usage: +// +// void MyClass::MyMethod() { +// MutexLock l(&mu_); // mu_ is an instance variable +// ... some complex code, possibly with multiple return paths ... +// } + +class SCOPED_LOCKABLE MutexLock { + public: + explicit MutexLock(port::Mutex* mu) EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { + this->mu_->Lock(); + } + ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } + + MutexLock(const MutexLock&) = delete; + MutexLock& operator=(const MutexLock&) = delete; + + private: + port::Mutex* const mu_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ diff --git a/leveldb/util/no_destructor.h b/leveldb/util/no_destructor.h new file mode 100644 index 000000000..a0d3b8703 --- /dev/null +++ b/leveldb/util/no_destructor.h @@ -0,0 +1,46 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_NO_DESTRUCTOR_H_ +#define STORAGE_LEVELDB_UTIL_NO_DESTRUCTOR_H_ + +#include +#include + +namespace leveldb { + +// Wraps an instance whose destructor is never called. +// +// This is intended for use with function-level static variables. +template +class NoDestructor { + public: + template + explicit NoDestructor(ConstructorArgTypes&&... constructor_args) { + static_assert(sizeof(instance_storage_) >= sizeof(InstanceType), + "instance_storage_ is not large enough to hold the instance"); + static_assert( + alignof(decltype(instance_storage_)) >= alignof(InstanceType), + "instance_storage_ does not meet the instance's alignment requirement"); + new (&instance_storage_) + InstanceType(std::forward(constructor_args)...); + } + + ~NoDestructor() = default; + + NoDestructor(const NoDestructor&) = delete; + NoDestructor& operator=(const NoDestructor&) = delete; + + InstanceType* get() { + return reinterpret_cast(&instance_storage_); + } + + private: + typename std::aligned_storage::type instance_storage_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_NO_DESTRUCTOR_H_ diff --git a/leveldb/util/no_destructor_test.cc b/leveldb/util/no_destructor_test.cc new file mode 100644 index 000000000..e3602ccc5 --- /dev/null +++ b/leveldb/util/no_destructor_test.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/no_destructor.h" + +#include +#include +#include + +#include "gtest/gtest.h" + +namespace leveldb { + +namespace { + +struct DoNotDestruct { + public: + DoNotDestruct(uint32_t a, uint64_t b) : a(a), b(b) {} + ~DoNotDestruct() { std::abort(); } + + // Used to check constructor argument forwarding. + uint32_t a; + uint64_t b; +}; + +constexpr const uint32_t kGoldenA = 0xdeadbeef; +constexpr const uint64_t kGoldenB = 0xaabbccddeeffaabb; + +} // namespace + +TEST(NoDestructorTest, StackInstance) { + NoDestructor instance(kGoldenA, kGoldenB); + ASSERT_EQ(kGoldenA, instance.get()->a); + ASSERT_EQ(kGoldenB, instance.get()->b); +} + +TEST(NoDestructorTest, StaticInstance) { + static NoDestructor instance(kGoldenA, kGoldenB); + ASSERT_EQ(kGoldenA, instance.get()->a); + ASSERT_EQ(kGoldenB, instance.get()->b); +} + +} // namespace leveldb diff --git a/leveldb/util/options.cc b/leveldb/util/options.cc new file mode 100644 index 000000000..62de5bf0d --- /dev/null +++ b/leveldb/util/options.cc @@ -0,0 +1,14 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/options.h" + +#include "leveldb/comparator.h" +#include "leveldb/env.h" + +namespace leveldb { + +Options::Options() : comparator(BytewiseComparator()), env(Env::Default()) {} + +} // namespace leveldb diff --git a/leveldb/util/posix_logger.h b/leveldb/util/posix_logger.h new file mode 100644 index 000000000..6bbc1a085 --- /dev/null +++ b/leveldb/util/posix_logger.h @@ -0,0 +1,130 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Logger implementation that can be shared by all environments +// where enough posix functionality is available. + +#ifndef STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ +#define STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "leveldb/env.h" + +namespace leveldb { + +class PosixLogger final : public Logger { + public: + // Creates a logger that writes to the given file. + // + // The PosixLogger instance takes ownership of the file handle. + explicit PosixLogger(std::FILE* fp) : fp_(fp) { assert(fp != nullptr); } + + ~PosixLogger() override { std::fclose(fp_); } + + void Logv(const char* format, std::va_list arguments) override { + // Record the time as close to the Logv() call as possible. + struct ::timeval now_timeval; + ::gettimeofday(&now_timeval, nullptr); + const std::time_t now_seconds = now_timeval.tv_sec; + struct std::tm now_components; + ::localtime_r(&now_seconds, &now_components); + + // Record the thread ID. + constexpr const int kMaxThreadIdSize = 32; + std::ostringstream thread_stream; + thread_stream << std::this_thread::get_id(); + std::string thread_id = thread_stream.str(); + if (thread_id.size() > kMaxThreadIdSize) { + thread_id.resize(kMaxThreadIdSize); + } + + // We first attempt to print into a stack-allocated buffer. If this attempt + // fails, we make a second attempt with a dynamically allocated buffer. + constexpr const int kStackBufferSize = 512; + char stack_buffer[kStackBufferSize]; + static_assert(sizeof(stack_buffer) == static_cast(kStackBufferSize), + "sizeof(char) is expected to be 1 in C++"); + + int dynamic_buffer_size = 0; // Computed in the first iteration. + for (int iteration = 0; iteration < 2; ++iteration) { + const int buffer_size = + (iteration == 0) ? kStackBufferSize : dynamic_buffer_size; + char* const buffer = + (iteration == 0) ? stack_buffer : new char[dynamic_buffer_size]; + + // Print the header into the buffer. + int buffer_offset = std::snprintf( + buffer, buffer_size, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %s ", + now_components.tm_year + 1900, now_components.tm_mon + 1, + now_components.tm_mday, now_components.tm_hour, now_components.tm_min, + now_components.tm_sec, static_cast(now_timeval.tv_usec), + thread_id.c_str()); + + // The header can be at most 28 characters (10 date + 15 time + + // 3 delimiters) plus the thread ID, which should fit comfortably into the + // static buffer. + assert(buffer_offset <= 28 + kMaxThreadIdSize); + static_assert(28 + kMaxThreadIdSize < kStackBufferSize, + "stack-allocated buffer may not fit the message header"); + assert(buffer_offset < buffer_size); + + // Print the message into the buffer. + std::va_list arguments_copy; + va_copy(arguments_copy, arguments); + buffer_offset += + std::vsnprintf(buffer + buffer_offset, buffer_size - buffer_offset, + format, arguments_copy); + va_end(arguments_copy); + + // The code below may append a newline at the end of the buffer, which + // requires an extra character. + if (buffer_offset >= buffer_size - 1) { + // The message did not fit into the buffer. + if (iteration == 0) { + // Re-run the loop and use a dynamically-allocated buffer. The buffer + // will be large enough for the log message, an extra newline and a + // null terminator. + dynamic_buffer_size = buffer_offset + 2; + continue; + } + + // The dynamically-allocated buffer was incorrectly sized. This should + // not happen, assuming a correct implementation of std::(v)snprintf. + // Fail in tests, recover by truncating the log message in production. + assert(false); + buffer_offset = buffer_size - 1; + } + + // Add a newline if necessary. + if (buffer[buffer_offset - 1] != '\n') { + buffer[buffer_offset] = '\n'; + ++buffer_offset; + } + + assert(buffer_offset <= buffer_size); + std::fwrite(buffer, 1, buffer_offset, fp_); + std::fflush(fp_); + + if (iteration != 0) { + delete[] buffer; + } + break; + } + } + + private: + std::FILE* const fp_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ diff --git a/leveldb/util/random.h b/leveldb/util/random.h new file mode 100644 index 000000000..fe76ab443 --- /dev/null +++ b/leveldb/util/random.h @@ -0,0 +1,63 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_RANDOM_H_ +#define STORAGE_LEVELDB_UTIL_RANDOM_H_ + +#include + +namespace leveldb { + +// A very simple random number generator. Not especially good at +// generating truly random bits, but good enough for our needs in this +// package. +class Random { + private: + uint32_t seed_; + + public: + explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { + // Avoid bad seeds. + if (seed_ == 0 || seed_ == 2147483647L) { + seed_ = 1; + } + } + uint32_t Next() { + static const uint32_t M = 2147483647L; // 2^31-1 + static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 + // We are computing + // seed_ = (seed_ * A) % M, where M = 2^31-1 + // + // seed_ must not be zero or M, or else all subsequent computed values + // will be zero or M respectively. For all other values, seed_ will end + // up cycling through every number in [1,M-1] + uint64_t product = seed_ * A; + + // Compute (product % M) using the fact that ((x << 31) % M) == x. + seed_ = static_cast((product >> 31) + (product & M)); + // The first reduction may overflow by 1 bit, so we may need to + // repeat. mod == M is not possible; using > allows the faster + // sign-bit-based test. + if (seed_ > M) { + seed_ -= M; + } + return seed_; + } + // Returns a uniformly distributed value in the range [0..n-1] + // REQUIRES: n > 0 + uint32_t Uniform(int n) { return Next() % n; } + + // Randomly returns true ~"1/n" of the time, and false otherwise. + // REQUIRES: n > 0 + bool OneIn(int n) { return (Next() % n) == 0; } + + // Skewed: pick "base" uniformly from range [0,max_log] and then + // return "base" random bits. The effect is to pick a number in the + // range [0,2^max_log-1] with exponential bias towards smaller numbers. + uint32_t Skewed(int max_log) { return Uniform(1 << Uniform(max_log + 1)); } +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_RANDOM_H_ diff --git a/leveldb/util/status.cc b/leveldb/util/status.cc new file mode 100644 index 000000000..0559f5b17 --- /dev/null +++ b/leveldb/util/status.cc @@ -0,0 +1,77 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/status.h" + +#include + +#include "port/port.h" + +namespace leveldb { + +const char* Status::CopyState(const char* state) { + uint32_t size; + std::memcpy(&size, state, sizeof(size)); + char* result = new char[size + 5]; + std::memcpy(result, state, size + 5); + return result; +} + +Status::Status(Code code, const Slice& msg, const Slice& msg2) { + assert(code != kOk); + const uint32_t len1 = static_cast(msg.size()); + const uint32_t len2 = static_cast(msg2.size()); + const uint32_t size = len1 + (len2 ? (2 + len2) : 0); + char* result = new char[size + 5]; + std::memcpy(result, &size, sizeof(size)); + result[4] = static_cast(code); + std::memcpy(result + 5, msg.data(), len1); + if (len2) { + result[5 + len1] = ':'; + result[6 + len1] = ' '; + std::memcpy(result + 7 + len1, msg2.data(), len2); + } + state_ = result; +} + +std::string Status::ToString() const { + if (state_ == nullptr) { + return "OK"; + } else { + char tmp[30]; + const char* type; + switch (code()) { + case kOk: + type = "OK"; + break; + case kNotFound: + type = "NotFound: "; + break; + case kCorruption: + type = "Corruption: "; + break; + case kNotSupported: + type = "Not implemented: "; + break; + case kInvalidArgument: + type = "Invalid argument: "; + break; + case kIOError: + type = "IO error: "; + break; + default: + std::snprintf(tmp, sizeof(tmp), + "Unknown code(%d): ", static_cast(code())); + type = tmp; + break; + } + std::string result(type); + uint32_t length; + std::memcpy(&length, state_, sizeof(length)); + result.append(state_ + 5, length); + return result; + } +} + +} // namespace leveldb diff --git a/leveldb/util/status_test.cc b/leveldb/util/status_test.cc new file mode 100644 index 000000000..dbf5faaa5 --- /dev/null +++ b/leveldb/util/status_test.cc @@ -0,0 +1,39 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/status.h" + +#include + +#include "gtest/gtest.h" +#include "leveldb/slice.h" + +namespace leveldb { + +TEST(Status, MoveConstructor) { + { + Status ok = Status::OK(); + Status ok2 = std::move(ok); + + ASSERT_TRUE(ok2.ok()); + } + + { + Status status = Status::NotFound("custom NotFound status message"); + Status status2 = std::move(status); + + ASSERT_TRUE(status2.IsNotFound()); + ASSERT_EQ("NotFound: custom NotFound status message", status2.ToString()); + } + + { + Status self_moved = Status::IOError("custom IOError status message"); + + // Needed to bypass compiler warning about explicit move-assignment. + Status& self_moved_reference = self_moved; + self_moved_reference = std::move(self_moved); + } +} + +} // namespace leveldb diff --git a/leveldb/util/testutil.cc b/leveldb/util/testutil.cc new file mode 100644 index 000000000..5f77b0864 --- /dev/null +++ b/leveldb/util/testutil.cc @@ -0,0 +1,51 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/testutil.h" + +#include + +#include "util/random.h" + +namespace leveldb { +namespace test { + +Slice RandomString(Random* rnd, int len, std::string* dst) { + dst->resize(len); + for (int i = 0; i < len; i++) { + (*dst)[i] = static_cast(' ' + rnd->Uniform(95)); // ' ' .. '~' + } + return Slice(*dst); +} + +std::string RandomKey(Random* rnd, int len) { + // Make sure to generate a wide variety of characters so we + // test the boundary conditions for short-key optimizations. + static const char kTestChars[] = {'\0', '\1', 'a', 'b', 'c', + 'd', 'e', '\xfd', '\xfe', '\xff'}; + std::string result; + for (int i = 0; i < len; i++) { + result += kTestChars[rnd->Uniform(sizeof(kTestChars))]; + } + return result; +} + +Slice CompressibleString(Random* rnd, double compressed_fraction, size_t len, + std::string* dst) { + int raw = static_cast(len * compressed_fraction); + if (raw < 1) raw = 1; + std::string raw_data; + RandomString(rnd, raw, &raw_data); + + // Duplicate the random data until we have filled "len" bytes + dst->clear(); + while (dst->size() < len) { + dst->append(raw_data); + } + dst->resize(len); + return Slice(*dst); +} + +} // namespace test +} // namespace leveldb diff --git a/leveldb/util/testutil.h b/leveldb/util/testutil.h new file mode 100644 index 000000000..e0e2d64d5 --- /dev/null +++ b/leveldb/util/testutil.h @@ -0,0 +1,82 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_TESTUTIL_H_ +#define STORAGE_LEVELDB_UTIL_TESTUTIL_H_ + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "helpers/memenv/memenv.h" +#include "leveldb/env.h" +#include "leveldb/slice.h" +#include "util/random.h" + +namespace leveldb { +namespace test { + +MATCHER(IsOK, "") { return arg.ok(); } + +// Macros for testing the results of functions that return leveldb::Status or +// absl::StatusOr (for any type T). +#define EXPECT_LEVELDB_OK(expression) \ + EXPECT_THAT(expression, leveldb::test::IsOK()) +#define ASSERT_LEVELDB_OK(expression) \ + ASSERT_THAT(expression, leveldb::test::IsOK()) + +// Returns the random seed used at the start of the current test run. +inline int RandomSeed() { + return testing::UnitTest::GetInstance()->random_seed(); +} + +// Store in *dst a random string of length "len" and return a Slice that +// references the generated data. +Slice RandomString(Random* rnd, int len, std::string* dst); + +// Return a random key with the specified length that may contain interesting +// characters (e.g. \x00, \xff, etc.). +std::string RandomKey(Random* rnd, int len); + +// Store in *dst a string of length "len" that will compress to +// "N*compressed_fraction" bytes and return a Slice that references +// the generated data. +Slice CompressibleString(Random* rnd, double compressed_fraction, size_t len, + std::string* dst); + +// A wrapper that allows injection of errors. +class ErrorEnv : public EnvWrapper { + public: + bool writable_file_error_; + int num_writable_file_errors_; + + ErrorEnv() + : EnvWrapper(NewMemEnv(Env::Default())), + writable_file_error_(false), + num_writable_file_errors_(0) {} + ~ErrorEnv() override { delete target(); } + + Status NewWritableFile(const std::string& fname, + WritableFile** result) override { + if (writable_file_error_) { + ++num_writable_file_errors_; + *result = nullptr; + return Status::IOError(fname, "fake error"); + } + return target()->NewWritableFile(fname, result); + } + + Status NewAppendableFile(const std::string& fname, + WritableFile** result) override { + if (writable_file_error_) { + ++num_writable_file_errors_; + *result = nullptr; + return Status::IOError(fname, "fake error"); + } + return target()->NewAppendableFile(fname, result); + } +}; + +} // namespace test +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_TESTUTIL_H_ diff --git a/leveldb/util/windows_logger.h b/leveldb/util/windows_logger.h new file mode 100644 index 000000000..26e6c7ba0 --- /dev/null +++ b/leveldb/util/windows_logger.h @@ -0,0 +1,124 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Logger implementation for the Windows platform. + +#ifndef STORAGE_LEVELDB_UTIL_WINDOWS_LOGGER_H_ +#define STORAGE_LEVELDB_UTIL_WINDOWS_LOGGER_H_ + +#include +#include +#include +#include +#include +#include + +#include "leveldb/env.h" + +namespace leveldb { + +class WindowsLogger final : public Logger { + public: + // Creates a logger that writes to the given file. + // + // The PosixLogger instance takes ownership of the file handle. + explicit WindowsLogger(std::FILE* fp) : fp_(fp) { assert(fp != nullptr); } + + ~WindowsLogger() override { std::fclose(fp_); } + + void Logv(const char* format, std::va_list arguments) override { + // Record the time as close to the Logv() call as possible. + SYSTEMTIME now_components; + ::GetLocalTime(&now_components); + + // Record the thread ID. + constexpr const int kMaxThreadIdSize = 32; + std::ostringstream thread_stream; + thread_stream << std::this_thread::get_id(); + std::string thread_id = thread_stream.str(); + if (thread_id.size() > kMaxThreadIdSize) { + thread_id.resize(kMaxThreadIdSize); + } + + // We first attempt to print into a stack-allocated buffer. If this attempt + // fails, we make a second attempt with a dynamically allocated buffer. + constexpr const int kStackBufferSize = 512; + char stack_buffer[kStackBufferSize]; + static_assert(sizeof(stack_buffer) == static_cast(kStackBufferSize), + "sizeof(char) is expected to be 1 in C++"); + + int dynamic_buffer_size = 0; // Computed in the first iteration. + for (int iteration = 0; iteration < 2; ++iteration) { + const int buffer_size = + (iteration == 0) ? kStackBufferSize : dynamic_buffer_size; + char* const buffer = + (iteration == 0) ? stack_buffer : new char[dynamic_buffer_size]; + + // Print the header into the buffer. + int buffer_offset = std::snprintf( + buffer, buffer_size, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %s ", + now_components.wYear, now_components.wMonth, now_components.wDay, + now_components.wHour, now_components.wMinute, now_components.wSecond, + static_cast(now_components.wMilliseconds * 1000), + thread_id.c_str()); + + // The header can be at most 28 characters (10 date + 15 time + + // 3 delimiters) plus the thread ID, which should fit comfortably into the + // static buffer. + assert(buffer_offset <= 28 + kMaxThreadIdSize); + static_assert(28 + kMaxThreadIdSize < kStackBufferSize, + "stack-allocated buffer may not fit the message header"); + assert(buffer_offset < buffer_size); + + // Print the message into the buffer. + std::va_list arguments_copy; + va_copy(arguments_copy, arguments); + buffer_offset += + std::vsnprintf(buffer + buffer_offset, buffer_size - buffer_offset, + format, arguments_copy); + va_end(arguments_copy); + + // The code below may append a newline at the end of the buffer, which + // requires an extra character. + if (buffer_offset >= buffer_size - 1) { + // The message did not fit into the buffer. + if (iteration == 0) { + // Re-run the loop and use a dynamically-allocated buffer. The buffer + // will be large enough for the log message, an extra newline and a + // null terminator. + dynamic_buffer_size = buffer_offset + 2; + continue; + } + + // The dynamically-allocated buffer was incorrectly sized. This should + // not happen, assuming a correct implementation of std::(v)snprintf. + // Fail in tests, recover by truncating the log message in production. + assert(false); + buffer_offset = buffer_size - 1; + } + + // Add a newline if necessary. + if (buffer[buffer_offset - 1] != '\n') { + buffer[buffer_offset] = '\n'; + ++buffer_offset; + } + + assert(buffer_offset <= buffer_size); + std::fwrite(buffer, 1, buffer_offset, fp_); + std::fflush(fp_); + + if (iteration != 0) { + delete[] buffer; + } + break; + } + } + + private: + std::FILE* const fp_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_WINDOWS_LOGGER_H_ From edbcc560495e57b67f578183b3692aabe1dd2cec Mon Sep 17 00:00:00 2001 From: QingyangZ Date: Tue, 23 May 2023 21:00:45 +0800 Subject: [PATCH 22/22] rename tinyengine --- doc/{tinydb => tinyengine}/Home.md | 5 ++++- doc/{tinydb => tinyengine}/project1-LSMTree.md | 0 doc/{tinydb => tinyengine}/project2-KV Separation.md | 0 3 files changed, 4 insertions(+), 1 deletion(-) rename doc/{tinydb => tinyengine}/Home.md (90%) rename doc/{tinydb => tinyengine}/project1-LSMTree.md (100%) rename doc/{tinydb => tinyengine}/project2-KV Separation.md (100%) diff --git a/doc/tinydb/Home.md b/doc/tinyengine/Home.md similarity index 90% rename from doc/tinydb/Home.md rename to doc/tinyengine/Home.md index 1b0cd2ed9..559a46769 100644 --- a/doc/tinydb/Home.md +++ b/doc/tinyengine/Home.md @@ -6,10 +6,13 @@ After completing this course, you will have a better understanding of LSM-tree-b ## Course Introduction TinyEngine is forked from open source [TinyKV](https://github.com/talent-plan/tinykv), a key-value storage system with the Raft consensus algorithm. TinyKV focuses on the storage layer of a distributed database system, which uses [badger](https://github.com/dgraph-io/badger), a Go library to store keys and values, as its storage engine. In order to get closer to the actual implementation of TiKV, TinyEngine plans to replace the original storage engine badger with [LevelDB](https://github.com/google/leveldb)/[RocksDB](https://github.com/facebook/rocksdb) wrapped by Golang. Therefore, please modify your implementation of project1 to use the interface of levigo(a wrapper of LevelDB) or gorocksdb(a wrapper of RocksDB) rather than badger. -In this course, you need to finish the implementation of LevelDB and then implement an existing optimization method on LevelDB/RocksDB. We provide several projects in this folder, which introduce some classic and generally accepted optimization ideas presented in recent famous paper. Please choose one and implement it. +In this course, you need to finish the implementation of LevelDB and then implement an existing optimization method on LevelDB/RocksDB. We provide a KV +Separation project [WiscKey](https://dl.acm.org/doi/abs/10.1145/3033273) in this folder, which introduce a classic and generally accepted optimization idea presented in recent famous paper. Please implement it on LevelDB. After completing the implementation, you need to test and evaluate your optimization. We provide go-ycsb, which can be used to evaluate database performance. If you successfully implement a project, you will get better performance in reading or writing or some other dimension. Finally, you need to chart your evaluation results and submit a report and source code. The experts will give you an appropriate score depending on your optimization results and report. +After finish this course, you can try to implement other optimizations like [DiffKV](chrome-extension://efaidnbmnnnibpcajpcglclefindmkaj/https://www.usenix.org/system/files/atc21-li-yongkun.pdf). + ### LevelDB/RocksDB [LevelDB](https://github.com/google/leveldb)/[RocksDB](https://github.com/facebook/rocksdb) is a storage engine for server workloads on various storage media, with the initial focus on fast storage (especially Flash storage). It is a C++ library to store key-value pairs. It supports both point lookups and range scans, and provides different types of ACID guarantees. diff --git a/doc/tinydb/project1-LSMTree.md b/doc/tinyengine/project1-LSMTree.md similarity index 100% rename from doc/tinydb/project1-LSMTree.md rename to doc/tinyengine/project1-LSMTree.md diff --git a/doc/tinydb/project2-KV Separation.md b/doc/tinyengine/project2-KV Separation.md similarity index 100% rename from doc/tinydb/project2-KV Separation.md rename to doc/tinyengine/project2-KV Separation.md