From 418a2d8a801f9d97223909a69b1c834e206973dd Mon Sep 17 00:00:00 2001 From: jay tseng Date: Tue, 20 Sep 2022 14:26:33 -0400 Subject: [PATCH] implement DB options for customizing maxopenfiles backport from cosmos-db(#51) --- backend_test.go | 19 +++++++++---------- badger_db.go | 2 +- boltdb.go | 2 +- boltdb_test.go | 6 +++--- cleveldb.go | 27 ++++++++++++++++++++------- cleveldb_test.go | 6 +++--- db.go | 14 ++++++++++++-- go.mod | 1 + go.sum | 1 + goleveldb.go | 26 +++++++++++++++++++++++--- memdb.go | 2 +- rocksdb.go | 40 +++++++++++++++++++++++++++------------- rocksdb_test.go | 21 +++++++++++++++++++-- util.go | 12 ++++++++++++ 14 files changed, 133 insertions(+), 46 deletions(-) diff --git a/backend_test.go b/backend_test.go index aecae4bf5..c8e8ed70c 100644 --- a/backend_test.go +++ b/backend_test.go @@ -2,7 +2,6 @@ package db import ( "fmt" - "io/ioutil" "os" "path/filepath" "testing" @@ -14,7 +13,7 @@ import ( // Register a test backend for PrefixDB as well, with some unrelated junk data func init() { // nolint: errcheck - registerDBCreator("prefixdb", func(name, dir string) (DB, error) { + registerDBCreator("prefixdb", func(name, dir string, opts Options) (DB, error) { mdb := NewMemDB() mdb.Set([]byte("a"), []byte{1}) mdb.Set([]byte("b"), []byte{2}) @@ -35,11 +34,11 @@ func cleanupDBDir(dir, name string) { func testBackendGetSetDelete(t *testing.T, backend BackendType) { // Default - dirname, err := ioutil.TempDir("", fmt.Sprintf("test_backend_%s_", backend)) - require.Nil(t, err) - db, err := NewDB("testdb", backend, dirname) + dbName := "testdb" + dir := os.TempDir() + db, err := NewDB(dbName, backend, dir) require.NoError(t, err) - defer cleanupDBDir(dirname, "testdb") + defer os.RemoveAll(fmt.Sprintf("%s/%s.db", dir, dbName)) // A nonexistent key should return nil. value, err := db.Get([]byte("a")) @@ -166,7 +165,7 @@ func testDBIterator(t *testing.T, backend BackendType) { dir := os.TempDir() db, err := NewDB(name, backend, dir) require.NoError(t, err) - defer cleanupDBDir(dir, name) + defer os.RemoveAll(fmt.Sprintf("%s/%s.db", dir, name)) for i := 0; i < 10; i++ { if i != 6 { // but skip 6. @@ -302,11 +301,11 @@ func testDBIterator(t *testing.T, backend BackendType) { []int64(nil), "reverse iterator from 2 (ex) to 4") // Ensure that the iterators don't panic with an empty database. - dir2, err := ioutil.TempDir("", "tm-db-test") require.NoError(t, err) - db2, err := NewDB(name, backend, dir2) + name2 := fmt.Sprintf("test_%x", randStr(11)) + db2, err := NewDB(name2, backend, dir) require.NoError(t, err) - defer cleanupDBDir(dir2, name) + defer os.RemoveAll(fmt.Sprintf("%s/%s.db", dir, name2)) itr, err = db2.Iterator(nil, nil) require.NoError(t, err) diff --git a/badger_db.go b/badger_db.go index 467ed3361..a83dc35d4 100644 --- a/badger_db.go +++ b/badger_db.go @@ -14,7 +14,7 @@ import ( func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) } -func badgerDBCreator(dbName, dir string) (DB, error) { +func badgerDBCreator(dbName, dir string, opts Options) (DB, error) { return NewBadgerDB(dbName, dir) } diff --git a/boltdb.go b/boltdb.go index b1f69983d..2dbedd458 100644 --- a/boltdb.go +++ b/boltdb.go @@ -32,7 +32,7 @@ type BoltDB struct { var _ DB = (*BoltDB)(nil) // NewBoltDB returns a BoltDB with default options. -func NewBoltDB(name, dir string) (DB, error) { +func NewBoltDB(name, dir string, opts Options) (DB, error) { return NewBoltDBWithOpts(name, dir, bbolt.DefaultOptions) } diff --git a/boltdb_test.go b/boltdb_test.go index 02a9eaf44..720000eed 100644 --- a/boltdb_test.go +++ b/boltdb_test.go @@ -17,7 +17,7 @@ func TestBoltDBNewBoltDB(t *testing.T) { dir := os.TempDir() defer cleanupDBDir(dir, name) - db, err := NewBoltDB(name, dir) + db, err := NewBoltDB(name, dir, nil) require.NoError(t, err) db.Close() } @@ -26,7 +26,7 @@ func TestWithBoltDB(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "boltdb") - db, err := NewBoltDB(path, "") + db, err := NewBoltDB(path, "", nil) require.NoError(t, err) t.Run("BoltDB", func(t *testing.T) { Run(t, db) }) @@ -34,7 +34,7 @@ func TestWithBoltDB(t *testing.T) { func BenchmarkBoltDBRandomReadsWrites(b *testing.B) { name := fmt.Sprintf("test_%x", randStr(12)) - db, err := NewBoltDB(name, "") + db, err := NewBoltDB(name, "", nil) if err != nil { b.Fatal(err) } diff --git a/cleveldb.go b/cleveldb.go index a4b467aad..4351a7275 100644 --- a/cleveldb.go +++ b/cleveldb.go @@ -8,11 +8,12 @@ import ( "path/filepath" "github.com/jmhodges/levigo" + "github.com/spf13/cast" ) func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewCLevelDB(name, dir) + dbCreator := func(name string, dir string, opts Options) (DB, error) { + return NewCLevelDB(name, dir, opts) } registerDBCreator(CLevelDBBackend, dbCreator, false) } @@ -27,14 +28,26 @@ type CLevelDB struct { var _ DB = (*CLevelDB)(nil) -// NewCLevelDB creates a new CLevelDB. -func NewCLevelDB(name string, dir string) (*CLevelDB, error) { - dbPath := filepath.Join(dir, name+".db") - +func defaultCleveldbOptions() *levigo.Options { opts := levigo.NewOptions() opts.SetCache(levigo.NewLRUCache(1 << 30)) opts.SetCreateIfMissing(true) - db, err := levigo.Open(dbPath, opts) + return opts +} + +// NewCLevelDB creates a new CLevelDB. +func NewCLevelDB(name string, dir string, opts Options) (*CLevelDB, error) { + do := defaultCleveldbOptions() + + if opts != nil { + files := cast.ToInt(opts.Get("maxopenfiles")) + if files > 0 { + do.SetMaxOpenFiles(files) + } + } + + dbPath := filepath.Join(dir, name+".db") + db, err := levigo.Open(dbPath, do) if err != nil { return nil, err } diff --git a/cleveldb_test.go b/cleveldb_test.go index 55fca6b9f..cd5d05461 100644 --- a/cleveldb_test.go +++ b/cleveldb_test.go @@ -19,13 +19,13 @@ func TestWithClevelDB(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "cleveldb") - db, err := NewCLevelDB(path, "") + db, err := NewCLevelDB(path, "", nil) require.NoError(t, err) t.Run("ClevelDB", func(t *testing.T) { Run(t, db) }) } -//nolint: errcheck +// nolint: errcheck func BenchmarkRandomReadsWrites2(b *testing.B) { b.StopTimer() @@ -34,7 +34,7 @@ func BenchmarkRandomReadsWrites2(b *testing.B) { for i := 0; i < int(numItems); i++ { internal[int64(i)] = int64(0) } - db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "") + db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "", nil) if err != nil { b.Fatal(err.Error()) return diff --git a/db.go b/db.go index ef573f17e..1e7011308 100644 --- a/db.go +++ b/db.go @@ -37,7 +37,13 @@ const ( BadgerDBBackend BackendType = "badgerdb" ) -type dbCreator func(name string, dir string) (DB, error) +type ( + dbCreator func(name string, dir string, opts Options) (DB, error) + + Options interface { + Get(string) interface{} + } +) var backends = map[BackendType]dbCreator{} @@ -51,6 +57,10 @@ func registerDBCreator(backend BackendType, creator dbCreator, force bool) { // NewDB creates a new database of type backend with the given name. func NewDB(name string, backend BackendType, dir string) (DB, error) { + return NewDBwithOptions(name, backend, dir, nil) +} + +func NewDBwithOptions(name string, backend BackendType, dir string, opts Options) (DB, error) { dbCreator, ok := backends[backend] if !ok { keys := make([]string, 0, len(backends)) @@ -61,7 +71,7 @@ func NewDB(name string, backend BackendType, dir string) (DB, error) { backend, strings.Join(keys, ",")) } - db, err := dbCreator(name, dir) + db, err := dbCreator(name, dir, opts) if err != nil { return nil, fmt.Errorf("failed to initialize database: %w", err) } diff --git a/go.mod b/go.mod index 1395b1d16..8492cced2 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.2 github.com/jmhodges/levigo v1.0.0 + github.com/spf13/cast v1.3.0 github.com/stretchr/testify v1.8.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca go.etcd.io/bbolt v1.3.6 diff --git a/go.sum b/go.sum index 0d1afaab2..e99dda567 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= diff --git a/goleveldb.go b/goleveldb.go index 8dd3c3ce8..f5658e5b0 100644 --- a/goleveldb.go +++ b/goleveldb.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" + "github.com/spf13/cast" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" @@ -11,8 +12,8 @@ import ( ) func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewGoLevelDB(name, dir) + dbCreator := func(name string, dir string, opts Options) (DB, error) { + return OpenGoLevelDBWithOpts(name, dir, opts) } registerDBCreator(GoLevelDBBackend, dbCreator, false) } @@ -23,10 +24,29 @@ type GoLevelDB struct { var _ DB = (*GoLevelDB)(nil) +// NewGoLevelDB open new go level database with default options. +// Note: tendermint(v0.34.21) backend db is using this function, +// it can be removed when tendermint discard the tm-db dependencies. func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { - return NewGoLevelDBWithOpts(name, dir, nil) + return OpenGoLevelDBWithOpts(name, dir, nil) } +// OpenGoLevelDBWithOpts open new go level database with default options +// and custom options for the db clients. +func OpenGoLevelDBWithOpts(name string, dir string, opts Options) (*GoLevelDB, error) { + defaultOpts := &opt.Options{} + + if opts != nil { + files := cast.ToInt(opts.Get("maxopenfiles")) + if files > 0 { + defaultOpts.OpenFilesCacheCapacity = files + } + } + + return NewGoLevelDBWithOpts(name, dir, defaultOpts) +} + +// NewGoLevelDBWithOpts open new go level database with leveldb options. func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) { dbPath := filepath.Join(dir, name+".db") db, err := leveldb.OpenFile(dbPath, o) diff --git a/memdb.go b/memdb.go index feb1664fc..5861811fb 100644 --- a/memdb.go +++ b/memdb.go @@ -14,7 +14,7 @@ const ( ) func init() { - registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) { + registerDBCreator(MemDBBackend, func(name, dir string, opts Options) (DB, error) { return NewMemDB(), nil }, false) } diff --git a/rocksdb.go b/rocksdb.go index 4994ecad5..34347e32b 100644 --- a/rocksdb.go +++ b/rocksdb.go @@ -9,11 +9,12 @@ import ( "runtime" "github.com/cosmos/gorocksdb" + "github.com/spf13/cast" ) func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewRocksDB(name, dir) + dbCreator := func(name string, dir string, opts Options) (DB, error) { + return NewRocksDB(name, dir, opts) } registerDBCreator(RocksDBBackend, dbCreator, false) } @@ -28,23 +29,36 @@ type RocksDB struct { var _ DB = (*RocksDB)(nil) -func NewRocksDB(name string, dir string) (*RocksDB, error) { - // default rocksdb option, good enough for most cases, including heavy workloads. - // 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). - // compression: snappy as default, need to -lsnappy to enable. +// defaultRocksdbOptions, good enough for most cases, including heavy workloads. +// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). +// compression: snappy as default, need to -lsnappy to enable. +func defaultRocksdbOptions() *gorocksdb.Options { bbto := gorocksdb.NewDefaultBlockBasedTableOptions() bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30)) bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(10)) - opts := gorocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) + rocksdbOpts := gorocksdb.NewDefaultOptions() + rocksdbOpts.SetBlockBasedTableFactory(bbto) // SetMaxOpenFiles to 4096 seems to provide a reliable performance boost - opts.SetMaxOpenFiles(4096) - opts.SetCreateIfMissing(true) - opts.IncreaseParallelism(runtime.NumCPU()) + rocksdbOpts.SetMaxOpenFiles(4096) + rocksdbOpts.SetCreateIfMissing(true) + rocksdbOpts.IncreaseParallelism(runtime.NumCPU()) // 1.5GB maximum memory use for writebuffer. - opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) - return NewRocksDBWithOptions(name, dir, opts) + rocksdbOpts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) + return rocksdbOpts +} + +func NewRocksDB(name string, dir string, opts Options) (*RocksDB, error) { + defaultOpts := defaultRocksdbOptions() + + if opts != nil { + files := cast.ToInt(opts.Get("maxopenfiles")) + if files > 0 { + defaultOpts.SetMaxOpenFiles(files) + } + } + + return NewRocksDBWithOptions(name, dir, defaultOpts) } func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) { diff --git a/rocksdb_test.go b/rocksdb_test.go index bd6793420..b176fe890 100644 --- a/rocksdb_test.go +++ b/rocksdb_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "testing" + "github.com/spf13/cast" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -28,7 +29,7 @@ func TestWithRocksDB(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "rocksdb") - db, err := NewRocksDB(path, "") + db, err := NewRocksDB(path, "", nil) require.NoError(t, err) t.Run("RocksDB", func(t *testing.T) { Run(t, db) }) @@ -44,4 +45,20 @@ func TestRocksDBStats(t *testing.T) { assert.NotEmpty(t, db.Stats()) } -// TODO: Add tests for rocksdb +func TestRocksDBWithOptions(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rocksdb") + + opts := make(OptionsMap, 0) + opts["maxopenfiles"] = 1000 + + defaultOpts := defaultRocksdbOptions() + files := cast.ToInt(opts.Get("maxopenfiles")) + defaultOpts.SetMaxOpenFiles(files) + require.Equal(t, opts["maxopenfiles"], defaultOpts.GetMaxOpenFiles()) + + db, err := NewRocksDB(path, "", opts) + require.NoError(t, err) + + t.Run("RocksDB", func(t *testing.T) { Run(t, db) }) +} diff --git a/util.go b/util.go index da0b635c1..da2b005c5 100644 --- a/util.go +++ b/util.go @@ -49,3 +49,15 @@ func FileExists(filePath string) bool { _, err := os.Stat(filePath) return !os.IsNotExist(err) } + +// OptionsMap is a stub implementing Options which can get data from a map +type OptionsMap map[string]interface{} + +func (m OptionsMap) Get(key string) interface{} { + v, ok := m[key] + if !ok { + return interface{}(nil) + } + + return v +}