diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e889fe439..440bc907b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,3 @@ # CODEOWNERS: https://help.github.com/articles/about-codeowners/ -# Everything goes through Anton and Tess. -* @melekes @tessr +* @alexanderbez @cmwaters @creachadair @marbar3778 @tychoish @williambanfield diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69a312899..a8beee60b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,14 +14,17 @@ jobs: if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'" Test: + # The custom image here contains pre-built libraries for leveldb and + # rocksdb, which are needed to build and test those modules. + # To update the container image, see docker.yml. runs-on: ubuntu-latest container: tendermintdev/docker-tm-db-testing steps: - uses: actions/checkout@v2 - name: test & coverage report creation run: | - go test ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic -tags cleveldb,boltdb,rocksdb,badgerdb -v - - uses: codecov/codecov-action@v1 + CGO_ENABLED=1 go test ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic -tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,badgerdb -v + - uses: codecov/codecov-action@v2.0.3 with: file: ./coverage.txt fail_ci_if_error: true diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a675ecb6c..1eb94db47 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,3 +1,11 @@ +# This workflow builds and pushes a new version of the build container image +# when the tools directory changes on master. Edit tools/Dockerfile. +# +# This workflow does not push a new image until it is merged, so tests that +# depend on changes in this image will not pass until this workflow succeeds. +# For that reason, changes here should be done in a separate PR in advance of +# work that depends on them. + name: Build & Push TM-DB-Testing on: pull_request: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0a90a2653..15e3b20cb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,12 +7,13 @@ on: jobs: golangci: + # We need to run the linter on the same image we use for building, since it + # needs the C libraries installed for the dependencies to typecheck. runs-on: ubuntu-latest + container: tendermintdev/docker-tm-db-testing steps: - uses: actions/checkout@v2 - - uses: golangci/golangci-lint-action@v2.3.0 + - uses: golangci/golangci-lint-action@v2 with: - # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.30 args: --timeout 10m github-token: ${{ secrets.github_token }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 610e2e29e..032ca431e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,12 +7,14 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v3 + - uses: actions/stale@v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions." - days-before-stale: 10 - days-before-close: 4 - exempt-pr-labels: "S:wip" + days-before-stale: -1 + days-before-close: -1 + days-before-pr-stale: 20 + days-before-pr-close: 10 + exempt-pr-labels: "pinned, security, proposal, blocked" diff --git a/.golangci.yml b/.golangci.yml index 39ba9aaf5..abf03e7ac 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -16,7 +16,7 @@ linters: # - godox - gofmt - goimports - - golint + - revive - gosec - gosimple - govet @@ -24,7 +24,6 @@ linters: # - interfacer - lll - misspell - - maligned - nakedret - prealloc # - scopelint diff --git a/.mergify.yml b/.mergify.yml index 136bb1148..ece75d8f2 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,9 +1,15 @@ +queue_rules: + - name: default + conditions: + - base=master + - label=S:automerge + pull_request_rules: - name: automerge to master with label S:automerge and branch protection passing conditions: - base=master - label=S:automerge actions: - merge: + queue: method: squash - strict: true + name: default diff --git a/CHANGELOG.md b/CHANGELOG.md index cb43176ca..a3071cd86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,37 @@ * Add an option for creating a mem db with no mutex +## 0.6.7 + +**2022-2-21** + +- Use cosmos/rocksdb instead of techbot/rocksdb +- Add `ForceCopmact` to goleveldb database + +## 0.6.6 + +**2021-11-08** + +**Important note:** Version v0.6.5 was accidentally tagged and should be +avoided. This version is identical to v0.6.4 in package structure and API, but +has updated the version marker so that normal `go get` upgrades will not +require modifying existing use of v0.6.4. + +### Version bumps (since v0.6.4) + +- Bump grpc from to 1.42.0. +- Bump dgraph/badger to v2 2.2007.3. +- Bump go.etcd.io/bbolt to 1.3.6. + +## 0.6.5 + +**2021-08-04** + +**Important note**: This version was tagged by accident, and should not be +used. The tag now points to the [package-reorg +branch](https://github.com/tendermint/tm-db/tree/package-reorg) so that +any existing dependencies should not break. + ## 0.6.4 **2021-02-09** diff --git a/README.md b/README.md index 42d9adc29..2aa6efb53 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Go 1.13+ - **[BoltDB](https://github.com/etcd-io/bbolt) [experimental]:** A [fork](https://github.com/etcd-io/bbolt) of [BoltDB](https://github.com/boltdb/bolt). Uses B+trees for on-disk storage, which have good performance for read-heavy workloads and range scans. Supports serializable ACID transactions. -- **[RocksDB](https://github.com/tecbot/gorocksdb) [experimental]:** A [Go wrapper](https://github.com/tecbot/gorocksdb) around [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees for on-disk storage, but is optimized for fast storage media such as SSDs and memory. Supports atomic transactions, but not full ACID transactions. +- **[RocksDB](https://github.com/cosmos/gorocksdb) [experimental]:** A [Go wrapper](https://github.com/cosmos/gorocksdb) around [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees for on-disk storage, but is optimized for fast storage media such as SSDs and memory. Supports atomic transactions, but not full ACID transactions. - **[BadgerDB](https://github.com/dgraph-io/badger) [experimental]:** A key-value database written as a pure-Go alternative to e.g. LevelDB and RocksDB, with LSM-tree storage. Makes use of multiple goroutines for performance, and includes advanced features such as serializable ACID transactions, write batches, compression, and more. diff --git a/backend_test.go b/backend_test.go index 63630beca..782fc472a 100644 --- a/backend_test.go +++ b/backend_test.go @@ -14,7 +14,7 @@ import ( // Register a test backend for PrefixDB as well, with some unrelated junk data func init() { // nolint: errcheck - registerDBCreator("prefixdb", func(name, dir string) (DB, error) { + registerDBCreator("prefixdb", func(name, dir string, opts Options) (DB, error) { mdb := NewMemDB() mdb.Set([]byte("a"), []byte{1}) mdb.Set([]byte("b"), []byte{2}) diff --git a/cleveldb.go b/cleveldb.go index 377956deb..f2665108c 100644 --- a/cleveldb.go +++ b/cleveldb.go @@ -10,8 +10,8 @@ import ( ) func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewCLevelDB(name, dir) + dbCreator := func(name string, dir string, opts Options) (DB, error) { + return NewCLevelDB(name, dir, opts) } registerDBCreator(CLevelDBBackend, dbCreator, false) } @@ -26,14 +26,26 @@ type CLevelDB struct { var _ DB = (*CLevelDB)(nil) -// NewCLevelDB creates a new CLevelDB. -func NewCLevelDB(name string, dir string) (*CLevelDB, error) { - dbPath := filepath.Join(dir, name+".db") - +func defaultCleveldbOptions() *levigo.Options { opts := levigo.NewOptions() opts.SetCache(levigo.NewLRUCache(1 << 30)) opts.SetCreateIfMissing(true) - db, err := levigo.Open(dbPath, opts) + return opts +} + +// NewCLevelDB creates a new CLevelDB. +func NewCLevelDB(name string, dir string, opts Options) (*CLevelDB, error) { + do := defaultCleveldbOptions() + + if opts != nil { + files := cast.ToInt(opts.Get("maxopenfiles")) + if files > 0 { + do.SetMaxOpenFiles(files) + } + } + + dbPath := filepath.Join(dir, name+".db") + db, err := levigo.Open(dbPath, do) if err != nil { return nil, err } diff --git a/common_test.go b/common_test.go index f48efd7aa..e73a2fc5e 100644 --- a/common_test.go +++ b/common_test.go @@ -127,7 +127,6 @@ func benchmarkRandomReadsWrites(b *testing.B, db DB) { val := internal[idx] idxBytes := int642Bytes(idx) valBytes := int642Bytes(val) - //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) err := db.Set(idxBytes, valBytes) if err != nil { // require.NoError() is very expensive (according to profiler), so check manually @@ -145,7 +144,6 @@ func benchmarkRandomReadsWrites(b *testing.B, db DB) { // require.NoError() is very expensive (according to profiler), so check manually b.Fatal(b, err) } - //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) if valExp == 0 { if !bytes.Equal(valBytes, nil) { b.Errorf("Expected %v for %v, got %X", nil, idx, valBytes) diff --git a/db.go b/db.go index 4d518c0b0..acffb350f 100644 --- a/db.go +++ b/db.go @@ -28,7 +28,7 @@ const ( // - may be faster is some use-cases (random reads - indexer) // - use boltdb build tag (go build -tags boltdb) BoltDBBackend BackendType = "boltdb" - // RocksDBBackend represents rocksdb (uses github.com/tecbot/gorocksdb) + // RocksDBBackend represents rocksdb (uses github.com/cosmos/gorocksdb) // - EXPERIMENTAL // - requires gcc // - use rocksdb build tag (go build -tags rocksdb) @@ -37,7 +37,13 @@ const ( BadgerDBBackend BackendType = "badgerdb" ) -type dbCreator func(name string, dir string) (DB, error) +type ( + dbCreator func(name string, dir string, opts Options) (DB, error) + + Options interface { + Get(string) interface{} + } +) var backends = map[BackendType]dbCreator{} @@ -51,6 +57,11 @@ func registerDBCreator(backend BackendType, creator dbCreator, force bool) { // NewDB creates a new database of type backend with the given name. func NewDB(name string, backend BackendType, dir string) (DB, error) { + return NewDBwithOptions(name, backend, dir, nil) +} + +// NewDBwithOptions creates a new database of type backend with the given name and the db options. +func NewDBwithOptions(name string, backend BackendType, dir string, opts Options) (DB, error) { dbCreator, ok := backends[backend] if !ok { keys := make([]string, 0, len(backends)) @@ -61,7 +72,7 @@ func NewDB(name string, backend BackendType, dir string) (DB, error) { backend, strings.Join(keys, ",")) } - db, err := dbCreator(name, dir) + db, err := dbCreator(name, dir, opts) if err != nil { return nil, fmt.Errorf("failed to initialize database: %w", err) } diff --git a/docs/how_to_release.md b/docs/how_to_release.md new file mode 100644 index 000000000..59ac878a4 --- /dev/null +++ b/docs/how_to_release.md @@ -0,0 +1,30 @@ +# How To Release tm db + +This document provides a step-by-step guide for creating a release of tm-db. + +## 1. Update the changelog + +Open the `CHANGELOG.md` at the root of the repository. +Amend the top of this file with a section for the latest version (0.6.x etc). +Be sure to include any bug fixes, improvements, dependency upgrades, and breaking changes included in this version. +(It's OK to exclude changes to tooling dependencies, like updates to Github Actions.) +Finally, create a pull request for the changelog update. +Once the tests pass and the pull request is approved, merge the change into master. + +## 2. Tag the latest commit with the latest version + +tm-db is provided as a golang [module](https://blog.golang.org/publishing-go-modules), which rely on git tags for versioning information. + +Tag the changelog commit in master created in step 1 with the latest version. +Be sure to prefix the version tag with `v`. For example, `v0.6.5` for version 0.6.5. +This tagging can be done [using github](https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/managing-commits/managing-tags#creating-a-tag) or [using git](https://git-scm.com/book/en/v2/Git-Basics-Tagging) on the command line. + +Note that the golang modules tooling expects tags to be immutable. +If you make a mistake after pushing a tag, make a new tag and start over rather than fix and re-push the old tag. +## 3. Create a github release + +Finally, create a github release. +To create a github release, follow the steps in the [github release documentation](https://docs.github.com/en/github/administering-a-repository/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release). + +When creating the github release, select the `Tag version` created in step 2. +Use the version tag as the release title and paste in the changelog information for this release in the `Describe this release` section. diff --git a/go.mod b/go.mod index 03bf90110..85091f5e2 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,38 @@ module github.com/tendermint/tm-db -go 1.12 +go 1.17 require ( + github.com/cosmos/gorocksdb v1.2.0 github.com/dgraph-io/badger/v2 v2.2007.2 - github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect - github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect - github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect github.com/gogo/protobuf v1.3.2 + github.com/golang/protobuf v1.4.3 // indirect github.com/google/btree v1.0.0 github.com/jmhodges/levigo v1.0.0 + github.com/spf13/cast v1.3.0 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca - github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c - go.etcd.io/bbolt v1.3.5 - google.golang.org/grpc v1.35.0 + go.etcd.io/bbolt v1.3.6 + google.golang.org/grpc v1.44.0 ) + +require ( + github.com/DataDog/zstd v1.4.1 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/pkg/errors v0.8.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect + golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect + golang.org/x/text v0.3.3 // indirect + google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect +) + +// Breaking changes were released with the wrong tag (use v0.6.6 or later). +retract v0.6.5 diff --git a/go.sum b/go.sum index e87eb3e4a..e8dcd28f5 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,29 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= +github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -28,50 +37,50 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= @@ -101,17 +110,18 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -120,14 +130,13 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -141,42 +150,40 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed h1:J22ig1FUekjjkmZUM7pTKixYm8DvrYsvrBZdunYeIuQ= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= @@ -191,45 +198,42 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/goleveldb.go b/goleveldb.go index fd1bffd18..072ea48fa 100644 --- a/goleveldb.go +++ b/goleveldb.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" + "github.com/spf13/cast" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" @@ -11,8 +12,8 @@ import ( ) func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewGoLevelDB(name, dir) + dbCreator := func(name string, dir string, opts Options) (DB, error) { + return NewGoLevelDBWithDbOpts(name, dir, opts) } registerDBCreator(GoLevelDBBackend, dbCreator, false) } @@ -23,10 +24,24 @@ type GoLevelDB struct { var _ DB = (*GoLevelDB)(nil) +// NewGoLevelDB creates a new GoLevelDB. func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { return NewGoLevelDBWithOpts(name, dir, nil) } +func NewGoLevelDBWithDbOpts(name string, dir string, opts Options) (*GoLevelDB, error) { + defaultOpts := &opt.Options{} + + if opts != nil { + files := cast.ToInt(opts.Get("maxopenfiles")) + if files > 0 { + defaultOpts.OpenFilesCacheCapacity = files + } + } + + return NewGoLevelDBWithOpts(name, dir, defaultOpts) +} + func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) { dbPath := filepath.Join(dir, name+".db") db, err := leveldb.OpenFile(dbPath, o) @@ -166,6 +181,10 @@ func (db *GoLevelDB) Stats() map[string]string { return stats } +func (db *GoLevelDB) ForceCompact(start, limit []byte) error { + return db.db.CompactRange(util.Range{Start: start, Limit: limit}) +} + // NewBatch implements DB. func (db *GoLevelDB) NewBatch() Batch { return newGoLevelDBBatch(db) diff --git a/memdb.go b/memdb.go index d019af230..c065e5035 100644 --- a/memdb.go +++ b/memdb.go @@ -14,7 +14,7 @@ const ( ) func init() { - registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) { + registerDBCreator(MemDBBackend, func(name, dir string, opts Options) (DB, error) { return NewMemDB(), nil }, false) } diff --git a/rocksdb.go b/rocksdb.go index c3ec29f40..a0cb6bb99 100644 --- a/rocksdb.go +++ b/rocksdb.go @@ -7,12 +7,13 @@ import ( "path/filepath" "runtime" - "github.com/tecbot/gorocksdb" + "github.com/cosmos/gorocksdb" + "github.com/spf13/cast" ) func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewRocksDB(name, dir) + dbCreator := func(name string, dir string, opts Options) (DB, error) { + return NewRocksDB(name, dir, opts) } registerDBCreator(RocksDBBackend, dbCreator, false) } @@ -27,21 +28,36 @@ type RocksDB struct { var _ DB = (*RocksDB)(nil) -func NewRocksDB(name string, dir string) (*RocksDB, error) { - // default rocksdb option, good enough for most cases, including heavy workloads. - // 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). - // compression: snappy as default, need to -lsnappy to enable. +// defaultRocksdbOptions, good enough for most cases, including heavy workloads. +// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). +// compression: snappy as default, need to -lsnappy to enable. +func defaultRocksdbOptions() *grocksdb.Options { bbto := gorocksdb.NewDefaultBlockBasedTableOptions() bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30)) bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(10)) - opts := gorocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.IncreaseParallelism(runtime.NumCPU()) + rocksdbOpts := grocksdb.NewDefaultOptions() + rocksdbOpts.SetBlockBasedTableFactory(bbto) + // SetMaxOpenFiles to 4096 seems to provide a reliable performance boost + rocksdbOpts.SetMaxOpenFiles(4096) + rocksdbOpts.SetCreateIfMissing(true) + rocksdbOpts.IncreaseParallelism(runtime.NumCPU()) // 1.5GB maximum memory use for writebuffer. - opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) - return NewRocksDBWithOptions(name, dir, opts) + rocksdbOpts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) + return rocksdbOpts +} + +func NewRocksDB(name string, dir string, opts Options) (*RocksDB, error) { + defaultOpts := defaultRocksdbOptions() + + if opts != nil { + files := cast.ToInt(opts.Get("maxopenfiles")) + if files > 0 { + defaultOpts.SetMaxOpenFiles(files) + } + } + + return NewRocksDBWithOptions(name, dir, defaultOpts) } func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) { diff --git a/rocksdb_batch.go b/rocksdb_batch.go index a5d296b40..af7e65c60 100644 --- a/rocksdb_batch.go +++ b/rocksdb_batch.go @@ -2,7 +2,7 @@ package db -import "github.com/tecbot/gorocksdb" +import "github.com/cosmos/gorocksdb" type rocksDBBatch struct { db *RocksDB diff --git a/rocksdb_iterator.go b/rocksdb_iterator.go index 0e7d405fe..9970c169e 100644 --- a/rocksdb_iterator.go +++ b/rocksdb_iterator.go @@ -5,7 +5,7 @@ package db import ( "bytes" - "github.com/tecbot/gorocksdb" + "github.com/cosmos/gorocksdb" ) type rocksDBIterator struct { diff --git a/rocksdb_test.go b/rocksdb_test.go index 6bbe51133..95d124286 100644 --- a/rocksdb_test.go +++ b/rocksdb_test.go @@ -32,4 +32,20 @@ func TestRocksDBStats(t *testing.T) { assert.NotEmpty(t, db.Stats()) } -// TODO: Add tests for rocksdb +func TestRocksDBWithOptions(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rocksdb") + + opts := make(OptionsMap, 0) + opts["maxopenfiles"] = 1000 + + defaultOpts := defaultRocksdbOptions() + files := cast.ToInt(opts.Get("maxopenfiles")) + defaultOpts.SetMaxOpenFiles(files) + require.Equal(t, opts["maxopenfiles"], defaultOpts.GetMaxOpenFiles()) + + db, err := NewRocksDB(path, "", opts) + require.NoError(t, err) + + t.Run("RocksDB", func(t *testing.T) { Run(t, db) }) +} diff --git a/tools/Dockerfile b/tools/Dockerfile index 2c26e0b9c..855414aab 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -1,30 +1,44 @@ -FROM golang +# This file defines the container image used to build and test tm-db in CI. +# The CI workflows use the latest tag of tendermintdev/docker-tm-db-testing +# built from these settings. +# +# The jobs defined in the Build & Push workflow will build and update the image +# when changes to this file are merged. If you have other changes that require +# updates here, merge the changes here first and let the image get updated (or +# push a new version manually) before PRs that depend on them. + +FROM golang:1.17-bullseye AS build ENV LD_LIBRARY_PATH=/usr/local/lib RUN apt-get update && apt-get install -y --no-install-recommends \ - libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev libzstd-dev make tar wget + libbz2-dev libgflags-dev libsnappy-dev libzstd-dev zlib1g-dev \ + make tar wget + +FROM build AS install +ARG LEVELDB=1.20 +ARG ROCKSDB=6.24.2 # Install cleveldb RUN \ - wget https://github.com/google/leveldb/archive/v1.20.tar.gz \ - && tar xvf v1.20.tar.gz \ - && cd leveldb-1.20 \ + wget -q https://github.com/google/leveldb/archive/v${LEVELDB}.tar.gz \ + && tar xvf v${LEVELDB}.tar.gz \ + && cd leveldb-${LEVELDB} \ && make \ && cp -a out-static/lib* out-shared/lib* /usr/local/lib \ && cd include \ && cp -a leveldb /usr/local/include \ && ldconfig \ && cd ../.. \ - && rm -rf v1.20.tar.gz leveldb-1.20 + && rm -rf v${LEVELDB}.tar.gz leveldb-${LEVELDB} # Install Rocksdb RUN \ - wget https://github.com/facebook/rocksdb/archive/v6.6.4.tar.gz \ - && tar -zxf v6.6.4.tar.gz \ - && cd rocksdb-6.6.4 \ + wget -q https://github.com/facebook/rocksdb/archive/v${ROCKSDB}.tar.gz \ + && tar -zxf v${ROCKSDB}.tar.gz \ + && cd rocksdb-${ROCKSDB} \ && DEBUG_LEVEL=0 make -j4 shared_lib \ && make install-shared \ && ldconfig \ && cd .. \ - && rm -rf v6.6.4.tar.gz rocksdb-6.6.4 + && rm -rf v${ROCKSDB}.tar.gz rocksdb-${ROCKSDB} diff --git a/util.go b/util.go index da0b635c1..da2b005c5 100644 --- a/util.go +++ b/util.go @@ -49,3 +49,15 @@ func FileExists(filePath string) bool { _, err := os.Stat(filePath) return !os.IsNotExist(err) } + +// OptionsMap is a stub implementing Options which can get data from a map +type OptionsMap map[string]interface{} + +func (m OptionsMap) Get(key string) interface{} { + v, ok := m[key] + if !ok { + return interface{}(nil) + } + + return v +} diff --git a/util_test.go b/util_test.go index d58cd2fd7..411abe1ee 100644 --- a/util_test.go +++ b/util_test.go @@ -113,7 +113,7 @@ func TestPrefixIteratorMatches1N(t *testing.T) { // Bad! checkNext(t, itr, false) - //Once invalid... + // Once invalid... checkInvalid(t, itr) }) }