implement DB options for customizing maxopenfiles

.github/CODEOWNERS

@@ -1,4 +1,3 @@
 # CODEOWNERS: https://help.github.com/articles/about-codeowners/
 
-# Everything goes through Anton and Tess.
-*       @melekes @tessr
+*       @alexanderbez @cmwaters @creachadair @marbar3778 @tychoish @williambanfield

.github/workflows/ci.yml

@@ -14,14 +14,17 @@ jobs:
     if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'"
 
   Test:
+    # The custom image here contains pre-built libraries for leveldb and
+    # rocksdb, which are needed to build and test those modules.
+    # To update the container image, see docker.yml.
     runs-on: ubuntu-latest
     container: tendermintdev/docker-tm-db-testing
     steps:
       - uses: actions/checkout@v2
       - name: test & coverage report creation
         run: |
-          go test ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic -tags cleveldb,boltdb,rocksdb,badgerdb -v
-      - uses: codecov/codecov-action@v1
+          CGO_ENABLED=1 go test ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic -tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,badgerdb -v
+      - uses: codecov/codecov-action@v2.0.3
         with:
           file: ./coverage.txt
           fail_ci_if_error: true  

.github/workflows/docker.yml

@@ -1,3 +1,11 @@
+# This workflow builds and pushes a new version of the build container image
+# when the tools directory changes on master. Edit tools/Dockerfile.
+#
+# This workflow does not push a new image until it is merged, so tests that
+# depend on changes in this image will not pass until this workflow succeeds.
+# For that reason, changes here should be done in a separate PR in advance of
+# work that depends on them.
+
 name: Build & Push TM-DB-Testing
 on:
   pull_request:

.github/workflows/lint.yml

@@ -7,12 +7,13 @@ on:
 
 jobs:
   golangci:
+    # We need to run the linter on the same image we use for building, since it
+    # needs the C libraries installed for the dependencies to typecheck.
     runs-on: ubuntu-latest
+    container: tendermintdev/docker-tm-db-testing
     steps:
       - uses: actions/checkout@v2
-      - uses: golangci/golangci-lint-action@v2.3.0
+      - uses: golangci/golangci-lint-action@v2
         with:
-          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.30
           args: --timeout 10m
           github-token: ${{ secrets.github_token }}

.github/workflows/stale.yml

@@ -7,12 +7,14 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v3
+      - uses: actions/stale@v4
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           stale-pr-message: "This pull request has been automatically marked as stale because it has not had
     recent activity. It will be closed if no further activity occurs. Thank you
     for your contributions."
-          days-before-stale: 10
-          days-before-close: 4
-          exempt-pr-labels: "S:wip"
+          days-before-stale: -1
+          days-before-close: -1
+          days-before-pr-stale: 20
+          days-before-pr-close: 10
+          exempt-pr-labels: "pinned, security, proposal, blocked"

.golangci.yml

@@ -16,15 +16,14 @@ linters:
     # - godox
     - gofmt
     - goimports
-    - golint
+    - revive
     - gosec
     - gosimple
     - govet
     - ineffassign
     # - interfacer
     - lll
     - misspell
-    - maligned
     - nakedret
     - prealloc
     # - scopelint

.mergify.yml

@@ -1,9 +1,15 @@
+queue_rules:
+  - name: default
+    conditions:
+      - base=master
+      - label=S:automerge
+
 pull_request_rules:
   - name: automerge to master with label S:automerge and branch protection passing
     conditions:
       - base=master
       - label=S:automerge
     actions:
-      merge:
+      queue:
         method: squash
-        strict: true
+        name: default

CHANGELOG.md

@@ -4,6 +4,37 @@
 
 * Add an option for creating a mem db with no mutex
 
+## 0.6.7
+
+**2022-2-21**
+
+- Use cosmos/rocksdb instead of techbot/rocksdb
+- Add `ForceCopmact` to goleveldb database
+
+## 0.6.6
+
+**2021-11-08**
+
+**Important note:** Version v0.6.5 was accidentally tagged and should be
+avoided.  This version is identical to v0.6.4 in package structure and API, but
+has updated the version marker so that normal `go get` upgrades will not
+require modifying existing use of v0.6.4.
+
+### Version bumps (since v0.6.4)
+
+- Bump grpc from to 1.42.0.
+- Bump dgraph/badger to v2 2.2007.3.
+- Bump go.etcd.io/bbolt to 1.3.6.
+
+## 0.6.5
+
+**2021-08-04**
+
+**Important note**: This version was tagged by accident, and should not be
+used. The tag now points to the [package-reorg
+branch](https://github.com/tendermint/tm-db/tree/package-reorg) so that
+any existing dependencies should not break.
+
 ## 0.6.4
 
 **2021-02-09**

README.md

@@ -24,7 +24,7 @@ Go 1.13+
 
 - **[BoltDB](https://github.com/etcd-io/bbolt) [experimental]:** A [fork](https://github.com/etcd-io/bbolt) of [BoltDB](https://github.com/boltdb/bolt). Uses B+trees for on-disk storage, which have good performance for read-heavy workloads and range scans. Supports serializable ACID transactions.
 
-- **[RocksDB](https://github.com/tecbot/gorocksdb) [experimental]:** A [Go wrapper](https://github.com/tecbot/gorocksdb) around [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees for on-disk storage, but is optimized for fast storage media such as SSDs and memory. Supports atomic transactions, but not full ACID transactions.
+- **[RocksDB](https://github.com/cosmos/gorocksdb) [experimental]:** A [Go wrapper](https://github.com/cosmos/gorocksdb) around [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees for on-disk storage, but is optimized for fast storage media such as SSDs and memory. Supports atomic transactions, but not full ACID transactions.
 
 - **[BadgerDB](https://github.com/dgraph-io/badger) [experimental]:** A key-value database written as a pure-Go alternative to e.g. LevelDB and RocksDB, with LSM-tree storage. Makes use of multiple goroutines for performance, and includes advanced features such as serializable ACID transactions, write batches, compression, and more.
 

backend_test.go

@@ -14,7 +14,7 @@ import (
 // Register a test backend for PrefixDB as well, with some unrelated junk data
 func init() {
 	// nolint: errcheck
-	registerDBCreator("prefixdb", func(name, dir string) (DB, error) {
+	registerDBCreator("prefixdb", func(name, dir string, opts Options) (DB, error) {
 		mdb := NewMemDB()
 		mdb.Set([]byte("a"), []byte{1})
 		mdb.Set([]byte("b"), []byte{2})

cleveldb.go

@@ -10,8 +10,8 @@ import (
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewCLevelDB(name, dir)
+	dbCreator := func(name string, dir string, opts Options) (DB, error) {
+		return NewCLevelDB(name, dir, opts)
 	}
 	registerDBCreator(CLevelDBBackend, dbCreator, false)
 }
@@ -26,14 +26,26 @@ type CLevelDB struct {
 
 var _ DB = (*CLevelDB)(nil)
 
-// NewCLevelDB creates a new CLevelDB.
-func NewCLevelDB(name string, dir string) (*CLevelDB, error) {
-	dbPath := filepath.Join(dir, name+".db")
-
+func defaultCleveldbOptions() *levigo.Options {
 	opts := levigo.NewOptions()
 	opts.SetCache(levigo.NewLRUCache(1 << 30))
 	opts.SetCreateIfMissing(true)
-	db, err := levigo.Open(dbPath, opts)
+	return opts
+}
+
+// NewCLevelDB creates a new CLevelDB.
+func NewCLevelDB(name string, dir string, opts Options) (*CLevelDB, error) {
+	do := defaultCleveldbOptions()
+
+	if opts != nil {
+		files := cast.ToInt(opts.Get("maxopenfiles"))
+		if files > 0 {
+			do.SetMaxOpenFiles(files)
+		}
+	}
+
+	dbPath := filepath.Join(dir, name+".db")
+	db, err := levigo.Open(dbPath, do)
 	if err != nil {
 		return nil, err
 	}

common_test.go

@@ -127,7 +127,6 @@ func benchmarkRandomReadsWrites(b *testing.B, db DB) {
 			val := internal[idx]
 			idxBytes := int642Bytes(idx)
 			valBytes := int642Bytes(val)
-			//fmt.Printf("Set %X -> %X\n", idxBytes, valBytes)
 			err := db.Set(idxBytes, valBytes)
 			if err != nil {
 				// require.NoError() is very expensive (according to profiler), so check manually
@@ -145,7 +144,6 @@ func benchmarkRandomReadsWrites(b *testing.B, db DB) {
 				// require.NoError() is very expensive (according to profiler), so check manually
 				b.Fatal(b, err)
 			}
-			//fmt.Printf("Get %X -> %X\n", idxBytes, valBytes)
 			if valExp == 0 {
 				if !bytes.Equal(valBytes, nil) {
 					b.Errorf("Expected %v for %v, got %X", nil, idx, valBytes)

db.go

@@ -28,7 +28,7 @@ const (
 	//   - may be faster is some use-cases (random reads - indexer)
 	//   - use boltdb build tag (go build -tags boltdb)
 	BoltDBBackend BackendType = "boltdb"
-	// RocksDBBackend represents rocksdb (uses github.com/tecbot/gorocksdb)
+	// RocksDBBackend represents rocksdb (uses github.com/cosmos/gorocksdb)
 	//   - EXPERIMENTAL
 	//   - requires gcc
 	//   - use rocksdb build tag (go build -tags rocksdb)
@@ -37,7 +37,13 @@ const (
 	BadgerDBBackend BackendType = "badgerdb"
 )
 
-type dbCreator func(name string, dir string) (DB, error)
+type (
+	dbCreator func(name string, dir string, opts Options) (DB, error)
+
+	Options interface {
+		Get(string) interface{}
+	}
+)
 
 var backends = map[BackendType]dbCreator{}
 
@@ -51,6 +57,11 @@ func registerDBCreator(backend BackendType, creator dbCreator, force bool) {
 
 // NewDB creates a new database of type backend with the given name.
 func NewDB(name string, backend BackendType, dir string) (DB, error) {
+	return NewDBwithOptions(name, backend, dir, nil)
+}
+
+// NewDBwithOptions creates a new database of type backend with the given name and the db options.
+func NewDBwithOptions(name string, backend BackendType, dir string, opts Options) (DB, error) {
 	dbCreator, ok := backends[backend]
 	if !ok {
 		keys := make([]string, 0, len(backends))
@@ -61,7 +72,7 @@ func NewDB(name string, backend BackendType, dir string) (DB, error) {
 			backend, strings.Join(keys, ","))
 	}
 
-	db, err := dbCreator(name, dir)
+	db, err := dbCreator(name, dir, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize database: %w", err)
 	}

docs/how_to_release.md

@@ -0,0 +1,30 @@
+# How To Release tm db
+
+This document provides a step-by-step guide for creating a release of tm-db.
+
+## 1. Update the changelog
+
+Open the `CHANGELOG.md` at the root of the repository. 
+Amend the top of this file with a section for the latest version (0.6.x etc).
+Be sure to include any bug fixes, improvements, dependency upgrades, and breaking changes included in this version. 
+(It's OK to exclude changes to tooling dependencies, like updates to Github Actions.)
+Finally, create a pull request for the changelog update.
+Once the tests pass and the pull request is approved, merge the change into master.
+
+## 2. Tag the latest commit with the latest version
+
+tm-db is provided as a golang [module](https://blog.golang.org/publishing-go-modules), which rely on git tags for versioning information. 
+
+Tag the changelog commit in master created in step 1 with the latest version.
+Be sure to prefix the version tag with `v`. For example, `v0.6.5` for version 0.6.5.
+This tagging can be done [using github](https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/managing-commits/managing-tags#creating-a-tag) or [using git](https://git-scm.com/book/en/v2/Git-Basics-Tagging) on the command line. 
+
+Note that the golang modules tooling expects tags to be immutable. 
+If you make a mistake after pushing a tag, make a new tag and start over rather than fix and re-push the old tag.
+## 3. Create a github release
+
+Finally, create a github release.
+To create a github release, follow the steps in the [github release documentation](https://docs.github.com/en/github/administering-a-repository/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release).
+
+When creating the github release, select the `Tag version` created in step 2. 
+Use the version tag as the release title and paste in the changelog information for this release in the `Describe this release` section.

go.mod

@@ -1,18 +1,38 @@
 module github.com/tendermint/tm-db
 
-go 1.12
+go 1.17
 
 require (
+	github.com/cosmos/gorocksdb v1.2.0
 	github.com/dgraph-io/badger/v2 v2.2007.2
-	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
-	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/gogo/protobuf v1.3.2
+	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/google/btree v1.0.0
 	github.com/jmhodges/levigo v1.0.0
+	github.com/spf13/cast v1.3.0
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
-	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
-	go.etcd.io/bbolt v1.3.5
-	google.golang.org/grpc v1.35.0
+	go.etcd.io/bbolt v1.3.6
+	google.golang.org/grpc v1.44.0
 )
+
+require (
+	github.com/DataDog/zstd v1.4.1 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect
+	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
+	golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect
+	golang.org/x/text v0.3.3 // indirect
+	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)
+
+// Breaking changes were released with the wrong tag (use v0.6.6 or later).
+retract v0.6.5