Omnibus update branch

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
     # rocksdb, which are needed to build and test those modules.
     # To update the container image, see docker.yml.
     runs-on: ubuntu-latest
-    container: tendermintdev/docker-tm-db-testing
+    container: ghcr.io/faddat/cosmos
     steps:
       - uses: actions/checkout@v3
       - name: test & coverage report creation

.github/workflows/gaia

@@ -0,0 +1,53 @@
+# This workflow state syncs a live Gaia automatically with each commit.  It is intended to be used for:
+# * Monitoring functionality
+# * Measuring "time to live gaia" across different databases
+
+name: State Sync Gaia
+on:
+  pull_request:
+  push:
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: ghcr.io/faddat/gaia
+    continue-on-error: true
+    timeout-minutes: 10
+    env:
+      INTERVAL: 1000
+      GAIAD_STATESYNC_ENABLE: true
+      GAIAD_P2P_MAX_NUM_OUTBOUND_PEERS: 200
+      GAIAD_P2P_SEEDS: "bf8328b66dceb4987e5cd94430af66045e59899f@public-seed.cosmos.vtwit.com:26656,[email protected]:26656,[email protected]:26656,ba3bacc714817218562f743178228f23678b2873@public-seed-node.cosmoshub.certus.one:26656,[email protected]:26656,[email protected]:26656"
+      GAIAD_STATESYNC_RPC_SERVERS: "https://cosmoshub.validator.network:443,https://cosmoshub.validator.network:443"
+      GOPATH: /go
+
+
+
+
+    strategy: 
+      matrix:
+        database: [rocksdb, boltdb, badgerdb, goleveldb, cleveldb]
+
+    steps:
+      - run: git config --global --add safe.directory /__w/tendermint/tm-db
+      - uses: actions/checkout@v3
+      - name: state sync gaia with ${{ matrix.database }}
+        run: |
+          cd ..
+          git clone https://github.com/cosmos/gaia --branch release/v7.0.x
+          cd gaia
+          go mod edit -replace github.com/tendermint/tm-db=../tm-db
+          go mod tidy -go=1.18
+          go install -ldflags '-w -s -X github.com/cosmos/cosmos-sdk/types.DBBackend=${{ matrix.database }}' -tags ${{ matrix.database }} ./...
+          export LATEST_HEIGHT=$(curl -s https://cosmoshub.validator.network/block | jq -r .result.block.header.height);
+          export BLOCK_HEIGHT=$(($LATEST_HEIGHT-$INTERVAL)) 
+          export TRUST_HASH=$(curl -s "https://cosmoshub.validator.network/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash)
+          export GAIAD_STATESYNC_TRUST_HEIGHT=$BLOCK_HEIGHT
+          export GAIAD_STATESYNC_TRUST_HASH=$TRUST_HASH
+          export PATH=$PATH:/go/bin
+          echo "TRUST HEIGHT: $BLOCK_HEIGHT"
+          echo "TRUST HASH: $TRUST_HASH"
+          gaiad init gaia-matrix
+          cp /genesis.json ~/.gaia/config/genesis.json
+          gaiad start --x-crisis-skip-assert-invariants --db_backend ${{ matrix.database }}

.github/workflows/lint.yml

@@ -1,19 +1,42 @@
-name: Lint
+name: golangci-lint
 on:
   push:
-    branches:
-      - master
   pull_request:
-
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
 jobs:
   golangci:
-    # We need to run the linter on the same image we use for building, since it
-    # needs the C libraries installed for the dependencies to typecheck.
+    name: lint
     runs-on: ubuntu-latest
-    container: tendermintdev/docker-tm-db-testing
+    container: ghcr.io/faddat/cosmos
     steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
       - uses: actions/checkout@v3
+
       - uses: golangci/[email protected]
         with:
-          args: --timeout 10m
-          github-token: ${{ secrets.github_token }}
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          version: latest
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          # args: --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true then the all caching functionality will be complete disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true then the action don't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true

.gitpod.yml

@@ -0,0 +1,7 @@
+# This configuration file was automatically generated by Gitpod.
+# Please adjust to your needs (see https://www.gitpod.io/docs/config-gitpod-file)
+# and commit this file to your remote git repository to share the goodness with others.
+
+image: ghcr.io/notional-labs/cosmos
+
+

.golangci.yml

@@ -7,6 +7,7 @@ linters:
     - dogsled
     - dupl
     - errcheck
+    - gofumpt
     # - funlen
     # - gochecknoglobals
     # - gochecknoinits
@@ -40,6 +41,13 @@ linters:
     # - gocognit
     - nolintlint
 
+run:
+  build-tags:
+    - cleveldb
+    - rocksdb
+    - badgerdb
+    - boltdb
+
 issues:
   exclude-rules:
     - path: _test\.go

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "makefile.extensionOutputFolder": "./.vscode",
+    "go.buildTags": "rocksdb"
+}

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+
+- remove mutex from prefixdb
+
 ## 0.6.7
 
 **2022-2-21**

README.md

@@ -1,4 +1,4 @@
-# Tendermint DB 
+# Tendermint DB, Custom Edition
 
 [![version](https://img.shields.io/github/tag/tendermint/tm-db.svg)](https://github.com/tendermint/tm-db/releases/latest)
 [![license](https://img.shields.io/github/license/tendermint/tm-db.svg)](https://github.com/tendermint/tm-db/blob/master/LICENSE)
@@ -10,23 +10,41 @@
 
 Common database interface for various database backends. Primarily meant for applications built on [Tendermint](https://github.com/tendermint/tendermint), such as the [Cosmos SDK](https://github.com/cosmos/cosmos-sdk), but can be used independently of these as well.
 
+### Differences
+
+* This tm-db branch includes its own fully-working, known-good development environment so that you can easily test the one viable cgo database, rocksdb.
+* This tm-db branch removes bolt because it basically does not work, and badger because it is abandonware. 
+* This tm-db branch aims to support only viable options:
+
+  * goleveldb
+  * rocksdb
+  * pebbledb
+
+### Learnings
+
+Rocksdb uses the most RAM of any of these options.  We know this because it is killed in this run:
+
+https://github.com/notional-labs/tm-db/runs/6437891115 on a 4cpu runner.  We'll increase to 8 now. 
+
+buildjet-4vcpu-ubuntu-2004
+
+Yeilds 2m 30s to consensus reactor.
+
+
+
 ### Minimum Go Version
 
-Go 1.13+
+Go 1.18+
+
+
 
 ## Supported Database Backends
 
 - **[GoLevelDB](https://github.com/syndtr/goleveldb) [stable]**: A pure Go implementation of [LevelDB](https://github.com/google/leveldb) (see below). Currently the default on-disk database used in the Cosmos SDK.
 
 - **MemDB [stable]:** An in-memory database using [Google's B-tree package](https://github.com/google/btree). Has very high performance both for reads, writes, and range scans, but is not durable and will lose all data on process exit. Does not support transactions. Suitable for e.g. caches, working sets, and tests. Used for [IAVL](https://github.com/tendermint/iavl) working sets when the pruning strategy allows it.
 
-- **[LevelDB](https://github.com/google/leveldb) [experimental]:** A [Go wrapper](https://github.com/jmhodges/levigo) around [LevelDB](https://github.com/google/leveldb). Uses LSM-trees for on-disk storage, which have good performance for write-heavy workloads, particularly on spinning disks, but requires periodic compaction to maintain decent read performance and reclaim disk space. Does not support transactions.
-
-- **[BoltDB](https://github.com/etcd-io/bbolt) [experimental]:** A [fork](https://github.com/etcd-io/bbolt) of [BoltDB](https://github.com/boltdb/bolt). Uses B+trees for on-disk storage, which have good performance for read-heavy workloads and range scans. Supports serializable ACID transactions.
-
-- **[RocksDB](https://github.com/cosmos/gorocksdb) [experimental]:** A [Go wrapper](https://github.com/cosmos/gorocksdb) around [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees for on-disk storage, but is optimized for fast storage media such as SSDs and memory. Supports atomic transactions, but not full ACID transactions.
-
-- **[BadgerDB](https://github.com/dgraph-io/badger) [experimental]:** A key-value database written as a pure-Go alternative to e.g. LevelDB and RocksDB, with LSM-tree storage. Makes use of multiple goroutines for performance, and includes advanced features such as serializable ACID transactions, write batches, compression, and more.
+- **[RocksDB](https://github.com/linxGnu/grocksdb) [experimental]:** A [Go wrapper](https://github.com/linxGnu/grocksdb) around [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees for on-disk storage, but is optimized for fast storage media such as SSDs and memory. Supports atomic transactions, but not full ACID transactions.
 
 ## Meta-databases
 

backend_test.go

@@ -315,7 +315,6 @@ func testDBIterator(t *testing.T, backend BackendType) {
 	ritr, err = db2.ReverseIterator(nil, nil)
 	require.NoError(t, err)
 	verifyIterator(t, ritr, nil, "reverse iterator with empty db")
-
 }
 
 func verifyIterator(t *testing.T, itr Iterator, expected []int64, msg string) {