Merge branch 'master' into 2024-06-more-iaviewer-commands

.github/workflows/ci.yml

@@ -25,6 +25,7 @@ jobs:
         # integer overflow).
       - name: test & coverage report creation
         run: |
+          cd cmd/legacydump && go build -o legacydump main.go && cd ../..
           go test ./... -mod=readonly -timeout 10m -short -race -coverprofile=coverage.txt -covermode=atomic
           go test ./... -mod=readonly -timeout 15m
           GOARCH=386 go test ./... -mod=readonly -timeout 15m

.github/workflows/pr-reminder.yml

@@ -36,7 +36,7 @@ jobs:
             return table;
       - name: Send Slack Reminder
         if: steps.pr-list.outputs.result != ''
-        uses: rtCamp/[email protected].0
+        uses: rtCamp/[email protected].2
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
           SLACK_CHANNEL: pr-github

.gitignore

@@ -1,3 +1,4 @@
+cmd/legacydump/legacydump
 vendor
 .glide
 *.swp

Makefile

@@ -24,9 +24,12 @@ test-short:
 	@go test ./... $(LDFLAGS) -v --race --short
 .PHONY: test-short
 
-test:
+cmd/legacydump/legacydump: 
+	cd cmd/legacydump && go build -o legacydump main.go
+
+test: cmd/legacydump/legacydump
 	@echo "--> Running go test"
-	@go test ./... $(LDFLAGS) -v 
+	@go test ./... $(LDFLAGS) 
 .PHONY: test
 
 format:

docs/node/nodedb.md

@@ -19,7 +19,7 @@ When a version `v` is deleted, all nodes which removed in the current version wi
 ```golang
 // DeleteVersionsFrom permanently deletes all tree versions from the given version upwards.
 func (ndb *nodeDB) DeleteVersionsFrom(fromVersion int64) error {
-	latest, err := ndb.getLatestVersion()
+	_, latest, err := ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}

go.mod

@@ -5,14 +5,14 @@ go 1.23
 toolchain go1.23.1
 
 require (
-	cosmossdk.io/core v1.0.0-alpha.4
+	cosmossdk.io/core v1.0.0-alpha.6
 	github.com/cosmos/ics23/go v0.11.0
 	github.com/emicklei/dot v1.6.2
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/btree v1.1.3
 	github.com/stretchr/testify v1.9.0
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
-	go.uber.org/mock v0.4.0
+	go.uber.org/mock v0.5.0
 )
 
 require (

go.sum

@@ -1,5 +1,5 @@
-cosmossdk.io/core v1.0.0-alpha.4 h1:9iuroT9ejDYETCsGkzkvs/wAY/5UFl7nCIINFRxyMJY=
-cosmossdk.io/core v1.0.0-alpha.4/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY=
+cosmossdk.io/core v1.0.0-alpha.6 h1:5ukC4JcQKmemLQXcAgu/QoOvJI50hpBkIIg4ZT2EN8E=
+cosmossdk.io/core v1.0.0-alpha.6/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY=
 github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fro=
 github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0=
 github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU=
@@ -64,8 +64,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
-go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
+go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

immutable_tree.go

@@ -301,7 +301,7 @@ func (t *ImmutableTree) IsFastCacheEnabled() (bool, error) {
 }
 
 func (t *ImmutableTree) isLatestTreeVersion() (bool, error) {
-	latestVersion, err := t.ndb.getLatestVersion()
+	_, latestVersion, err := t.ndb.getLatestVersion()
 	if err != nil {
 		return false, err
 	}

iterator_test.go

@@ -182,7 +182,7 @@ func TestIterator_WithDelete_Full_Ascending_Success(t *testing.T) {
 	err = tree.DeleteVersionsTo(1)
 	require.NoError(t, err)
 
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	immutableTree, err := tree.GetImmutable(latestVersion)
 	require.NoError(t, err)
@@ -253,7 +253,7 @@ func setupIteratorAndMirror(t *testing.T, config *iteratorTestConfig) (corestore
 	_, _, err := tree.SaveVersion()
 	require.NoError(t, err)
 
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	immutableTree, err := tree.GetImmutable(latestVersion)
 	require.NoError(t, err)

migrate_test.go

@@ -177,7 +177,7 @@ func TestDeleteVersions(t *testing.T) {
 	// Test LoadVersionForOverwriting for the legacy version
 	err = tree.LoadVersionForOverwriting(int64(targetVersion))
 	require.NoError(t, err)
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, int64(targetVersion), latestVersion)
 	legacyLatestVersion, err := tree.ndb.getLegacyLatestVersion()

mutable_tree.go

@@ -40,6 +40,7 @@ type MutableTree struct {
 	unsavedFastNodeRemovals  *sync.Map      // map[string]interface{} FastNodes that have not yet been removed from disk
 	ndb                      *nodeDB
 	skipFastStorageUpgrade   bool // If true, the tree will work like no fast storage and always not upgrade fast storage
+	initialVersionSet        bool
 
 	mtx sync.Mutex
 }
@@ -62,6 +63,7 @@ func NewMutableTree(db corestore.KVStoreWithBatch, cacheSize int, skipFastStorag
 		unsavedFastNodeRemovals:  &sync.Map{},
 		ndb:                      ndb,
 		skipFastStorageUpgrade:   skipFastStorageUpgrade,
+		initialVersionSet:        opts.initialVersionSet,
 	}
 }
 
@@ -73,7 +75,8 @@ func (tree *MutableTree) IsEmpty() bool {
 
 // GetLatestVersion returns the latest version of the tree.
 func (tree *MutableTree) GetLatestVersion() (int64, error) {
-	return tree.ndb.getLatestVersion()
+	_, v, err := tree.ndb.getLatestVersion()
+	return v, err
 }
 
 // VersionExists returns whether or not a version exists.
@@ -90,10 +93,13 @@ func (tree *MutableTree) VersionExists(version int64) bool {
 	if err != nil {
 		return false
 	}
-	latestVersion, err := tree.ndb.getLatestVersion()
+	found, latestVersion, err := tree.ndb.getLatestVersion()
 	if err != nil {
 		return false
 	}
+	if !found {
+		return false
+	}
 
 	return firstVersion <= version && version <= latestVersion
 }
@@ -104,7 +110,7 @@ func (tree *MutableTree) AvailableVersions() []int {
 	if err != nil {
 		return nil
 	}
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	if err != nil {
 		return nil
 	}
@@ -146,7 +152,7 @@ func (tree *MutableTree) WorkingHash() []byte {
 
 func (tree *MutableTree) WorkingVersion() int64 {
 	version := tree.version + 1
-	if version == 1 && tree.ndb.opts.InitialVersion > 0 {
+	if version == 1 && tree.initialVersionSet {
 		version = int64(tree.ndb.opts.InitialVersion)
 	}
 	return version
@@ -449,21 +455,16 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) {
 			tree.ndb.opts.InitialVersion, firstVersion)
 	}
 
-	latestVersion, err := tree.ndb.getLatestVersion()
+	ok, latestVersion, err := tree.ndb.getLatestVersion()
 	if err != nil {
 		return 0, err
 	}
 
-	if firstVersion > 0 && firstVersion < int64(tree.ndb.opts.InitialVersion) {
-		return latestVersion, fmt.Errorf("initial version set to %v, but found earlier version %v",
-			tree.ndb.opts.InitialVersion, firstVersion)
-	}
-
 	if latestVersion < targetVersion {
 		return latestVersion, fmt.Errorf("wanted to load target %d but only found up to %d", targetVersion, latestVersion)
 	}
 
-	if firstVersion == 0 {
+	if !ok {
 		if targetVersion <= 0 {
 			if !tree.skipFastStorageUpgrade {
 				tree.mtx.Lock()
@@ -604,7 +605,7 @@ func (tree *MutableTree) enableFastStorageAndCommit() error {
 		return err
 	}
 
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}
@@ -707,6 +708,7 @@ func (tree *MutableTree) UnsetCommitting() {
 // the tree. Returns the hash and new version number.
 func (tree *MutableTree) SaveVersion() ([]byte, int64, error) {
 	version := tree.WorkingVersion()
+	tree.initialVersionSet = false
 
 	if tree.VersionExists(version) {
 		// If the version already exists, return an error as we're attempting to overwrite.
@@ -871,6 +873,7 @@ func (tree *MutableTree) saveFastNodeRemovals() error {
 // and is otherwise ignored.
 func (tree *MutableTree) SetInitialVersion(version uint64) {
 	tree.ndb.opts.InitialVersion = version
+	tree.initialVersionSet = true
 }
 
 // DeleteVersionsTo removes versions upto the given version from the MutableTree.

mutable_tree_test.go

@@ -769,6 +769,8 @@ func TestUpgradeStorageToFast_LatestVersion_Success(t *testing.T) {
 	require.False(t, isUpgradeable)
 	require.NoError(t, err)
 
+	_, _, err = tree.SaveVersion()
+	require.NoError(t, err)
 	isFastCacheEnabled, err = tree.IsFastCacheEnabled()
 	require.NoError(t, err)
 	require.True(t, isFastCacheEnabled)
@@ -907,7 +909,7 @@ func TestFastStorageReUpgradeProtection_NoForceUpgrade_Success(t *testing.T) {
 
 	// Pretend that we called Load and have the latest state in the tree
 	tree.version = latestTreeVersion
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, latestVersion, int64(latestTreeVersion))
 
@@ -1001,7 +1003,7 @@ func TestFastStorageReUpgradeProtection_ForceUpgradeFirstTime_NoForceSecondTime_
 
 	// Pretend that we called Load and have the latest state in the tree
 	tree.version = latestTreeVersion
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, latestVersion, int64(latestTreeVersion))
 
@@ -1479,3 +1481,16 @@ func TestMutableTreeClose(t *testing.T) {
 
 	require.NoError(t, tree.Close())
 }
+
+func TestMutableTree_InitialVersionZero(t *testing.T) {
+	db := dbm.NewMemDB()
+
+	tree := NewMutableTree(db, 0, false, NewNopLogger(), InitialVersionOption(0))
+
+	_, err := tree.Set([]byte("hello"), []byte("world"))
+	require.NoError(t, err)
+
+	_, version, err := tree.SaveVersion()
+	require.NoError(t, err)
+	require.Equal(t, int64(0), version)
+}

nodedb.go

@@ -331,7 +331,7 @@ func (ndb *nodeDB) shouldForceFastStorageUpgrade() (bool, error) {
 	versions := strings.Split(ndb.storageVersion, fastStorageVersionDelimiter)
 
 	if len(versions) == 2 {
-		latestVersion, err := ndb.getLatestVersion()
+		_, latestVersion, err := ndb.getLatestVersion()
 		if err != nil {
 			// TODO: should be true or false as default? (removed panic here)
 			return false, err
@@ -525,7 +525,7 @@ func (ndb *nodeDB) deleteLegacyVersions(legacyLatestVersion int64) error {
 
 // DeleteVersionsFrom permanently deletes all tree versions from the given version upwards.
 func (ndb *nodeDB) DeleteVersionsFrom(fromVersion int64) error {
-	latest, err := ndb.getLatestVersion()
+	_, latest, err := ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}
@@ -642,7 +642,7 @@ func (ndb *nodeDB) deleteVersionsTo(toVersion int64) error {
 		return err
 	}
 
-	latest, err := ndb.getLatestVersion()
+	_, latest, err := ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}
@@ -727,7 +727,7 @@ func (ndb *nodeDB) getFirstVersion() (int64, error) {
 		return version, nil
 	}
 	// Find the first version
-	latestVersion, err := ndb.getLatestVersion()
+	_, latestVersion, err := ndb.getLatestVersion()
 	if err != nil {
 		return 0, err
 	}
@@ -797,21 +797,21 @@ func (ndb *nodeDB) resetLegacyLatestVersion(version int64) {
 	ndb.legacyLatestVersion = version
 }
 
-func (ndb *nodeDB) getLatestVersion() (int64, error) {
+func (ndb *nodeDB) getLatestVersion() (bool, int64, error) {
 	ndb.mtx.Lock()
 	latestVersion := ndb.latestVersion
 	ndb.mtx.Unlock()
 
 	if latestVersion > 0 {
-		return latestVersion, nil
+		return true, latestVersion, nil
 	}
 
 	itr, err := ndb.db.ReverseIterator(
 		nodeKeyPrefixFormat.KeyInt64(int64(1)),
 		nodeKeyPrefixFormat.KeyInt64(int64(math.MaxInt64)),
 	)
 	if err != nil {
-		return 0, err
+		return false, 0, err
 	}
 	defer itr.Close()
 
@@ -821,24 +821,25 @@ func (ndb *nodeDB) getLatestVersion() (int64, error) {
 		nodeKeyFormat.Scan(k, &nk)
 		latestVersion = GetNodeKey(nk).version
 		ndb.resetLatestVersion(latestVersion)
-		return latestVersion, nil
+		return true, latestVersion, nil
 	}
 
 	if err := itr.Error(); err != nil {
-		return 0, err
+		return false, 0, err
 	}
 
 	// If there are no versions, try to get the latest version from the legacy format.
 	latestVersion, err = ndb.getLegacyLatestVersion()
 	if err != nil {
-		return 0, err
+		return false, 0, err
 	}
 	if latestVersion > 0 {
 		ndb.resetLatestVersion(latestVersion)
-		return latestVersion, nil
+		return true, latestVersion, nil
 	}
 
-	return 0, nil
+	return false, 0, nil
+	// return -1, nil
 }
 
 func (ndb *nodeDB) resetLatestVersion(version int64) {
@@ -1246,7 +1247,7 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(
 	if startVersion < firstVersion {
 		startVersion = firstVersion
 	}
-	latestVersion, err := ndb.getLatestVersion()
+	_, latestVersion, err := ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}

nodedb_test.go

@@ -87,7 +87,7 @@ func TestSetStorageVersion_Success(t *testing.T) {
 	ndb := newNodeDB(db, 0, DefaultOptions(), NewNopLogger())
 	require.Equal(t, defaultStorageVersionValue, ndb.getStorageVersion())
 
-	latestVersion, err := ndb.getLatestVersion()
+	_, latestVersion, err := ndb.getLatestVersion()
 	require.NoError(t, err)
 
 	err = ndb.SetFastStorageVersionToBatch(latestVersion)
@@ -404,7 +404,7 @@ func TestDeleteVersionsFromNoDeadlock(t *testing.T) {
 	err := ndb.SetFastStorageVersionToBatch(ndb.latestVersion)
 	require.NoError(t, err)
 
-	latestVersion, err := ndb.getLatestVersion()
+	_, latestVersion, err := ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, expectedVersion+fastStorageVersionDelimiter+strconv.Itoa(int(latestVersion)), ndb.getStorageVersion())
 	require.NoError(t, ndb.batch.Write())