diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c27196377..5aeacea458 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Changelog for NeoFS Node ### Added ### Fixed +- Incomplete metabase migration to version 3 leading to node start failure (#3048) ### Changed diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 2311411c5a..f08c080cd7 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -52,27 +52,7 @@ func (db *DB) openBolt() error { db.log.Debug("opened boltDB instance for Metabase") - db.log.Debug("checking metabase version") - return db.boltDB.View(func(tx *bbolt.Tx) error { - // The safest way to check if the metabase is fresh is to check if it has no buckets. - // However, shard info can be present. So here we check that the number of buckets is - // at most 1. - // Another thing to consider is that tests do not persist shard ID, we want to support - // this case too. - var n int - err := tx.ForEach(func([]byte, *bbolt.Bucket) error { - if n++; n >= 2 { // do not iterate a lot - return errBreakBucketForEach - } - return nil - }) - - if errors.Is(err, errBreakBucketForEach) { - db.initialized = true - err = nil - } - return err - }) + return nil } // Init initializes metabase. It creates static (CID-independent) buckets in underlying BoltDB instance. @@ -153,7 +133,7 @@ func (db *DB) init(reset bool) error { if err != nil { return err } - err = updateVersion(tx, version) + err = updateVersion(tx, currentMetaVersion) if err != nil { return err } diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 3f582fd83c..7684ee2a2c 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -40,8 +40,6 @@ type DB struct { matchers map[object.SearchMatchType]matcher boltDB *bbolt.DB - - initialized bool } // Option is an option of DB constructor. diff --git a/pkg/local_object_storage/metabase/status.go b/pkg/local_object_storage/metabase/status.go index 1caa5c6ec9..7f2855adc4 100644 --- a/pkg/local_object_storage/metabase/status.go +++ b/pkg/local_object_storage/metabase/status.go @@ -47,7 +47,7 @@ func (db *DB) ObjectStatus(address oid.Address) (ObjectStatus, error) { } err = db.boltDB.View(func(tx *bbolt.Tx) error { - res.Version = getVersion(tx) + res.Version, _ = getVersion(tx) oID := address.Object() cID := address.Container() diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index 881ad37a3f..297c0e7f69 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -10,8 +10,8 @@ import ( "go.etcd.io/bbolt" ) -// version contains current metabase version. -const version = 3 +// currentMetaVersion contains current metabase version. +const currentMetaVersion = 3 var versionKey = []byte("version") @@ -21,39 +21,29 @@ var versionKey = []byte("version") var ErrOutdatedVersion = logicerr.New("invalid version, resynchronization is required") func (db *DB) checkVersion(tx *bbolt.Tx) error { - var knownVersion bool + stored, knownVersion := getVersion(tx) - b := tx.Bucket(shardInfoBucket) - if b != nil { - data := b.Get(versionKey) - if len(data) == 8 { - knownVersion = true - - stored := binary.LittleEndian.Uint64(data) - if stored != version { - migrate, ok := migrateFrom[stored] - if !ok { - return fmt.Errorf("%w: expected=%d, stored=%d", ErrOutdatedVersion, version, stored) - } - - err := migrate(db, tx) - if err != nil { - return fmt.Errorf("migrating from %d to %d version failed, consider database resync: %w", stored, version, err) - } - } - } + switch { + case !knownVersion: + // new database, write version + return updateVersion(tx, currentMetaVersion) + case stored == currentMetaVersion: + return nil + case stored > currentMetaVersion: + return fmt.Errorf("%w: expected=%d, stored=%d", ErrOutdatedVersion, currentMetaVersion, stored) } - if !db.initialized { - // new database, write version - return updateVersion(tx, version) - } else if !knownVersion { - // db is initialized but no version - // has been found; that could happen - // if the db is corrupted or the version - // is <2 (is outdated and requires resync - // anyway) - return ErrOutdatedVersion + // Outdated, but can be migrated. + for i := stored; i < currentMetaVersion; i++ { + migrate, ok := migrateFrom[i] + if !ok { + return fmt.Errorf("%w: expected=%d, stored=%d", ErrOutdatedVersion, currentMetaVersion, stored) + } + + err := migrate(db, tx) + if err != nil { + return fmt.Errorf("migrating from meta version %d failed, consider database resync: %w", i, err) + } } return nil @@ -70,16 +60,16 @@ func updateVersion(tx *bbolt.Tx, version uint64) error { return b.Put(versionKey, data) } -func getVersion(tx *bbolt.Tx) uint64 { +func getVersion(tx *bbolt.Tx) (uint64, bool) { b := tx.Bucket(shardInfoBucket) if b != nil { data := b.Get(versionKey) if len(data) == 8 { - return binary.LittleEndian.Uint64(data) + return binary.LittleEndian.Uint64(data), true } } - return 0 + return 0, false } var migrateFrom = map[uint64]func(*DB, *bbolt.Tx) error{ @@ -95,7 +85,11 @@ func migrateFrom2Version(db *DB, tx *bbolt.Tx) error { c := bkt.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { - if l := len(v); l != addressKeySize { + l := len(v) + if l == addressKeySize+8 { // Because of a 0.44.0 bug we can have a migrated DB with version 2. + continue + } + if l != addressKeySize { return fmt.Errorf("graveyard value with unexpected %d length", l) } @@ -109,5 +103,5 @@ func migrateFrom2Version(db *DB, tx *bbolt.Tx) error { } } - return nil + return updateVersion(tx, 3) } diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index c21911a3dc..cc6afc415c 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -11,6 +11,7 @@ import ( "testing" objectconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -42,8 +43,8 @@ func TestVersion(t *testing.T) { if len(data) != 8 { return errors.New("invalid version data") } - if stored := binary.LittleEndian.Uint64(data); stored != version { - return fmt.Errorf("invalid version: %d != %d", stored, version) + if stored := binary.LittleEndian.Uint64(data); stored != currentMetaVersion { + return fmt.Errorf("invalid version: %d != %d", stored, currentMetaVersion) } return nil })) @@ -77,7 +78,7 @@ func TestVersion(t *testing.T) { db := newDB(t) require.NoError(t, db.Open(false)) require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { - return updateVersion(tx, version+1) + return updateVersion(tx, currentMetaVersion+1) })) require.NoError(t, db.Close()) @@ -307,16 +308,20 @@ func TestMigrate2to3(t *testing.T) { }) require.NoError(t, err) + // inhumeV2 stores data in the old format, but new DB has current version by default, force old version. err = db.boltDB.Update(func(tx *bbolt.Tx) error { - err = updateVersion(tx, 2) - if err != nil { - return err - } - - return migrateFrom2Version(db, tx) + return updateVersion(tx, 2) }) require.NoError(t, err) + db.mode = mode.DegradedReadOnly // Force reload. + ok, err := db.Reload(WithPath(db.info.Path), WithEpochState(epochState{})) + require.NoError(t, err) + require.True(t, ok) + + err = db.Init() // Migration happens here. + require.NoError(t, err) + err = db.boltDB.View(func(tx *bbolt.Tx) error { return tx.Bucket(graveyardBucketName).ForEach(func(k, v []byte) error { require.Len(t, v, addressKeySize+8) @@ -327,4 +332,15 @@ func TestMigrate2to3(t *testing.T) { }) }) require.NoError(t, err) + err = db.boltDB.View(func(tx *bbolt.Tx) error { + gotV, ok := getVersion(tx) + if !ok { + return errors.New("missing version") + } + if gotV != currentMetaVersion { + return errors.New("version was not updated") + } + return nil + }) + require.NoError(t, err) }