From a9d94e8a9d3d4458a27f38173a7fb9cafe57dbde Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 27 Oct 2023 16:08:26 +0200 Subject: [PATCH] save per-block conversion pointers, without ended and started --- core/blockchain.go | 6 +-- core/chain_makers.go | 2 +- core/overlay_transition.go | 42 +++++++++--------- core/state/database.go | 90 ++++++++++++++++++++++---------------- core/state_processor.go | 3 +- eth/catalyst/api.go | 2 +- light/trie.go | 22 +++++----- miner/worker.go | 2 +- 8 files changed, 92 insertions(+), 77 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 01da77d8e453..2a1738d664e4 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1746,7 +1746,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) } if parent.Number.Uint64() == conversionBlock { - bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), &parent.Time) + bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), &parent.Time, parent.Root) bc.stateCache.SetLastMerkleRoot(parent.Root) } statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps) @@ -2532,8 +2532,8 @@ func (bc *BlockChain) GetTrieFlushInterval() time.Duration { return time.Duration(bc.flushInterval.Load()) } -func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64) { - bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, pragueTime) +func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64, root common.Hash) { + bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, pragueTime, root) } func (bc *BlockChain) ReorgThroughVerkleTransition() { bc.stateCache.ReorgThroughVerkleTransition() diff --git a/core/chain_makers.go b/core/chain_makers.go index 5d8ade8a0fb0..9b925f9e78bd 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -425,8 +425,8 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine return nil, nil } var snaps *snapshot.Tree + triedb := state.NewDatabaseWithConfig(db, &trie.Config{Verkle: true}) for i := 0; i < n; i++ { - triedb := state.NewDatabaseWithConfig(db, nil) triedb.EndVerkleTransition() statedb, err := state.New(parent.Root(), triedb, snaps) if err != nil { diff --git a/core/overlay_transition.go b/core/overlay_transition.go index 35c09d22d938..24bb7d5e6c02 100644 --- a/core/overlay_transition.go +++ b/core/overlay_transition.go @@ -35,7 +35,7 @@ import ( ) // OverlayVerkleTransition contains the overlay conversion logic -func OverlayVerkleTransition(statedb *state.StateDB) error { +func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash) error { migrdb := statedb.Database() // verkle transition: if the conversion process is in progress, move @@ -47,7 +47,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { mpt = tt.Base() vkt = tt.Overlay() hasPreimagesBin = false - preimageSeek = migrdb.GetCurrentPreimageOffset() + preimageSeek = migrdb.GetCurrentPreimageOffset(root) fpreimages *bufio.Reader ) @@ -65,7 +65,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { hasPreimagesBin = true } - accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash()) + accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash(root)) if err != nil { return err } @@ -73,7 +73,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { accIt.Next() // If we're about to start with the migration process, we have to read the first account hash preimage. - if migrdb.GetCurrentAccountAddress() == nil { + if migrdb.GetCurrentAccountAddress(root) == nil { var addr common.Address if hasPreimagesBin { if _, err := io.ReadFull(fpreimages, addr[:]); err != nil { @@ -85,8 +85,8 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { return fmt.Errorf("addr len is zero is not 32: %d", len(addr)) } } - migrdb.SetCurrentAccountAddress(addr) - if migrdb.GetCurrentAccountHash() != accIt.Hash() { + migrdb.SetCurrentAccountAddress(addr, root) + if migrdb.GetCurrentAccountHash(root) != accIt.Hash() { return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash()) } preimageSeek += int64(len(addr)) @@ -108,7 +108,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { log.Error("Invalid account encountered during traversal", "error", err) return err } - vkt.SetStorageRootConversion(*migrdb.GetCurrentAccountAddress(), acc.Root) + vkt.SetStorageRootConversion(*migrdb.GetCurrentAccountAddress(root), acc.Root) // Start with processing the storage, because once the account is // converted, the `stateRoot` field loses its meaning. Which means @@ -120,7 +120,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { // to during normal block execution. A mitigation strategy has been // introduced with the `*StorageRootConversion` fields in VerkleDB. if acc.HasStorage() { - stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash()) + stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash(root)) if err != nil { return err } @@ -132,7 +132,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { // processing the storage for that account where we left off. // If the entire storage was processed, then the iterator was // created in vain, but it's ok as this will not happen often. - for ; !migrdb.GetStorageProcessed() && count < maxMovedCount; count++ { + for ; !migrdb.GetStorageProcessed(root) && count < maxMovedCount; count++ { var ( value []byte // slot value after RLP decoding safeValue [32]byte // 32-byte aligned value @@ -160,12 +160,12 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { } preimageSeek += int64(len(slotnr)) - mkv.addStorageSlot(migrdb.GetCurrentAccountAddress().Bytes(), slotnr, safeValue[:]) + mkv.addStorageSlot(migrdb.GetCurrentAccountAddress(root).Bytes(), slotnr, safeValue[:]) // advance the storage iterator - migrdb.SetStorageProcessed(!stIt.Next()) - if !migrdb.GetStorageProcessed() { - migrdb.SetCurrentSlotHash(stIt.Hash()) + migrdb.SetStorageProcessed(!stIt.Next(), root) + if !migrdb.GetStorageProcessed(root) { + migrdb.SetCurrentSlotHash(stIt.Hash(), root) } } stIt.Release() @@ -178,20 +178,20 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { if count < maxMovedCount { count++ // count increase for the account itself - mkv.addAccount(migrdb.GetCurrentAccountAddress().Bytes(), acc) - vkt.ClearStrorageRootConversion(*migrdb.GetCurrentAccountAddress()) + mkv.addAccount(migrdb.GetCurrentAccountAddress(root).Bytes(), acc) + vkt.ClearStrorageRootConversion(*migrdb.GetCurrentAccountAddress(root)) // Store the account code if present if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) { code := rawdb.ReadCode(statedb.Database().DiskDB(), common.BytesToHash(acc.CodeHash)) chunks := trie.ChunkifyCode(code) - mkv.addAccountCode(migrdb.GetCurrentAccountAddress().Bytes(), uint64(len(code)), chunks) + mkv.addAccountCode(migrdb.GetCurrentAccountAddress(root).Bytes(), uint64(len(code)), chunks) } // reset storage iterator marker for next account - migrdb.SetStorageProcessed(false) - migrdb.SetCurrentSlotHash(common.Hash{}) + migrdb.SetStorageProcessed(false, root) + migrdb.SetCurrentSlotHash(common.Hash{}, root) // Move to the next account, if available - or end // the transition otherwise. @@ -212,7 +212,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash()) } preimageSeek += int64(len(addr)) - migrdb.SetCurrentAccountAddress(addr) + migrdb.SetCurrentAccountAddress(addr, root) } else { // case when the account iterator has // reached the end but count < maxCount @@ -221,9 +221,9 @@ func OverlayVerkleTransition(statedb *state.StateDB) error { } } } - migrdb.SetCurrentPreimageOffset(preimageSeek) + migrdb.SetCurrentPreimageOffset(preimageSeek, root) - log.Info("Collected key values from base tree", "count", count, "duration", time.Since(now), "last account", statedb.Database().GetCurrentAccountHash()) + log.Info("Collected key values from base tree", "count", count, "duration", time.Since(now), "last account", statedb.Database().GetCurrentAccountHash(root)) // Take all the collected key-values and prepare the new leaf values. // This fires a background routine that will start doing the work that diff --git a/core/state/database.go b/core/state/database.go index af8cc1a36e30..8598b178b421 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -64,7 +64,7 @@ type Database interface { // TrieDB retrieves the low level trie database used for data storage. TrieDB() *trie.Database - StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunTime *uint64) + StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunTime *uint64, root common.Hash) ReorgThroughVerkleTransition() @@ -74,27 +74,27 @@ type Database interface { Transitioned() bool - SetCurrentSlotHash(hash common.Hash) + SetCurrentSlotHash(common.Hash, common.Hash) - GetCurrentAccountAddress() *common.Address + GetCurrentAccountAddress(common.Hash) *common.Address - SetCurrentAccountAddress(common.Address) + SetCurrentAccountAddress(common.Address, common.Hash) - GetCurrentAccountHash() common.Hash + GetCurrentAccountHash(common.Hash) common.Hash - GetCurrentSlotHash() common.Hash + GetCurrentSlotHash(common.Hash) common.Hash - SetStorageProcessed(bool) + SetStorageProcessed(bool, common.Hash) - GetStorageProcessed() bool + GetStorageProcessed(common.Hash) bool - GetCurrentPreimageOffset() int64 + GetCurrentPreimageOffset(common.Hash) int64 - SetCurrentPreimageOffset(int64) + SetCurrentPreimageOffset(int64, common.Hash) AddRootTranslation(originalRoot, translatedRoot common.Hash) - SetLastMerkleRoot(root common.Hash) + SetLastMerkleRoot(common.Hash) } // Trie is a Ethereum Merkle Patricia trie. @@ -187,6 +187,10 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), triedb: trie.NewDatabaseWithConfig(db, config), addrToPoint: utils.NewPointCache(), + StorageProcessed: map[common.Hash]bool{}, + CurrentAccountAddress: map[common.Hash]*common.Address{}, + CurrentSlotHash: map[common.Hash]common.Hash{}, + CurrentPreimageOffset: map[common.Hash]int64{}, } } @@ -199,6 +203,10 @@ func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database { triedb: triedb, addrToPoint: utils.NewPointCache(), ended: triedb.IsVerkle(), + StorageProcessed: map[common.Hash]bool{}, + CurrentAccountAddress: map[common.Hash]*common.Address{}, + CurrentSlotHash: map[common.Hash]common.Hash{}, + CurrentPreimageOffset: map[common.Hash]int64{}, } } @@ -211,7 +219,7 @@ func (db *cachingDB) Transitioned() bool { } // Fork implements the fork -func (db *cachingDB) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64) { +func (db *cachingDB) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64, root common.Hash) { fmt.Println(` __________.__ .__ .__ __ .__ .__ ____ \__ ___| |__ ____ ____ | | ____ ______ | |__ _____ _____/ |_ | |__ _____ ______ __ _ _|__| ____ / ___\ ______ @@ -224,7 +232,13 @@ func (db *cachingDB) StartVerkleTransition(originalRoot, translatedRoot common.H // db.AddTranslation(originalRoot, translatedRoot) db.baseRoot = originalRoot // initialize so that the first storage-less accounts are processed - db.StorageProcessed = true + db.StorageProcessed[root] = true + + // Reinitialize values in case of a reorg + db.CurrentAccountAddress[root] = &(common.Address{}) + db.CurrentSlotHash[root] = common.Hash{} + db.CurrentPreimageOffset[root] = 0 + if pragueTime != nil { chainConfig.PragueTime = pragueTime } @@ -263,14 +277,14 @@ type cachingDB struct { addrToPoint *utils.PointCache baseRoot common.Hash // hash of the read-only base tree - CurrentAccountAddress *common.Address // addresss of the last translated account - CurrentSlotHash common.Hash // hash of the last translated storage slot - CurrentPreimageOffset int64 // next byte to read from the preimage file + CurrentAccountAddress map[common.Hash]*common.Address // addresss of the last translated account + CurrentSlotHash map[common.Hash]common.Hash // hash of the last translated storage slot + CurrentPreimageOffset map[common.Hash]int64 // next byte to read from the preimage file // Mark whether the storage for an account has been processed. This is useful if the // maximum number of leaves of the conversion is reached before the whole storage is // processed. - StorageProcessed bool + StorageProcessed map[common.Hash]bool } func (db *cachingDB) openMPTTrie(root common.Hash) (Trie, error) { @@ -450,49 +464,49 @@ func (db *cachingDB) GetTreeKeyHeader(addr []byte) *verkle.Point { return db.addrToPoint.GetTreeKeyHeader(addr) } -func (db *cachingDB) SetCurrentAccountAddress(addr common.Address) { - db.CurrentAccountAddress = &addr +func (db *cachingDB) SetCurrentAccountAddress(addr common.Address, root common.Hash) { + db.CurrentAccountAddress[root] = &addr } -func (db *cachingDB) GetCurrentAccountHash() common.Hash { +func (db *cachingDB) GetCurrentAccountHash(root common.Hash) common.Hash { var addrHash common.Hash - if db.CurrentAccountAddress != nil { - addrHash = crypto.Keccak256Hash(db.CurrentAccountAddress[:]) + if db.CurrentAccountAddress[root] != nil { + addrHash = crypto.Keccak256Hash(db.CurrentAccountAddress[root][:]) } return addrHash } -func (db *cachingDB) GetCurrentAccountAddress() *common.Address { - return db.CurrentAccountAddress +func (db *cachingDB) GetCurrentAccountAddress(root common.Hash) *common.Address { + return db.CurrentAccountAddress[root] } -func (db *cachingDB) GetCurrentPreimageOffset() int64 { - return db.CurrentPreimageOffset +func (db *cachingDB) GetCurrentPreimageOffset(root common.Hash) int64 { + return db.CurrentPreimageOffset[root] } -func (db *cachingDB) SetCurrentPreimageOffset(offset int64) { - db.CurrentPreimageOffset = offset +func (db *cachingDB) SetCurrentPreimageOffset(offset int64, root common.Hash) { + db.CurrentPreimageOffset[root] = offset } -func (db *cachingDB) SetCurrentSlotHash(hash common.Hash) { - db.CurrentSlotHash = hash +func (db *cachingDB) SetCurrentSlotHash(hash common.Hash, root common.Hash) { + db.CurrentSlotHash[root] = hash } -func (db *cachingDB) GetCurrentSlotHash() common.Hash { - return db.CurrentSlotHash +func (db *cachingDB) GetCurrentSlotHash(root common.Hash) common.Hash { + return db.CurrentSlotHash[root] } -func (db *cachingDB) SetStorageProcessed(processed bool) { - db.StorageProcessed = processed +func (db *cachingDB) SetStorageProcessed(processed bool, root common.Hash) { + db.StorageProcessed[root] = processed } -func (db *cachingDB) GetStorageProcessed() bool { - return db.StorageProcessed +func (db *cachingDB) GetStorageProcessed(root common.Hash) bool { + return db.StorageProcessed[root] } func (db *cachingDB) AddRootTranslation(originalRoot, translatedRoot common.Hash) { } -func (db *cachingDB) SetLastMerkleRoot(root common.Hash) { - db.LastMerkleRoot = root +func (db *cachingDB) SetLastMerkleRoot(merkleRoot common.Hash) { + db.LastMerkleRoot = merkleRoot } diff --git a/core/state_processor.go b/core/state_processor.go index 5d10bceb1817..605480fcb975 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -106,7 +106,8 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } // Perform the overlay transition, if relevant - if err := OverlayVerkleTransition(statedb); err != nil { + parent := p.bc.GetHeaderByHash(header.ParentHash) + if err := OverlayVerkleTransition(statedb, parent.Root); err != nil { return nil, nil, 0, fmt.Errorf("error performing verkle overlay transition: %w", err) } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 63079415fc14..864b3efe84f5 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -532,7 +532,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe if api.eth.BlockChain().Config().IsPrague(block.Number(), block.Time()) && !api.eth.BlockChain().Config().IsPrague(parent.Number(), parent.Time()) { parent := api.eth.BlockChain().GetHeaderByNumber(block.NumberU64() - 1) if !api.eth.BlockChain().Config().IsPrague(parent.Number, parent.Time) { - api.eth.BlockChain().StartVerkleTransition(parent.Root, common.Hash{}, api.eth.BlockChain().Config(), nil) + api.eth.BlockChain().StartVerkleTransition(parent.Root, common.Hash{}, api.eth.BlockChain().Config(), nil, parent.Root) } } // Reset db merge state in case of a reorg diff --git a/light/trie.go b/light/trie.go index 53d54615d909..409963094edb 100644 --- a/light/trie.go +++ b/light/trie.go @@ -121,47 +121,47 @@ func (db *odrDatabase) Transitioned() bool { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) SetCurrentSlotHash(hash common.Hash) { +func (db *odrDatabase) SetCurrentSlotHash(common.Hash, common.Hash) { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) GetCurrentAccountAddress() *common.Address { +func (db *odrDatabase) GetCurrentAccountAddress(common.Hash) *common.Address { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) SetCurrentAccountAddress(_ common.Address) { +func (db *odrDatabase) SetCurrentAccountAddress(common.Address, common.Hash) { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) GetCurrentAccountHash() common.Hash { +func (db *odrDatabase) GetCurrentAccountHash(common.Hash) common.Hash { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) GetCurrentSlotHash() common.Hash { +func (db *odrDatabase) GetCurrentSlotHash(common.Hash) common.Hash { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) SetStorageProcessed(_ bool) { +func (db *odrDatabase) SetStorageProcessed(bool, common.Hash) { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) GetStorageProcessed() bool { +func (db *odrDatabase) GetStorageProcessed(common.Hash) bool { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) GetCurrentPreimageOffset() int64 { +func (db *odrDatabase) GetCurrentPreimageOffset(common.Hash) int64 { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) SetCurrentPreimageOffset(_ int64) { +func (db *odrDatabase) SetCurrentPreimageOffset(int64, common.Hash) { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) AddRootTranslation(originalRoot common.Hash, translatedRoot common.Hash) { +func (db *odrDatabase) AddRootTranslation(common.Hash, common.Hash) { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) SetLastMerkleRoot(root common.Hash) { +func (db *odrDatabase) SetLastMerkleRoot(common.Hash) { panic("not implemented") // TODO: Implement } diff --git a/miner/worker.go b/miner/worker.go index 124c93212262..38c76b431328 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -905,7 +905,7 @@ func (w *worker) prepareWork(genParams *generateParams) (*environment, error) { return nil, err } if w.chain.Config().IsPrague(header.Number, header.Time) { - core.OverlayVerkleTransition(state) + core.OverlayVerkleTransition(state, parent.Root) } // Run the consensus preparation with the default or customized consensus engine. if err := w.engine.Prepare(w.chain, header); err != nil {