Skip to content

Commit

Permalink
save per-block conversion pointers, without ended and started
Browse files Browse the repository at this point in the history
  • Loading branch information
@gballet
gballet committed Oct 27, 2023
1 parent 66ad9be commit a9d94e8
Show file tree
Hide file tree
Showing 8 changed files with 92 additions and 77 deletions.
6 changes: 3 additions & 3 deletions core/blockchain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1746,7 +1746,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
}

if parent.Number.Uint64() == conversionBlock {
bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), &parent.Time)
bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), &parent.Time, parent.Root)
bc.stateCache.SetLastMerkleRoot(parent.Root)
}
statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps)
Expand Down Expand Up @@ -2532,8 +2532,8 @@ func (bc *BlockChain) GetTrieFlushInterval() time.Duration {
return time.Duration(bc.flushInterval.Load())
}

func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64) {
bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, pragueTime)
func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64, root common.Hash) {
bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, pragueTime, root)
}
func (bc *BlockChain) ReorgThroughVerkleTransition() {
bc.stateCache.ReorgThroughVerkleTransition()
Expand Down
2 changes: 1 addition & 1 deletion core/chain_makers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,8 +425,8 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
return nil, nil
}
var snaps *snapshot.Tree
triedb := state.NewDatabaseWithConfig(db, &trie.Config{Verkle: true})
for i := 0; i < n; i++ {
triedb := state.NewDatabaseWithConfig(db, nil)
triedb.EndVerkleTransition()
statedb, err := state.New(parent.Root(), triedb, snaps)
if err != nil {
Expand Down
42 changes: 21 additions & 21 deletions core/overlay_transition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import (
)

// OverlayVerkleTransition contains the overlay conversion logic
func OverlayVerkleTransition(statedb *state.StateDB) error {
func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash) error {
migrdb := statedb.Database()

// verkle transition: if the conversion process is in progress, move
Expand All @@ -47,7 +47,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
mpt = tt.Base()
vkt = tt.Overlay()
hasPreimagesBin = false
preimageSeek = migrdb.GetCurrentPreimageOffset()
preimageSeek = migrdb.GetCurrentPreimageOffset(root)
fpreimages *bufio.Reader
)

Expand All @@ -65,15 +65,15 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
hasPreimagesBin = true
}

accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash())
accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash(root))
if err != nil {
return err
}
defer accIt.Release()
accIt.Next()

// If we're about to start with the migration process, we have to read the first account hash preimage.
if migrdb.GetCurrentAccountAddress() == nil {
if migrdb.GetCurrentAccountAddress(root) == nil {
var addr common.Address
if hasPreimagesBin {
if _, err := io.ReadFull(fpreimages, addr[:]); err != nil {
Expand All @@ -85,8 +85,8 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
return fmt.Errorf("addr len is zero is not 32: %d", len(addr))
}
}
migrdb.SetCurrentAccountAddress(addr)
if migrdb.GetCurrentAccountHash() != accIt.Hash() {
migrdb.SetCurrentAccountAddress(addr, root)
if migrdb.GetCurrentAccountHash(root) != accIt.Hash() {
return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
}
preimageSeek += int64(len(addr))
Expand All @@ -108,7 +108,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
log.Error("Invalid account encountered during traversal", "error", err)
return err
}
vkt.SetStorageRootConversion(*migrdb.GetCurrentAccountAddress(), acc.Root)
vkt.SetStorageRootConversion(*migrdb.GetCurrentAccountAddress(root), acc.Root)

// Start with processing the storage, because once the account is
// converted, the `stateRoot` field loses its meaning. Which means
Expand All @@ -120,7 +120,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
// to during normal block execution. A mitigation strategy has been
// introduced with the `*StorageRootConversion` fields in VerkleDB.
if acc.HasStorage() {
stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash())
stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash(root))
if err != nil {
return err
}
Expand All @@ -132,7 +132,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
// processing the storage for that account where we left off.
// If the entire storage was processed, then the iterator was
// created in vain, but it's ok as this will not happen often.
for ; !migrdb.GetStorageProcessed() && count < maxMovedCount; count++ {
for ; !migrdb.GetStorageProcessed(root) && count < maxMovedCount; count++ {
var (
value []byte // slot value after RLP decoding
safeValue [32]byte // 32-byte aligned value
Expand Down Expand Up @@ -160,12 +160,12 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
}
preimageSeek += int64(len(slotnr))

mkv.addStorageSlot(migrdb.GetCurrentAccountAddress().Bytes(), slotnr, safeValue[:])
mkv.addStorageSlot(migrdb.GetCurrentAccountAddress(root).Bytes(), slotnr, safeValue[:])

// advance the storage iterator
migrdb.SetStorageProcessed(!stIt.Next())
if !migrdb.GetStorageProcessed() {
migrdb.SetCurrentSlotHash(stIt.Hash())
migrdb.SetStorageProcessed(!stIt.Next(), root)
if !migrdb.GetStorageProcessed(root) {
migrdb.SetCurrentSlotHash(stIt.Hash(), root)
}
}
stIt.Release()
Expand All @@ -178,20 +178,20 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
if count < maxMovedCount {
count++ // count increase for the account itself

mkv.addAccount(migrdb.GetCurrentAccountAddress().Bytes(), acc)
vkt.ClearStrorageRootConversion(*migrdb.GetCurrentAccountAddress())
mkv.addAccount(migrdb.GetCurrentAccountAddress(root).Bytes(), acc)
vkt.ClearStrorageRootConversion(*migrdb.GetCurrentAccountAddress(root))

// Store the account code if present
if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) {
code := rawdb.ReadCode(statedb.Database().DiskDB(), common.BytesToHash(acc.CodeHash))
chunks := trie.ChunkifyCode(code)

mkv.addAccountCode(migrdb.GetCurrentAccountAddress().Bytes(), uint64(len(code)), chunks)
mkv.addAccountCode(migrdb.GetCurrentAccountAddress(root).Bytes(), uint64(len(code)), chunks)
}

// reset storage iterator marker for next account
migrdb.SetStorageProcessed(false)
migrdb.SetCurrentSlotHash(common.Hash{})
migrdb.SetStorageProcessed(false, root)
migrdb.SetCurrentSlotHash(common.Hash{}, root)

// Move to the next account, if available - or end
// the transition otherwise.
Expand All @@ -212,7 +212,7 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
}
preimageSeek += int64(len(addr))
migrdb.SetCurrentAccountAddress(addr)
migrdb.SetCurrentAccountAddress(addr, root)
} else {
// case when the account iterator has
// reached the end but count < maxCount
Expand All @@ -221,9 +221,9 @@ func OverlayVerkleTransition(statedb *state.StateDB) error {
}
}
}
migrdb.SetCurrentPreimageOffset(preimageSeek)
migrdb.SetCurrentPreimageOffset(preimageSeek, root)

log.Info("Collected key values from base tree", "count", count, "duration", time.Since(now), "last account", statedb.Database().GetCurrentAccountHash())
log.Info("Collected key values from base tree", "count", count, "duration", time.Since(now), "last account", statedb.Database().GetCurrentAccountHash(root))

// Take all the collected key-values and prepare the new leaf values.
// This fires a background routine that will start doing the work that
Expand Down
90 changes: 52 additions & 38 deletions core/state/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ type Database interface {
// TrieDB retrieves the low level trie database used for data storage.
TrieDB() *trie.Database

StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunTime *uint64)
StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunTime *uint64, root common.Hash)

ReorgThroughVerkleTransition()

Expand All @@ -74,27 +74,27 @@ type Database interface {

Transitioned() bool

SetCurrentSlotHash(hash common.Hash)
SetCurrentSlotHash(common.Hash, common.Hash)

GetCurrentAccountAddress() *common.Address
GetCurrentAccountAddress(common.Hash) *common.Address

SetCurrentAccountAddress(common.Address)
SetCurrentAccountAddress(common.Address, common.Hash)

GetCurrentAccountHash() common.Hash
GetCurrentAccountHash(common.Hash) common.Hash

GetCurrentSlotHash() common.Hash
GetCurrentSlotHash(common.Hash) common.Hash

SetStorageProcessed(bool)
SetStorageProcessed(bool, common.Hash)

GetStorageProcessed() bool
GetStorageProcessed(common.Hash) bool

GetCurrentPreimageOffset() int64
GetCurrentPreimageOffset(common.Hash) int64

SetCurrentPreimageOffset(int64)
SetCurrentPreimageOffset(int64, common.Hash)

AddRootTranslation(originalRoot, translatedRoot common.Hash)

SetLastMerkleRoot(root common.Hash)
SetLastMerkleRoot(common.Hash)
}

// Trie is a Ethereum Merkle Patricia trie.
Expand Down Expand Up @@ -187,6 +187,10 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
triedb: trie.NewDatabaseWithConfig(db, config),
addrToPoint: utils.NewPointCache(),
StorageProcessed: map[common.Hash]bool{},
CurrentAccountAddress: map[common.Hash]*common.Address{},
CurrentSlotHash: map[common.Hash]common.Hash{},
CurrentPreimageOffset: map[common.Hash]int64{},
}
}

Expand All @@ -199,6 +203,10 @@ func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database {
triedb: triedb,
addrToPoint: utils.NewPointCache(),
ended: triedb.IsVerkle(),
StorageProcessed: map[common.Hash]bool{},
CurrentAccountAddress: map[common.Hash]*common.Address{},
CurrentSlotHash: map[common.Hash]common.Hash{},
CurrentPreimageOffset: map[common.Hash]int64{},
}
}

Expand All @@ -211,7 +219,7 @@ func (db *cachingDB) Transitioned() bool {
}

// Fork implements the fork
func (db *cachingDB) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64) {
func (db *cachingDB) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64, root common.Hash) {
fmt.Println(`
__________.__ .__ .__ __ .__ .__ ____
\__ ___| |__ ____ ____ | | ____ ______ | |__ _____ _____/ |_ | |__ _____ ______ __ _ _|__| ____ / ___\ ______
Expand All @@ -224,7 +232,13 @@ func (db *cachingDB) StartVerkleTransition(originalRoot, translatedRoot common.H
// db.AddTranslation(originalRoot, translatedRoot)
db.baseRoot = originalRoot
// initialize so that the first storage-less accounts are processed
db.StorageProcessed = true
db.StorageProcessed[root] = true

// Reinitialize values in case of a reorg
db.CurrentAccountAddress[root] = &(common.Address{})
db.CurrentSlotHash[root] = common.Hash{}
db.CurrentPreimageOffset[root] = 0

if pragueTime != nil {
chainConfig.PragueTime = pragueTime
}
Expand Down Expand Up @@ -263,14 +277,14 @@ type cachingDB struct {
addrToPoint *utils.PointCache

baseRoot common.Hash // hash of the read-only base tree
CurrentAccountAddress *common.Address // addresss of the last translated account
CurrentSlotHash common.Hash // hash of the last translated storage slot
CurrentPreimageOffset int64 // next byte to read from the preimage file
CurrentAccountAddress map[common.Hash]*common.Address // addresss of the last translated account
CurrentSlotHash map[common.Hash]common.Hash // hash of the last translated storage slot
CurrentPreimageOffset map[common.Hash]int64 // next byte to read from the preimage file

// Mark whether the storage for an account has been processed. This is useful if the
// maximum number of leaves of the conversion is reached before the whole storage is
// processed.
StorageProcessed bool
StorageProcessed map[common.Hash]bool
}

func (db *cachingDB) openMPTTrie(root common.Hash) (Trie, error) {
Expand Down Expand Up @@ -450,49 +464,49 @@ func (db *cachingDB) GetTreeKeyHeader(addr []byte) *verkle.Point {
return db.addrToPoint.GetTreeKeyHeader(addr)
}

func (db *cachingDB) SetCurrentAccountAddress(addr common.Address) {
db.CurrentAccountAddress = &addr
func (db *cachingDB) SetCurrentAccountAddress(addr common.Address, root common.Hash) {
db.CurrentAccountAddress[root] = &addr
}

func (db *cachingDB) GetCurrentAccountHash() common.Hash {
func (db *cachingDB) GetCurrentAccountHash(root common.Hash) common.Hash {
var addrHash common.Hash
if db.CurrentAccountAddress != nil {
addrHash = crypto.Keccak256Hash(db.CurrentAccountAddress[:])
if db.CurrentAccountAddress[root] != nil {
addrHash = crypto.Keccak256Hash(db.CurrentAccountAddress[root][:])
}
return addrHash
}

func (db *cachingDB) GetCurrentAccountAddress() *common.Address {
return db.CurrentAccountAddress
func (db *cachingDB) GetCurrentAccountAddress(root common.Hash) *common.Address {
return db.CurrentAccountAddress[root]
}

func (db *cachingDB) GetCurrentPreimageOffset() int64 {
return db.CurrentPreimageOffset
func (db *cachingDB) GetCurrentPreimageOffset(root common.Hash) int64 {
return db.CurrentPreimageOffset[root]
}

func (db *cachingDB) SetCurrentPreimageOffset(offset int64) {
db.CurrentPreimageOffset = offset
func (db *cachingDB) SetCurrentPreimageOffset(offset int64, root common.Hash) {
db.CurrentPreimageOffset[root] = offset
}

func (db *cachingDB) SetCurrentSlotHash(hash common.Hash) {
db.CurrentSlotHash = hash
func (db *cachingDB) SetCurrentSlotHash(hash common.Hash, root common.Hash) {
db.CurrentSlotHash[root] = hash
}

func (db *cachingDB) GetCurrentSlotHash() common.Hash {
return db.CurrentSlotHash
func (db *cachingDB) GetCurrentSlotHash(root common.Hash) common.Hash {
return db.CurrentSlotHash[root]
}

func (db *cachingDB) SetStorageProcessed(processed bool) {
db.StorageProcessed = processed
func (db *cachingDB) SetStorageProcessed(processed bool, root common.Hash) {
db.StorageProcessed[root] = processed
}

func (db *cachingDB) GetStorageProcessed() bool {
return db.StorageProcessed
func (db *cachingDB) GetStorageProcessed(root common.Hash) bool {
return db.StorageProcessed[root]
}

func (db *cachingDB) AddRootTranslation(originalRoot, translatedRoot common.Hash) {
}

func (db *cachingDB) SetLastMerkleRoot(root common.Hash) {
db.LastMerkleRoot = root
func (db *cachingDB) SetLastMerkleRoot(merkleRoot common.Hash) {
db.LastMerkleRoot = merkleRoot
}
3 changes: 2 additions & 1 deletion core/state_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
}

// Perform the overlay transition, if relevant
if err := OverlayVerkleTransition(statedb); err != nil {
parent := p.bc.GetHeaderByHash(header.ParentHash)
if err := OverlayVerkleTransition(statedb, parent.Root); err != nil {
return nil, nil, 0, fmt.Errorf("error performing verkle overlay transition: %w", err)
}

Expand Down
2 changes: 1 addition & 1 deletion eth/catalyst/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe
if api.eth.BlockChain().Config().IsPrague(block.Number(), block.Time()) && !api.eth.BlockChain().Config().IsPrague(parent.Number(), parent.Time()) {
parent := api.eth.BlockChain().GetHeaderByNumber(block.NumberU64() - 1)
if !api.eth.BlockChain().Config().IsPrague(parent.Number, parent.Time) {
api.eth.BlockChain().StartVerkleTransition(parent.Root, common.Hash{}, api.eth.BlockChain().Config(), nil)
api.eth.BlockChain().StartVerkleTransition(parent.Root, common.Hash{}, api.eth.BlockChain().Config(), nil, parent.Root)
}
}
// Reset db merge state in case of a reorg
Expand Down
Loading

0 comments on commit a9d94e8

Please sign in to comment.