post-genesis transition (#426)

Support post-genesis transition

Reactivate a post-genesis transition for a kaustinen-like testnet that goes through the conversion.

It adds the ability:

 * To undergo the verkle transition "verge" past the genesis block, which so far was impossible with kaustinen-with-shapella.
 * To perform a shadow fork of holesky in the special kurtosis setup defined by the devops team.

It is also serving as the basis for multiple other branches, to be merged at a later time:

 * The branch to replay historical blocks in verkle 
 * The branch to execute transition tests

Co-authored-by: Ignacio Hagopian <[email protected]>

cmd/geth/config.go

@@ -175,6 +175,17 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		v := ctx.Uint64(utils.OverridePrague.Name)
 		cfg.Eth.OverridePrague = &v
 	}
+	if ctx.IsSet(utils.OverrideProofInBlock.Name) {
+		v := ctx.Bool(utils.OverrideProofInBlock.Name)
+		cfg.Eth.OverrideProofInBlock = &v
+	}
+	if ctx.IsSet(utils.OverrideOverlayStride.Name) {
+		v := ctx.Uint64(utils.OverrideOverlayStride.Name)
+		cfg.Eth.OverrideOverlayStride = &v
+	}
+	if ctx.IsSet(utils.ClearVerkleCosts.Name) {
+		params.ClearVerkleWitnessCosts()
+	}
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 
 	// Configure log filter RPC API.

cmd/geth/main.go

@@ -67,8 +67,11 @@ var (
 		utils.NoUSBFlag,
 		utils.USBFlag,
 		utils.SmartCardDaemonPathFlag,
+		utils.OverrideOverlayStride,
 		utils.OverrideCancun,
 		utils.OverridePrague,
+		utils.OverrideProofInBlock,
+		utils.ClearVerkleCosts,
 		utils.EnablePersonal,
 		utils.TxPoolLocalsFlag,
 		utils.TxPoolNoLocalsFlag,

cmd/utils/flags.go

@@ -263,6 +263,12 @@ var (
 		Value:    2048,
 		Category: flags.EthCategory,
 	}
+	OverrideOverlayStride = &cli.Uint64Flag{
+		Name:     "override.overlay-stride",
+		Usage:    "Manually specify the stride of the overlay transition, overriding the bundled setting",
+		Value:    10000,
+		Category: flags.EthCategory,
+	}
 	OverrideCancun = &cli.Uint64Flag{
 		Name:     "override.cancun",
 		Usage:    "Manually specify the Cancun fork timestamp, overriding the bundled setting",
@@ -273,6 +279,17 @@ var (
 		Usage:    "Manually specify the Verkle fork timestamp, overriding the bundled setting",
 		Category: flags.EthCategory,
 	}
+	OverrideProofInBlock = &cli.BoolFlag{
+		Name:     "override.blockproof",
+		Usage:    "Manually specify the proof-in-block setting",
+		Value:    true,
+		Category: flags.EthCategory,
+	}
+	ClearVerkleCosts = &cli.BoolFlag{
+		Name:     "clear.verkle.costs",
+		Usage:    "Clear verkle costs (for shadow forks)",
+		Category: flags.EthCategory,
+	}
 	// Light server and client settings
 	LightServeFlag = &cli.IntFlag{
 		Name:     "light.serve",

consensus/beacon/consensus.go

@@ -25,8 +25,10 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
+	"github.com/ethereum/go-ethereum/core/overlay"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/trie"
@@ -358,6 +360,15 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.
 		// The returned gas is not charged
 		state.Witness().TouchFullAccount(w.Address[:], true)
 	}
+
+	if chain.Config().IsPrague(header.Number, header.Time) {
+		// uncomment when debugging
+		// fmt.Println("at block", header.Number, "performing transition?", state.Database().InTransition())
+		parent := chain.GetHeaderByHash(header.ParentHash)
+		if err := overlay.OverlayVerkleTransition(state, parent.Root, chain.Config().OverlayStride); err != nil {
+			log.Error("error performing the transition", "err", err)
+		}
+	}
 }
 
 // FinalizeAndAssemble implements consensus.Engine, setting the final state and
@@ -382,51 +393,72 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea
 
 	// Assign the final state root to header.
 	header.Root = state.IntermediateRoot(true)
+	// Associate current conversion state to computed state
+	// root and store it in the database for later recovery.
+	state.Database().SaveTransitionState(header.Root)
 
 	var (
 		p    *verkle.VerkleProof
 		k    verkle.StateDiff
 		keys = state.Witness().Keys()
 	)
-	if chain.Config().IsPrague(header.Number, header.Time) && chain.Config().ProofInBlocks {
+	if chain.Config().IsPrague(header.Number, header.Time) {
 		// Open the pre-tree to prove the pre-state against
 		parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1)
 		if parent == nil {
 			return nil, fmt.Errorf("nil parent header for block %d", header.Number)
 		}
 
-		preTrie, err := state.Database().OpenTrie(parent.Root)
-		if err != nil {
-			return nil, fmt.Errorf("error opening pre-state tree root: %w", err)
-		}
+		// Load transition state at beginning of block, because
+		// OpenTrie needs to know what the conversion status is.
+		state.Database().LoadTransitionState(parent.Root)
 
-		var okpre, okpost bool
-		var vtrpre, vtrpost *trie.VerkleTrie
-		switch pre := preTrie.(type) {
-		case *trie.VerkleTrie:
-			vtrpre, okpre = preTrie.(*trie.VerkleTrie)
-			vtrpost, okpost = state.GetTrie().(*trie.VerkleTrie)
-		case *trie.TransitionTrie:
-			vtrpre = pre.Overlay()
-			okpre = true
-			post, _ := state.GetTrie().(*trie.TransitionTrie)
-			vtrpost = post.Overlay()
-			okpost = true
-		default:
-			// This should only happen for the first block of the
-			// conversion, when the previous tree is a merkle tree.
-			//  Logically, the "previous" verkle tree is an empty tree.
-			okpre = true
-			vtrpre = trie.NewVerkleTrie(verkle.New(), state.Database().TrieDB(), utils.NewPointCache(), false)
-			post := state.GetTrie().(*trie.TransitionTrie)
-			vtrpost = post.Overlay()
-			okpost = true
-		}
-		if okpre && okpost {
-			if len(keys) > 0 {
-				p, k, err = trie.ProveAndSerialize(vtrpre, vtrpost, keys, vtrpre.FlatdbNodeResolver)
-				if err != nil {
-					return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err)
+		if chain.Config().ProofInBlocks {
+			preTrie, err := state.Database().OpenTrie(parent.Root)
+			if err != nil {
+				return nil, fmt.Errorf("error opening pre-state tree root: %w", err)
+			}
+
+			var okpre, okpost bool
+			var vtrpre, vtrpost *trie.VerkleTrie
+			switch pre := preTrie.(type) {
+			case *trie.VerkleTrie:
+				vtrpre, okpre = preTrie.(*trie.VerkleTrie)
+				switch tr := state.GetTrie().(type) {
+				case *trie.VerkleTrie:
+					vtrpost = tr
+					okpost = true
+				// This is to handle a situation right at the start of the conversion:
+				// the post trie is a transition tree when the pre tree is an empty
+				// verkle tree.
+				case *trie.TransitionTrie:
+					vtrpost = tr.Overlay()
+					okpost = true
+				default:
+					okpost = false
+				}
+			case *trie.TransitionTrie:
+				vtrpre = pre.Overlay()
+				okpre = true
+				post, _ := state.GetTrie().(*trie.TransitionTrie)
+				vtrpost = post.Overlay()
+				okpost = true
+			default:
+				// This should only happen for the first block of the
+				// conversion, when the previous tree is a merkle tree.
+				//  Logically, the "previous" verkle tree is an empty tree.
+				okpre = true
+				vtrpre = trie.NewVerkleTrie(verkle.New(), state.Database().TrieDB(), utils.NewPointCache(), false)
+				post := state.GetTrie().(*trie.TransitionTrie)
+				vtrpost = post.Overlay()
+				okpost = true
+			}
+			if okpre && okpost {
+				if len(keys) > 0 {
+					p, k, err = trie.ProveAndSerialize(vtrpre, vtrpost, keys, vtrpre.FlatdbNodeResolver)
+					if err != nil {
+						return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err)
+					}
 				}
 			}
 		}

core/block_validator.go

@@ -131,6 +131,10 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
 	if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {
 		return fmt.Errorf("invalid merkle root (remote: %x local: %x) dberr: %w", header.Root, root, statedb.Error())
 	}
+	// Verify that the advertised root is correct before
+	// it can be used as an identifier for the conversion
+	// status.
+	statedb.Database().SaveTransitionState(header.Root)
 	return nil
 }
 

core/blockchain.go

@@ -251,6 +251,14 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok {
 		return nil, genesisErr
 	}
+	if overrides != nil {
+		if overrides.OverrideProofInBlock != nil {
+			chainConfig.ProofInBlocks = *overrides.OverrideProofInBlock
+		}
+		if overrides.OverrideOverlayStride != nil {
+			chainConfig.OverlayStride = *overrides.OverrideOverlayStride
+		}
+	}
 	log.Info("")
 	log.Info(strings.Repeat("-", 153))
 	for _, line := range strings.Split(chainConfig.Description(), "\n") {
@@ -312,7 +320,13 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	head := bc.CurrentBlock()
 
 	// Declare the end of the verkle transition if need be
-	if bc.chainConfig.Rules(head.Number, false /* XXX */, head.Time).IsPrague {
+	if bc.chainConfig.IsPrague(head.Number, head.Time) {
+		// TODO this only works when resuming a chain that has already gone
+		// through the conversion. All pointers should be saved to the DB
+		// for it to be able to recover if interrupted during the transition
+		// but that's left out to a later PR since there's not really a need
+		// right now.
+		bc.stateCache.InitTransitionStatus(true, true)
 		bc.stateCache.EndVerkleTransition()
 	}
 
@@ -1746,8 +1760,22 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
 		}
 
+		if bc.Config().IsPrague(block.Number(), block.Time()) {
+			bc.stateCache.LoadTransitionState(parent.Root)
+
+			// pragueTime has been reached. If the transition isn't active, it means this
+			// is the fork block and that the conversion needs to be marked at started.
+			if !bc.stateCache.InTransition() && !bc.stateCache.Transitioned() {
+				bc.stateCache.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), bc.Config().PragueTime, parent.Root)
+			}
+		} else {
+			// If the verkle activation time hasn't started, declare it as "not started".
+			// This is so that if the miner activates the conversion, the insertion happens
+			// in the correct mode.
+			bc.stateCache.InitTransitionStatus(false, false)
+		}
 		if parent.Number.Uint64() == conversionBlock {
-			bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), &parent.Time)
+			bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), &parent.Time, parent.Root)
 			bc.stateCache.SetLastMerkleRoot(parent.Root)
 		}
 		statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps)
@@ -1989,7 +2017,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i
 		parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1)
 	}
 	if parent == nil {
-		return it.index, errors.New("missing parent")
+		return it.index, fmt.Errorf("missing parent: hash=%x, number=%d", current.Hash(), current.Number)
 	}
 	// Import all the pruned blocks to make the state available
 	var (
@@ -2050,7 +2078,7 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error)
 		}
 	}
 	if parent == nil {
-		return common.Hash{}, errors.New("missing parent")
+		return common.Hash{}, fmt.Errorf("missing parent during ancestor recovery: hash=%x, number=%d", block.ParentHash(), block.Number())
 	}
 	// Import all the pruned blocks to make the state available
 	for i := len(hashes) - 1; i >= 0; i-- {
@@ -2288,6 +2316,7 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) {
 	defer bc.chainmu.Unlock()
 
 	// Re-execute the reorged chain in case the head state is missing.
+	log.Trace("looking for state", "root", head.Root(), "has state", bc.HasState(head.Root()))
 	if !bc.HasState(head.Root()) {
 		if latestValidHash, err := bc.recoverAncestors(head); err != nil {
 			return latestValidHash, err
@@ -2533,8 +2562,8 @@ func (bc *BlockChain) GetTrieFlushInterval() time.Duration {
 	return time.Duration(bc.flushInterval.Load())
 }
 
-func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64) {
-	bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, pragueTime)
+func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, pragueTime *uint64, root common.Hash) {
+	bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, pragueTime, root)
 }
 func (bc *BlockChain) ReorgThroughVerkleTransition() {
 	bc.stateCache.ReorgThroughVerkleTransition()

core/chain_makers.go

@@ -365,7 +365,7 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int,
 	return db, blocks, receipts
 }
 
-func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) {
+func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, diskdb ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) {
 	if config == nil {
 		config = params.TestChainConfig
 	}
@@ -434,13 +434,16 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
 		return nil, nil
 	}
 	var snaps *snapshot.Tree
-	triedb := state.NewDatabaseWithConfig(db, nil)
-	triedb.EndVerkleTransition()
+	db := state.NewDatabaseWithConfig(diskdb, nil)
+	db.StartVerkleTransition(common.Hash{}, common.Hash{}, config, config.PragueTime, common.Hash{})
+	db.EndVerkleTransition()
+	db.SaveTransitionState(parent.Root())
 	for i := 0; i < n; i++ {
-		statedb, err := state.New(parent.Root(), triedb, snaps)
+		statedb, err := state.New(parent.Root(), db, snaps)
 		if err != nil {
 			panic(fmt.Sprintf("could not find state for block %d: err=%v, parent root=%x", i, err, parent.Root()))
 		}
+		statedb.NewAccessWitness()
 		block, receipt := genblock(i, parent, statedb)
 		blocks[i] = block
 		receipts[i] = receipt

core/genesis.go

@@ -126,6 +126,7 @@ func (ga *GenesisAlloc) deriveHash(cfg *params.ChainConfig, timestamp uint64) (c
 	// all the derived states will be discarded to not pollute disk.
 	db := state.NewDatabase(rawdb.NewMemoryDatabase())
 	if cfg.IsPrague(big.NewInt(int64(0)), timestamp) {
+		db.StartVerkleTransition(common.Hash{}, common.Hash{}, cfg, &timestamp, common.Hash{})
 		db.EndVerkleTransition()
 	}
 	statedb, err := state.New(types.EmptyRootHash, db, nil)
@@ -146,15 +147,17 @@ func (ga *GenesisAlloc) deriveHash(cfg *params.ChainConfig, timestamp uint64) (c
 // flush is very similar with deriveHash, but the main difference is
 // all the generated states will be persisted into the given database.
 // Also, the genesis state specification will be flushed as well.
-func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash, cfg *params.ChainConfig) error {
-	statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil)
-	if err != nil {
-		return err
+func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash, cfg *params.ChainConfig, timestamp *uint64) error {
+	database := state.NewDatabaseWithNodeDB(db, triedb)
+	// End the verkle conversion at genesis if the fork block is 0
+	if timestamp != nil && cfg.IsPrague(big.NewInt(int64(0)), *timestamp) {
+		database.StartVerkleTransition(common.Hash{}, common.Hash{}, cfg, timestamp, common.Hash{})
+		database.EndVerkleTransition()
 	}
 
-	// End the verkle conversion at genesis if the fork block is 0
-	if triedb.IsVerkle() {
-		statedb.Database().EndVerkleTransition()
+	statedb, err := state.New(types.EmptyRootHash, database, nil)
+	if err != nil {
+		return err
 	}
 
 	for addr, account := range *ga {
@@ -221,7 +224,7 @@ func CommitGenesisState(db ethdb.Database, triedb *trie.Database, blockhash comm
 			return errors.New("not found")
 		}
 	}
-	return alloc.flush(db, triedb, blockhash, config)
+	return alloc.flush(db, triedb, blockhash, config, nil)
 }
 
 // GenesisAccount is an account in the state of the genesis block.
@@ -288,8 +291,10 @@ func (e *GenesisMismatchError) Error() string {
 
 // ChainOverrides contains the changes to chain config.
 type ChainOverrides struct {
-	OverrideCancun *uint64
-	OverridePrague *uint64
+	OverrideCancun        *uint64
+	OverridePrague        *uint64
+	OverrideProofInBlock  *bool
+	OverrideOverlayStride *uint64
 }
 
 // SetupGenesisBlock writes or updates the genesis block in db.
@@ -536,7 +541,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block
 	// All the checks has passed, flush the states derived from the genesis
 	// specification as well as the specification itself into the provided
 	// database.
-	if err := g.Alloc.flush(db, triedb, block.Hash(), g.Config); err != nil {
+	if err := g.Alloc.flush(db, triedb, block.Hash(), g.Config, &g.Timestamp); err != nil {
 		return nil, err
 	}
 	rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty())