t8n: verkle genesis (#466)

* t8n: verkle-genesis Signed-off-by: Ignacio Hagopian <[email protected]> * review feedback Signed-off-by: Ignacio Hagopian <[email protected]> * imprv Signed-off-by: Ignacio Hagopian <[email protected]> * t8n/evm: add new cli command Signed-off-by: Ignacio Hagopian <[email protected]> * fixes to run conversion fixture Signed-off-by: Ignacio Hagopian <[email protected]> * t8n/evm: use pre alloc to save code in rawdb Signed-off-by: Ignacio Hagopian <[email protected]> * vm: fix gas underflow Signed-off-by: Ignacio Hagopian <[email protected]> * ci: add verkle-genesis filling and consumption Signed-off-by: Ignacio Hagopian <[email protected]> * ci: update runner Signed-off-by: Ignacio Hagopian <[email protected]> * ci: fixes Signed-off-by: Ignacio Hagopian <[email protected]> * ci: use consume direct to run fixtures Signed-off-by: Ignacio Hagopian <[email protected]> * ci: fixes Signed-off-by: Ignacio Hagopian <[email protected]> * ci: include 4762 Signed-off-by: Ignacio Hagopian <[email protected]> * ci: add transition tests Signed-off-by: Ignacio Hagopian <[email protected]> * ci: parallelize running Signed-off-by: Ignacio Hagopian <[email protected]> * ci: include conversion tests | use matrix. (#470) * feat: tweak eest ci (matrix). * chore: remove prev ci. * t8n/evm: fix pre-state for forks before verkle Signed-off-by: Ignacio Hagopian <[email protected]> * t8n: remove EIP6800Transition from forks Signed-off-by: Ignacio Hagopian <[email protected]> * ci: change execution-spec-tests target branch Signed-off-by: Ignacio Hagopian <[email protected]> * ci: fill more tests Signed-off-by: Ignacio Hagopian <[email protected]> * Revert "t8n: remove EIP6800Transition from forks" This reverts commit 660e2eb. * ci: use verkle/main branch for tests Signed-off-by: Ignacio Hagopian <[email protected]> * add todo Signed-off-by: Ignacio Hagopian <[email protected]> * preimages fixes Signed-off-by: Ignacio Hagopian <[email protected]> * fix genesis Signed-off-by: Ignacio Hagopian <[email protected]> * record preimages in genesis commit Signed-off-by: Ignacio Hagopian <[email protected]> * review feedback Signed-off-by: Ignacio Hagopian <[email protected]> --------- Signed-off-by: Ignacio Hagopian <[email protected]> Co-authored-by: spencer <[email protected]>
gballet · Aug 23, 2024 · 9b27fba · 9b27fba
1 parent 9be0b58
commit 9b27fba
Show file tree

Hide file tree

Showing 9 changed files with 252 additions and 47 deletions.
diff --git a/.github/workflows/spec-tests.yml b/.github/workflows/spec-tests.yml
@@ -0,0 +1,118 @@
+name: Execution Spec Tests Fill and Consume (Verkle genesis & conversion)
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master, kaustinen-with-shapella]
+  workflow_dispatch:
+
+env:
+  EEST_USER: "ethereum"
+  EEST_BRANCH: "verkle/main"
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout go-ethereum
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12.4"
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22.4
+
+      - name: Build geth evm
+        run: |
+          go build -v ./cmd/evm
+          mkdir -p ${{ github.workspace }}/bin
+          mv evm ${{ github.workspace }}/bin/evm
+          chmod +x ${{ github.workspace }}/bin/evm
+
+      - name: Archive built evm
+        uses: actions/upload-artifact@v4
+        with:
+          name: evm
+          path: ${{ github.workspace }}/bin/evm
+
+  fill:
+    runs-on: ubuntu-latest
+    needs: setup
+    strategy:
+      matrix:
+        test-type: [genesis, conversion]
+    steps:
+      - name: Download geth evm
+        uses: actions/download-artifact@v4
+        with:
+          name: evm
+          path: ./bin
+
+      - name: Make evm binary executable and add to PATH
+        run: |
+          chmod +x ./bin/evm
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+      - name: Clone execution-spec-tests and fill tests
+        run: |
+          git clone https://github.com/${{ env.EEST_USER }}/execution-spec-tests -b ${{ env.EEST_BRANCH }}
+          cd execution-spec-tests
+          python3 -m venv venv
+          . venv/bin/activate
+          pip install --upgrade pip
+          pip install -e ".[docs,lint,test]"
+          solc-select use 0.8.24 --always-install
+          if [ "${{ matrix.test-type }}" == "genesis" ]; then
+            fill --fork Verkle --output=../fixtures-${{ matrix.test-type }} -v -m blockchain_test -n auto
+          else
+            fill --from Shanghai --until EIP6800Transition --output=../fixtures-${{ matrix.test-type }} -v -m blockchain_test -n auto
+          fi
+        shell: bash
+
+      - name: Upload fixtures
+        uses: actions/upload-artifact@v4
+        with:
+          name: fixtures-${{ matrix.test-type }}
+          path: fixtures-${{ matrix.test-type }}
+
+  consume:
+    runs-on: ubuntu-latest
+    needs: fill
+    strategy:
+      matrix:
+        test-type: [genesis, conversion]
+    steps:
+      - name: Download geth evm
+        uses: actions/download-artifact@v4
+        with:
+          name: evm
+          path: ./bin
+
+      - name: Make evm binary executable and add to PATH
+        run: |
+          chmod +x ./bin/evm
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+      - name: Download fixtures
+        uses: actions/download-artifact@v4
+        with:
+          name: fixtures-${{ matrix.test-type }}
+          path: ./fixtures-${{ matrix.test-type }}
+
+      - name: Clone execution-spec-tests and consume tests
+        run: |
+          git clone https://github.com/${{ env.EEST_USER }}/execution-spec-tests -b ${{ env.EEST_BRANCH }}
+          cd execution-spec-tests
+          python3 -m venv venv
+          . venv/bin/activate
+          pip install --upgrade pip
+          pip install -e ".[docs,lint,test]"
+          solc-select use 0.8.24 --always-install
+          consume direct --input=../fixtures-${{ matrix.test-type }} -n auto
+        shell: bash
diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go
@@ -64,7 +64,7 @@ func blockTestCmd(ctx *cli.Context) error {
 		return err
 	}
 	for i, test := range tests {
-		if err := test.Run(false, tracer); err != nil {
+		if err := test.Run(true, tracer); err != nil {
 			return fmt.Errorf("test %v: %w", i, err)
 		}
 	}

diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go
@@ -185,17 +185,16 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		GasLimit:    pre.Env.GasLimit,
 		GetHash:     getHash,
 	}
-	// Save pre verkle tree to build the proof at the end
-	if pre.VKT != nil && len(pre.VKT) > 0 {
-		switch tr := statedb.GetTrie().(type) {
-		case *trie.VerkleTrie:
-			vtrpre = tr.Copy()
-		case *trie.TransitionTrie:
-			vtrpre = tr.Overlay().Copy()
-		default:
-			panic("invalid trie type")
-		}
+
+	// We save the current state of the Verkle Tree before applying the transactions.
+	// Note that if the Verkle fork isn't active, this will be a noop.
+	switch tr := statedb.GetTrie().(type) {
+	case *trie.VerkleTrie:
+		vtrpre = tr.Copy()
+	case *trie.TransitionTrie:
+		vtrpre = tr.Overlay().Copy()
 	}
+
 	// If currentBaseFee is defined, add it to the vmContext.
 	if pre.Env.BaseFee != nil {
 		vmContext.BaseFee = new(big.Int).Set(pre.Env.BaseFee)
@@ -428,8 +427,33 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 func MakePreState(db ethdb.Database, chainConfig *params.ChainConfig, pre *Prestate, verkle bool) *state.StateDB {
 	// Start with generating the MPT DB, which should be empty if it's post-verkle transition
 	sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true, Verkle: false})
+
+	if pre.Env.Ended != nil && *pre.Env.Ended {
+		sdb.InitTransitionStatus(true, true, common.Hash{})
+	}
+
 	statedb, _ := state.New(types.EmptyRootHash, sdb, nil)
 
+	if pre.Env.Ended != nil && *pre.Env.Ended {
+		vtr := statedb.GetTrie().(*trie.VerkleTrie)
+
+		// create the vkt, should be empty on first insert
+		for k, v := range pre.VKT {
+			values := make([][]byte, 256)
+			values[k[31]] = make([]byte, 32)
+			copy(values[k[31]], v)
+			vtr.UpdateStem(k.Bytes(), values)
+		}
+
+		codeWriter := statedb.Database().DiskDB()
+		for _, acc := range pre.Pre {
+			codeHash := crypto.Keccak256Hash(acc.Code)
+			rawdb.WriteCode(codeWriter, codeHash, acc.Code)
+		}
+
+		return statedb
+	}
+
 	// MPT pre is the same as the pre state for first conversion block
 	for addr, a := range pre.Pre {
 		statedb.SetCode(addr, a.Code)
@@ -440,14 +464,21 @@ func MakePreState(db ethdb.Database, chainConfig *params.ChainConfig, pre *Prest
 		}
 	}
 
+	state.NoBanner()
+	// Commit db an create a snapshot from it.
+	mptRoot, err := statedb.Commit(0, false)
+	if err != nil {
+		panic(err)
+	}
+
 	// If verkle mode started, establish the conversion
 	if verkle {
-		state.NoBanner()
-		// Commit db an create a snapshot from it.
-		mptRoot, err := statedb.Commit(0, false)
-		if err != nil {
-			panic(err)
+		// If the current tree is a VerkleTrie, it means the state conversion has ended.
+		// We don't need to continue with conversion setups and can return early.
+		if _, ok := statedb.GetTrie().(*trie.VerkleTrie); ok {
+			return statedb
 		}
+
 		rawdb.WritePreimages(sdb.DiskDB(), statedb.Preimages())
 		sdb.TrieDB().WritePreimages()
 		snaps, err := snapshot.New(snapshot.Config{AsyncBuild: false, CacheSize: 10}, sdb.DiskDB(), sdb.TrieDB(), mptRoot)
@@ -522,6 +553,11 @@ func MakePreState(db ethdb.Database, chainConfig *params.ChainConfig, pre *Prest
 		if err != nil {
 			panic(err)
 		}
+	} else {
+		statedb, err = state.New(mptRoot, sdb, nil)
+		if err != nil {
+			panic(err)
+		}
 	}
 
 	if statedb.Database().InTransition() || statedb.Database().Transitioned() {

diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go
@@ -548,13 +548,65 @@ func VerkleKeys(ctx *cli.Context) error {
 		}
 	}
 
+	vkt, err := genVktFromAlloc(alloc)
+	if err != nil {
+		return fmt.Errorf("error generating vkt: %w", err)
+	}
+
+	collector := make(map[common.Hash]hexutil.Bytes)
+	it, err := vkt.NodeIterator(nil)
+	if err != nil {
+		panic(err)
+	}
+	for it.Next(true) {
+		if it.Leaf() {
+			collector[common.BytesToHash(it.LeafKey())] = it.LeafBlob()
+		}
+	}
+
+	output, err := json.MarshalIndent(collector, "", "")
+	if err != nil {
+		return fmt.Errorf("error outputting tree: %w", err)
+	}
+
+	fmt.Println(string(output))
+
+	return nil
+}
+
+// VerkleRoot computes the root of a VKT from a genesis alloc.
+func VerkleRoot(ctx *cli.Context) error {
+	var allocStr = ctx.String(InputAllocFlag.Name)
+	var alloc core.GenesisAlloc
+	if allocStr == stdinSelector {
+		decoder := json.NewDecoder(os.Stdin)
+		if err := decoder.Decode(&alloc); err != nil {
+			return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err))
+		}
+	}
+	if allocStr != stdinSelector {
+		if err := readFile(allocStr, "alloc", &alloc); err != nil {
+			return err
+		}
+	}
+
+	vkt, err := genVktFromAlloc(alloc)
+	if err != nil {
+		return fmt.Errorf("error generating vkt: %w", err)
+	}
+	fmt.Println(vkt.Hash().Hex())
+
+	return nil
+}
+
+func genVktFromAlloc(alloc core.GenesisAlloc) (*trie.VerkleTrie, error) {
 	vkt := trie.NewVerkleTrie(verkle.New(), trie.NewDatabase(rawdb.NewMemoryDatabase()), utils.NewPointCache(), true)
 
 	for addr, acc := range alloc {
 		for slot, value := range acc.Storage {
 			err := vkt.UpdateStorage(addr, slot.Bytes(), value.Big().Bytes())
 			if err != nil {
-				return fmt.Errorf("error inserting storage: %w", err)
+				return nil, fmt.Errorf("error inserting storage: %w", err)
 			}
 		}
 
@@ -566,34 +618,15 @@ func VerkleKeys(ctx *cli.Context) error {
 		}
 		err := vkt.UpdateAccount(addr, account)
 		if err != nil {
-			return fmt.Errorf("error inserting account: %w", err)
+			return nil, fmt.Errorf("error inserting account: %w", err)
 		}
 
 		err = vkt.UpdateContractCode(addr, common.BytesToHash(account.CodeHash), acc.Code)
 		if err != nil {
-			return fmt.Errorf("error inserting code: %w", err)
-		}
-	}
-
-	collector := make(map[common.Hash]hexutil.Bytes)
-	it, err := vkt.NodeIterator(nil)
-	if err != nil {
-		panic(err)
-	}
-	for it.Next(true) {
-		if it.Leaf() {
-			collector[common.BytesToHash(it.LeafKey())] = it.LeafBlob()
+			return nil, fmt.Errorf("error inserting code: %w", err)
 		}
 	}
-
-	output, err := json.MarshalIndent(collector, "", "")
-	if err != nil {
-		return fmt.Errorf("error outputting tree: %w", err)
-	}
-
-	fmt.Println(string(output))
-
-	return nil
+	return vkt, nil
 }
 
 // VerkleCodeChunkKey computes the tree key of a code-chunk for a given address.

diff --git a/cmd/evm/main.go b/cmd/evm/main.go
@@ -172,22 +172,31 @@ var verkleCommand = &cli.Command{
 		},
 		{
 			Name:    "single-key",
-			Aliases: []string{"V"},
+			Aliases: []string{"vk"},
 			Usage:   "compute the verkle tree key given an address and optional slot number",
 			Action:  t8ntool.VerkleKey,
 		},
 		{
 			Name:    "code-chunk-key",
-			Aliases: []string{"VCK"},
+			Aliases: []string{"vck"},
 			Usage:   "compute the verkle tree key given an address and chunk number",
 			Action:  t8ntool.VerkleCodeChunkKey,
 		},
 		{
 			Name:    "chunkify-code",
-			Aliases: []string{"VCC"},
+			Aliases: []string{"vcc"},
 			Usage:   "chunkify a given bytecode",
 			Action:  t8ntool.VerkleChunkifyCode,
 		},
+		{
+			Name:    "state-root",
+			Aliases: []string{"vsr"},
+			Usage:   "compute the state-root of a verkle tree for the given alloc",
+			Action:  t8ntool.VerkleRoot,
+			Flags: []cli.Flag{
+				t8ntool.InputAllocFlag,
+			},
+		},
 	},
 }
 

diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go
@@ -367,6 +367,7 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.
 		parent := chain.GetHeaderByHash(header.ParentHash)
 		if err := overlay.OverlayVerkleTransition(state, parent.Root, chain.Config().OverlayStride); err != nil {
 			log.Error("error performing the transition", "err", err)
+			panic(fmt.Sprintf("error performing the transition: %s", err))
 		}
 	}
 }