Merge branch 'master' into secp256r1-precompile

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          submodules: true
+          submodules: recursive
 
       - name: Set up Go
         uses: actions/setup-go@v4

build/checksums.txt

@@ -1,25 +1,30 @@
 # This file contains sha256 checksums of optional build dependencies.
 
+# version:spec-tests 1.0.6
 # https://github.com/ethereum/execution-spec-tests/releases
-24bac679f3a2d8240d8e08e7f6a70b70c2dabf673317d924cf1d1887b9fe1f81  fixtures.tar.gz
+# https://github.com/ethereum/execution-spec-tests/releases/download/v1.0.6/
+485af7b66cf41eb3a8c1bd46632913b8eb95995df867cf665617bbc9b4beedd1  fixtures_develop.tar.gz
 
+# version:golang 1.21.3
 # https://go.dev/dl/
-bfa36bf75e9a1e9cbbdb9abcf9d1707e479bd3a07880a8ae3564caee5711cb99  go1.21.1.src.tar.gz
-809f5b0ef4f7dcdd5f51e9630a5b2e5a1006f22a047126d61560cdc365678a19  go1.21.1.darwin-amd64.tar.gz
-ffd40391a1e995855488b008ad9326ff8c2e81803a6e80894401003bae47fcf1  go1.21.1.darwin-arm64.tar.gz
-9919a9a4dc82371aba3da5b7c830bcb6249fc1502cd26d959eb340a60e41ee01  go1.21.1.freebsd-386.tar.gz
-2571f10f6047e04d87c1f5986a05e5e8f7b511faf98803ef12b66d563845d2a1  go1.21.1.freebsd-amd64.tar.gz
-b93850666cdadbd696a986cf7b03111fe99db8c34a9aaa113d7c96d0081e1901  go1.21.1.linux-386.tar.gz
-b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae  go1.21.1.linux-amd64.tar.gz
-7da1a3936a928fd0b2602ed4f3ef535b8cd1990f1503b8d3e1acc0fa0759c967  go1.21.1.linux-arm64.tar.gz
-f3716a43f59ae69999841d6007b42c9e286e8d8ce470656fb3e70d7be2d7ca85  go1.21.1.linux-armv6l.tar.gz
-eddf018206f8a5589bda75252b72716d26611efebabdca5d0083ec15e9e41ab7  go1.21.1.linux-ppc64le.tar.gz
-a83b3e8eb4dbf76294e773055eb51397510ff4d612a247bad9903560267bba6d  go1.21.1.linux-s390x.tar.gz
-170256c820f466f29d64876f25f4dfa4029ed9902a0a9095d8bd603aecf4d83b  go1.21.1.windows-386.zip
-10a4f5b63215d11d1770453733dbcbf024f3f74872f84e28d7ea59f0250316c6  go1.21.1.windows-amd64.zip
-41135ce6e0ced4bc1e459cb96bd4090c9dc2062e24179c3f337d855af9b560ef  go1.21.1.windows-arm64.zip
+186f2b6f8c8b704e696821b09ab2041a5c1ee13dcbc3156a13adcf75931ee488  go1.21.3.src.tar.gz
+27014fc69e301d7588a169ca239b3cc609f0aa1abf38528bf0d20d3b259211eb  go1.21.3.darwin-amd64.tar.gz
+65302a7a9f7a4834932b3a7a14cb8be51beddda757b567a2f9e0cbd0d7b5a6ab  go1.21.3.darwin-arm64.tar.gz
+8e0cd2f66cf1bde9d07b4aee01e3d7c3cfdd14e20650488e1683da4b8492594a  go1.21.3.freebsd-386.tar.gz
+6e74f65f586e93d1f3947894766f69e9b2ebda488592a09df61f36f06bfe58a8  go1.21.3.freebsd-amd64.tar.gz
+fb209fd070db500a84291c5a95251cceeb1723e8f6142de9baca5af70a927c0e  go1.21.3.linux-386.tar.gz
+1241381b2843fae5a9707eec1f8fb2ef94d827990582c7c7c32f5bdfbfd420c8  go1.21.3.linux-amd64.tar.gz
+fc90fa48ae97ba6368eecb914343590bbb61b388089510d0c56c2dde52987ef3  go1.21.3.linux-arm64.tar.gz
+a1ddcaaf0821a12a800884c14cb4268ce1c1f5a0301e9060646f1e15e611c6c7  go1.21.3.linux-armv6l.tar.gz
+3b0e10a3704f164a6e85e0377728ec5fd21524fabe4c925610e34076586d5826  go1.21.3.linux-ppc64le.tar.gz
+4c78e2e6f4c684a3d5a9bdc97202729053f44eb7be188206f0627ef3e18716b6  go1.21.3.linux-s390x.tar.gz
+e36737f4f2fadb4d2f919ec4ce517133a56e06064cca6e82fc883bb000c4d56c  go1.21.3.windows-386.zip
+27c8daf157493f288d42a6f38debc6a2cb391f6543139eba9152fceca0be2a10  go1.21.3.windows-amd64.zip
+bfb7a5c56f9ded07d8ae0e0b3702ac07b65e68fa8f33da24ed6df4ce01fe2c5c  go1.21.3.windows-arm64.zip
 
-# https://github.com/golangci/golangci-lint/releases
+# version:golangci 1.51.1
+# https://github.com/golangci/golangci-lint/releases/
+# https://github.com/golangci/golangci-lint/releases/download/v1.51.1/
 fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc  golangci-lint-1.51.1-darwin-amd64.tar.gz
 75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2  golangci-lint-1.51.1-darwin-arm64.tar.gz
 e06b3459aaed356e1667580be00b05f41f3b2e29685d12cdee571c23e1edb414  golangci-lint-1.51.1-freebsd-386.tar.gz
@@ -48,4 +53,12 @@ bce02f7232723cb727755ee11f168a700a00896a25d37f87c4b173bce55596b4  golangci-lint-
 cf6403f84707ce8c98664736772271bc8874f2e760c2fd0f00cf3e85963507e9  golangci-lint-1.51.1-windows-armv7.zip
 
 # This is the builder on PPA that will build Go itself (inception-y), don't modify!
+#
+# This version is fine to be old and full of security holes, we just use it
+# to build the latest Go. Don't change it. If it ever becomes insufficient,
+# we need to switch over to a recursive builder to jump across supported
+# versions.
+#
+# version:ppa-builder 1.19.6
+# https://go.dev/dl/
 d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767  go1.19.6.src.tar.gz

build/ci.go

@@ -136,23 +136,6 @@ var (
 		"golang-go":   "/usr/lib/go",
 	}
 
-	// This is the version of Go that will be downloaded by
-	//
-	//     go run ci.go install -dlgo
-	dlgoVersion = "1.21.1"
-
-	// This is the version of Go that will be used to bootstrap the PPA builder.
-	//
-	// This version is fine to be old and full of security holes, we just use it
-	// to build the latest Go. Don't change it. If it ever becomes insufficient,
-	// we need to switch over to a recursive builder to jumpt across supported
-	// versions.
-	gobootVersion = "1.19.6"
-
-	// This is the version of execution-spec-tests that we are using.
-	// When updating, you must also update build/checksums.txt.
-	executionSpecTestsVersion = "1.0.2"
-
 	// This is where the tests should be unpacked.
 	executionSpecTestsDir = "tests/spec-tests"
 )
@@ -192,6 +175,8 @@ func main() {
 		doWindowsInstaller(os.Args[2:])
 	case "purge":
 		doPurge(os.Args[2:])
+	case "sanitycheck":
+		doSanityCheck()
 	default:
 		log.Fatal("unknown command ", os.Args[1])
 	}
@@ -213,9 +198,8 @@ func doInstall(cmdline []string) {
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
 		csdb := build.MustLoadChecksums("build/checksums.txt")
-		tc.Root = build.DownloadGo(csdb, dlgoVersion)
+		tc.Root = build.DownloadGo(csdb)
 	}
-
 	// Disable CLI markdown doc generation in release builds.
 	buildTags := []string{"urfave_cli_no_docs"}
 
@@ -312,7 +296,7 @@ func doTest(cmdline []string) {
 	// Configure the toolchain.
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
-		tc.Root = build.DownloadGo(csdb, dlgoVersion)
+		tc.Root = build.DownloadGo(csdb)
 	}
 	gotest := tc.Go("test")
 
@@ -345,8 +329,12 @@ func doTest(cmdline []string) {
 
 // downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
 func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
+	executionSpecTestsVersion, err := build.Version(csdb, "spec-tests")
+	if err != nil {
+		log.Fatal(err)
+	}
 	ext := ".tar.gz"
-	base := "fixtures" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename
+	base := "fixtures_develop" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename
 	url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/v%s/%s%s", executionSpecTestsVersion, base, ext)
 	archivePath := filepath.Join(cachedir, base+ext)
 	if err := csdb.DownloadFile(url, archivePath); err != nil {
@@ -377,9 +365,11 @@ func doLint(cmdline []string) {
 
 // downloadLinter downloads and unpacks golangci-lint.
 func downloadLinter(cachedir string) string {
-	const version = "1.51.1"
-
 	csdb := build.MustLoadChecksums("build/checksums.txt")
+	version, err := build.Version(csdb, "golangci")
+	if err != nil {
+		log.Fatal(err)
+	}
 	arch := runtime.GOARCH
 	ext := ".tar.gz"
 
@@ -761,6 +751,10 @@ func doDebianSource(cmdline []string) {
 // to bootstrap the builder Go.
 func downloadGoBootstrapSources(cachedir string) string {
 	csdb := build.MustLoadChecksums("build/checksums.txt")
+	gobootVersion, err := build.Version(csdb, "ppa-builder")
+	if err != nil {
+		log.Fatal(err)
+	}
 	file := fmt.Sprintf("go%s.src.tar.gz", gobootVersion)
 	url := "https://dl.google.com/go/" + file
 	dst := filepath.Join(cachedir, file)
@@ -773,6 +767,10 @@ func downloadGoBootstrapSources(cachedir string) string {
 // downloadGoSources downloads the Go source tarball.
 func downloadGoSources(cachedir string) string {
 	csdb := build.MustLoadChecksums("build/checksums.txt")
+	dlgoVersion, err := build.Version(csdb, "golang")
+	if err != nil {
+		log.Fatal(err)
+	}
 	file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion)
 	url := "https://dl.google.com/go/" + file
 	dst := filepath.Join(cachedir, file)
@@ -1099,3 +1097,7 @@ func doPurge(cmdline []string) {
 		log.Fatal(err)
 	}
 }
+
+func doSanityCheck() {
+	build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt"))
+}

cmd/clef/main.go

@@ -1206,7 +1206,7 @@ func GenDoc(ctx *cli.Context) error {
 						URL:     accounts.URL{Path: ".. ignored .."},
 					},
 					{
-						Address: common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"),
+						Address: common.MaxAddress,
 					},
 				}})
 	}

cmd/devp2p/dns_cloudflare.go

@@ -114,7 +114,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 	records = lrecords
 
 	log.Info(fmt.Sprintf("Retrieving existing TXT records on %s", name))
-	entries, err := c.DNSRecords(context.Background(), c.zoneID, cloudflare.DNSRecord{Type: "TXT"})
+	entries, _, err := c.ListDNSRecords(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), cloudflare.ListDNSRecordsParams{Type: "TXT"})
 	if err != nil {
 		return err
 	}
@@ -141,14 +141,25 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 			if path != name {
 				ttl = treeNodeTTLCloudflare // Max TTL permitted by Cloudflare
 			}
-			record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl}
-			_, err = c.CreateDNSRecord(context.Background(), c.zoneID, record)
+			record := cloudflare.CreateDNSRecordParams{Type: "TXT", Name: path, Content: val, TTL: ttl}
+			_, err = c.CreateDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), record)
 		} else if old.Content != val {
 			// Entry already exists, only change its content.
 			log.Info(fmt.Sprintf("Updating %s from %q to %q", path, old.Content, val))
 			updated++
-			old.Content = val
-			err = c.UpdateDNSRecord(context.Background(), c.zoneID, old.ID, old)
+
+			record := cloudflare.UpdateDNSRecordParams{
+				Type:     old.Type,
+				Name:     old.Name,
+				Content:  val,
+				Data:     old.Data,
+				ID:       old.ID,
+				Priority: old.Priority,
+				TTL:      old.TTL,
+				Proxied:  old.Proxied,
+				Tags:     old.Tags,
+			}
+			_, err = c.UpdateDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), record)
 		} else {
 			skipped++
 			log.Debug(fmt.Sprintf("Skipping %s = %q", path, val))
@@ -168,7 +179,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 		// Stale entry, nuke it.
 		log.Debug(fmt.Sprintf("Deleting %s = %q", path, entry.Content))
 		deleted++
-		if err := c.DeleteDNSRecord(context.Background(), c.zoneID, entry.ID); err != nil {
+		if err := c.DeleteDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), entry.ID); err != nil {
 			return fmt.Errorf("failed to delete %s: %v", path, err)
 		}
 	}

cmd/devp2p/internal/ethtest/snap.go

@@ -58,7 +58,7 @@ type accRangeTest struct {
 func (s *Suite) TestSnapGetAccountRange(t *utesting.T) {
 	var (
 		root           = s.chain.RootAt(999)
-		ffHash         = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
+		ffHash         = common.MaxHash
 		zero           = common.Hash{}
 		firstKeyMinus1 = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf29")
 		firstKey       = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a")
@@ -125,7 +125,7 @@ type stRangesTest struct {
 // TestSnapGetStorageRanges various forms of GetStorageRanges requests.
 func (s *Suite) TestSnapGetStorageRanges(t *utesting.T) {
 	var (
-		ffHash    = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
+		ffHash    = common.MaxHash
 		zero      = common.Hash{}
 		firstKey  = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a")
 		secondKey = common.HexToHash("0x09e47cd5056a689e708f22fe1f932709a320518e444f5f7d8d46a3da523d6606")
@@ -536,11 +536,7 @@ func (s *Suite) snapGetAccountRange(t *utesting.T, tc *accRangeTest) error {
 	}
 	proofdb := nodes.Set()
 
-	var end []byte
-	if len(keys) > 0 {
-		end = keys[len(keys)-1]
-	}
-	_, err = trie.VerifyRangeProof(tc.root, tc.origin[:], end, keys, accounts, proofdb)
+	_, err = trie.VerifyRangeProof(tc.root, tc.origin[:], keys, accounts, proofdb)
 	return err
 }
 

cmd/evm/blockrunner.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"sort"
 
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/vm"
@@ -60,9 +61,16 @@ func blockTestCmd(ctx *cli.Context) error {
 	if err = json.Unmarshal(src, &tests); err != nil {
 		return err
 	}
-	for i, test := range tests {
+	// run them in order
+	var keys []string
+	for key := range tests {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, name := range keys {
+		test := tests[name]
 		if err := test.Run(false, rawdb.HashScheme, tracer); err != nil {
-			return fmt.Errorf("test %v: %w", i, err)
+			return fmt.Errorf("test %v: %w", name, err)
 		}
 	}
 	return nil

cmd/evm/testdata/8/readme.md

@@ -32,7 +32,7 @@ dir=./testdata/8 && ./evm t8n --state.fork=Berlin --input.alloc=$dir/alloc.json
 {"pc":4,"op":84,"gas":"0x48456","gasCost":"0x64","memSize":0,"stack":["0x3"],"depth":1,"refund":0,"opName":"SLOAD"}
 ```
 
-Simlarly, we can provide the input transactions via `stdin` instead of as file: 
+Similarly, we can provide the input transactions via `stdin` instead of as file:
 
 ```
 $ dir=./testdata/8 \

cmd/utils/flags.go

@@ -961,17 +961,12 @@ var (
 		DataDirFlag,
 		AncientFlag,
 		RemoteDBFlag,
+		DBEngineFlag,
 		StateSchemeFlag,
 		HttpHeaderFlag,
 	}
 )
 
-func init() {
-	if rawdb.PebbleEnabled {
-		DatabaseFlags = append(DatabaseFlags, DBEngineFlag)
-	}
-}
-
 // MakeDataDir retrieves the currently requested data directory, terminating
 // if none (or the empty string) is specified. If the node is starting a testnet,
 // then a subdirectory of the specified datadir will be used.

common/types.go

@@ -44,6 +44,12 @@ const (
 var (
 	hashT    = reflect.TypeOf(Hash{})
 	addressT = reflect.TypeOf(Address{})
+
+	// MaxAddress represents the maximum possible address value.
+	MaxAddress = HexToAddress("0xffffffffffffffffffffffffffffffffffffffff")
+
+	// MaxHash represents the maximum possible hash value.
+	MaxHash = HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
 )
 
 // Hash represents the 32 byte Keccak256 hash of arbitrary data.

core/rawdb/database.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethdb/leveldb"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/ethdb/pebble"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/olekukonko/tablewriter"
 )
@@ -321,6 +322,16 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r
 	return NewDatabase(db), nil
 }
 
+// NewPebbleDBDatabase creates a persistent key-value database without a freezer
+// moving immutable chain segments into cold storage.
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) {
+	db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral)
+	if err != nil {
+		return nil, err
+	}
+	return NewDatabase(db), nil
+}
+
 const (
 	dbPebble  = "pebble"
 	dbLeveldb = "leveldb"
@@ -375,26 +386,16 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
 		return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
 	}
 	if o.Type == dbPebble || existingDb == dbPebble {
-		if PebbleEnabled {
-			log.Info("Using pebble as the backing database")
-			return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
-		} else {
-			return nil, errors.New("db.engine 'pebble' not supported on this platform")
-		}
+		log.Info("Using pebble as the backing database")
+		return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
 	}
 	if o.Type == dbLeveldb || existingDb == dbLeveldb {
 		log.Info("Using leveldb as the backing database")
 		return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
 	}
-	// No pre-existing database, no user-requested one either. Default to Pebble
-	// on supported platforms and LevelDB on anything else.
-	if PebbleEnabled {
-		log.Info("Defaulting to pebble as the backing database")
-		return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
-	} else {
-		log.Info("Defaulting to leveldb as the backing database")
-		return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
-	}
+	// No pre-existing database, no user-requested one either. Default to Pebble.
+	log.Info("Defaulting to pebble as the backing database")
+	return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
 }
 
 // Open opens both a disk-based key-value database such as leveldb or pebble, but also