Merge pull request #9 from hemilabs/clayton/heminetwork-update

heminetwork update

cmd/geth/main.go

@@ -18,7 +18,6 @@
 package main
 
 import (
-	"encoding/binary"
 	"fmt"
 	"os"
 	"sort"
@@ -167,6 +166,7 @@ var (
 		utils.TBCNetwork,
 		utils.TBCPrometheusAddress,
 		utils.TBCInitHeight,
+		utils.TBCSeeds,
 		configFileFlag,
 		utils.LogDebugFlag,
 		utils.LogBacktraceAtFlag,
@@ -405,70 +405,62 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon
 	if ctx.IsSet(utils.TBCPrometheusAddress.Name) {
 		tbcCfg.PrometheusListenAddress = ctx.String(utils.TBCPrometheusAddress.Name)
 	}
+	if ctx.IsSet(utils.TBCSeeds.Name) {
+		tbcCfg.Seeds = ctx.StringSlice(utils.TBCSeeds.Name)
+	}
 	// TODO: convert op-geth log level integer to TBC log level string
 
 	// Initialize TBC Bitcoin indexer to answer hVM queries
-	err := vm.SetupTBC(ctx.Context, tbcCfg)
+	if err := vm.SetupTBC(ctx.Context, tbcCfg); err != nil {
+		log.Crit(fmt.Sprintf("could not SetupTBC: %v", err))
+	}
 
 	// TODO: Review, give TBC time to warm up
 	time.Sleep(5 * time.Second)
 
-	firstStartup := false
-
-	utxoHeight, err := vm.TBCIndexer.DB().MetadataGet(ctx.Context, tbc.UtxoIndexHeightKey)
-	if err != nil {
-		log.Info("Unable to get UTXO height key from database, assuming first startup.")
-		firstStartup = true
-	}
-
-	txHeight, err := vm.TBCIndexer.DB().MetadataGet(ctx.Context, tbc.TxIndexHeightKey)
-	if err != nil {
-		log.Info("Unable to get Tx height key from database, assuming first startup.")
-		firstStartup = true
-	}
-
-	if !firstStartup {
-		log.Info("On op-geth startup, TBC index status: ", "utxoIndexHeight",
-			binary.BigEndian.Uint64(utxoHeight), "txIndexHeight", binary.BigEndian.Uint64(txHeight))
-	}
-
 	var initHeight uint64 = uint64(defaultTbcInitHeight)
 	if ctx.IsSet(utils.TBCInitHeight.Name) {
 		initHeight = ctx.Uint64(utils.TBCInitHeight.Name)
 	}
 
-	if firstStartup {
-		for {
-			log.Info(fmt.Sprintf("TBC has not downloaded the BTC chain up to %d yet."+
-				" Cannot progress Hemi chain until download is complete.", initHeight))
-			time.Sleep(5 * time.Second)
-			if vm.TBCBlocksAvailableToHeight(ctx.Context, 0, initHeight) {
-				log.Info("TBC Initial syncing is complete, continuing...")
-				break
-			} else {
-				log.Info("Geth still waiting for TBC initial sync", "initHeight", initHeight)
-			}
+	var syncInfo tbc.SyncInfo
+
+	for {
+		_, bhb, err := vm.TBCIndexer.BlockHeaderBest(ctx.Context)
+		if err != nil {
+			log.Crit(fmt.Sprintf("could not get BlockHeaderBest: %v", err))
 		}
 
-		err = vm.TBCIndexer.SyncIndexersToHeight(ctx.Context, initHeight)
+		bestHash := bhb.BlockHash()
 
-		log.Info("Finished initial indexing", "initHeight", initHeight)
-	}
+		if err := vm.TBCIndexer.SyncIndexersToHash(ctx.Context, &bestHash); err != nil {
+			log.Crit(fmt.Sprintf("could not SyncIndexersToHash: %v", err))
+		}
 
-	si := vm.TBCIndexer.Synced(ctx.Context)
+		if err := vm.TBCIndexer.TxIndexer(ctx.Context, &bestHash); err != nil {
+			log.Crit(fmt.Sprintf("could not TxIndexer: %v", err))
+		}
 
-	if si.UtxoHeight < initHeight {
-		log.Crit("TBC did not index UTXOs to initHeight!",
-			"utxoIndexHeight", si.UtxoHeight, "initHeight", initHeight)
-	}
+		if err := vm.TBCIndexer.UtxoIndexer(ctx.Context, &bestHash); err != nil {
+			log.Crit(fmt.Sprintf("could not UTXOIndexer: %v", err))
+		}
+
+		syncInfo = vm.TBCIndexer.Synced(ctx.Context)
 
-	if si.TxHeight < initHeight {
-		log.Crit("TBC did not index txs to initHeight!",
-			"txIndexHeight", si.TxHeight, "initHeight", initHeight)
+		log.Info(fmt.Sprintf("synced block headers to height %d, want to get to %d", syncInfo.BlockHeader.Height, initHeight))
+		if syncInfo.BlockHeader.Height >= initHeight {
+			break
+		}
+
+		select {
+		case <-time.After(500 * time.Millisecond):
+		case <-ctx.Context.Done():
+			log.Crit("context done")
+		}
 	}
 
-	log.Info("TBC initial sync completed", "headerHeight", si.BlockHeaderHeight,
-		"utxoIndexHeight", si.UtxoHeight, "txIndexHeight", si.TxHeight)
+	log.Info("TBC initial sync completed", "headerHeight", syncInfo.BlockHeader.Height,
+		"utxoIndexHeight", syncInfo.Utxo.Height, "txIndexHeight", syncInfo.Tx.Height)
 
 	vm.SetInitReady()
 

cmd/utils/flags.go

@@ -939,6 +939,12 @@ var (
 		Category: flags.RollupCategory,
 		Value:    2585811,
 	}
+	TBCSeeds = &cli.StringSliceFlag{
+		Name:     "tbc.seeds",
+		Usage:    "override tbc seeds when finding peers",
+		Category: flags.RollupCategory,
+		Value:    nil,
+	}
 
 	// Metrics flags
 	MetricsEnabledFlag = &cli.BoolFlag{

core/vm/contracts.go

@@ -37,7 +37,6 @@ import (
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/hemilabs/heminetwork/database"
 	"github.com/hemilabs/heminetwork/database/tbcd"
 	"github.com/hemilabs/heminetwork/service/tbc"
 	"golang.org/x/crypto/ripemd160"
@@ -86,23 +85,20 @@ func ProgressTip(ctx context.Context, currentTimestamp uint32) {
 	}
 
 	si := TBCIndexer.Synced(context.Background())
-	uh := si.UtxoHeight
-	th := si.TxHeight
+	uh := si.Utxo.Height
+	th := si.Tx.Height
 	log.Info("Checking for TBC progression available", "utxoIndexHeight", uh, "txIndexHeight", th)
 
 	if uh != th {
 		log.Crit("TBC is in an unexpected state, utxoIndexHeight != txIndexHeight!",
 			"utxoIndexHeight", uh, "txIndexHeight", th)
 	}
 
-	tipHeight, headers, err := TBCIndexer.BlockHeadersBest(ctx)
+	tipHeight, _, err := TBCIndexer.BlockHeaderBest(ctx)
 	if err != nil {
 		log.Crit("Unable to retrieve tip headers from TBC", "err", err)
 	}
 	log.Info(fmt.Sprintf("TBC download status: tipHeight=%d", tipHeight))
-	if len(headers) == 0 {
-		log.Crit("TBC has 0 headers at tip height", "tipHeight", tipHeight)
-	}
 
 	// Tip height is more than 2 blocks ahead, TBC can progress as far as timestamp is sufficiently past
 	if tipHeight > uh+TBCTipHeightLag {
@@ -129,96 +125,29 @@ func ProgressTip(ctx context.Context, currentTimestamp uint32) {
 				return
 			}
 
-			TBCIndexer.SyncIndexersToHeight(ctx, endingHeight)
-
-			done := TBCIndexer.Synced(ctx)
-			if done.UtxoHeight != endingHeight {
-				log.Crit(fmt.Sprintf("After indexing to block %d, UtxoHeight=%d!", endingHeight, done.UtxoHeight))
-			}
-			if done.TxHeight != endingHeight {
-				log.Crit(fmt.Sprintf("After indexing to block %d, TxHeight=%d!", endingHeight, done.TxHeight))
+			headers, err := TBCIndexer.BlockHeadersByHeight(ctx, endingHeight)
+			if err != nil {
+				log.Crit(fmt.Sprintf("could not get BlockHeadersByHeight %v", err))
 			}
-			log.Info("TBC progression done!", "utxoHeight", done.UtxoHeight, "txHeight", done.TxHeight)
-		}
-	}
-}
 
-func TBCIndexTxs(ctx context.Context, tipHeight uint64) error {
-	var h uint64
-	firstSync := false
-	// Get current indexed height
-	he, err := TBCIndexer.DB().MetadataGet(ctx, tbc.TxIndexHeightKey)
-	if err != nil {
-		if errors.Is(err, database.ErrNotFound) {
-			log.Info("No Tx indexing performed yet, starting height for Tx indexing set to 0.")
-			firstSync = true
-		} else {
-			// Error was something other than key not being found
-			return fmt.Errorf("error querying for Tx index metadata %v: %w",
-				string(tbc.UtxoIndexHeightKey), err)
-		}
-		he = make([]byte, 8)
-	}
-	h = binary.BigEndian.Uint64(he)
-	log.Info(fmt.Sprintf("TBC has indexed Txs to height %d", h))
+			if len(headers) != 1 {
+				log.Crit(fmt.Sprintf("received unexpected headers length %d", len(headers)))
+			}
 
-	if tipHeight <= h {
-		// Already indexed past this point
-		// TODO: decide whether to always check correct index height in upstream logic or throw error here
-		return nil
-	}
+			hash := headers[0].BlockHash()
 
-	count := tipHeight - h
-	if firstSync {
-		count++ // Genesis block also needs to be indexed, leave h=0
-	} else {
-		h++ // Start indexing at current indexed height + 1
-	}
-	log.Info(fmt.Sprintf("Indexing Txs starting at block %d, count %d", h, count))
-	err = TBCIndexer.TxIndexer(ctx, h, count)
-	if err != nil {
-		return fmt.Errorf("tx indexer error: %w", err)
-	}
-	return nil
-}
+			TBCIndexer.SyncIndexersToHash(ctx, &hash)
 
-func TBCIndexUTXOs(ctx context.Context, tipHeight uint64) error {
-	var h uint64
-	firstSync := false
-	// Get current indexed height
-	he, err := TBCIndexer.DB().MetadataGet(ctx, tbc.UtxoIndexHeightKey)
-	if err != nil {
-		if errors.Is(err, database.ErrNotFound) {
-			log.Info("No UTXO indexing performed yet, starting height for UTXO indexing set to 0.")
-			firstSync = true
-		} else {
-			// Error was something other than key not being found
-			return fmt.Errorf("error querying for UTXO Index metadata %v: %w",
-				string(tbc.UtxoIndexHeightKey), err)
+			done := TBCIndexer.Synced(ctx)
+			if done.Utxo.Height != endingHeight {
+				log.Crit(fmt.Sprintf("After indexing to block %d, UtxoHeight=%d!", endingHeight, done.Utxo.Height))
+			}
+			if done.Tx.Height != endingHeight {
+				log.Crit(fmt.Sprintf("After indexing to block %d, TxHeight=%d!", endingHeight, done.Tx.Height))
+			}
+			log.Info("TBC progression done!", "utxoHeight", done.Utxo.Height, "txHeight", done.Tx.Height)
 		}
-		he = make([]byte, 8)
-	}
-	h = binary.BigEndian.Uint64(he)
-	log.Info(fmt.Sprintf("TBC has indexed UTXOs to height %d", h))
-
-	if tipHeight <= h {
-		// Already indexed past this point
-		// TODO: decide whether to always check correct index height in upstream logic or throw error here
-		return nil
 	}
-
-	count := tipHeight - h
-	if firstSync {
-		count++ // Genesis block also needs to be indexed, leave h=0
-	} else {
-		h++ // Start indexing at current indexed height + 1
-	}
-	log.Info(fmt.Sprintf("Indexing UTXOs starting at block %d, count %d", h, count))
-	err = TBCIndexer.UtxoIndexer(ctx, h, count)
-	if err != nil {
-		return fmt.Errorf("UTXO indexer error: %w", err)
-	}
-	return nil
 }
 
 func TBCBlocksAvailableToHeight(ctx context.Context, startingHeight uint64, endingHeight uint64) bool {
@@ -630,18 +559,15 @@ func (c *btcLastHeader) Run(input []byte, blockContext common.Hash) ([]byte, err
 		}
 	}
 
-	height, headers, err := TBCIndexer.BlockHeadersBest(context.Background())
+	height, bestHeader, err := TBCIndexer.BlockHeaderBest(context.Background())
 
-	if err != nil || len(headers) == 0 {
+	if err != nil {
 		log.Warn("Unable to lookup best header!")
 		resp := make([]byte, 0)
 		hvmQueryMap[k] = resp
 		return resp, nil
 	}
 
-	// TODO: Canonical check
-	bestHeader := headers[0]
-
 	hash := bestHeader.BlockHash()
 	prevHash := bestHeader.PrevBlock
 	merkle := bestHeader.MerkleRoot

go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/gorilla/websocket v1.5.0
 	github.com/graph-gophers/graphql-go v1.3.0
 	github.com/hashicorp/go-bexpr v0.1.10
-	github.com/hemilabs/heminetwork v0.1.3-0.20240606145309-bbeed8bacbe5
+	github.com/hemilabs/heminetwork v0.2.0
 	github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7
 	github.com/holiman/bloomfilter/v2 v2.0.3
 	github.com/holiman/uint256 v1.2.4
@@ -152,7 +152,7 @@ require (
 	golang.org/x/net v0.21.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	nhooyr.io/websocket v1.8.10 // indirect
+	nhooyr.io/websocket v1.8.11 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 

go.sum

@@ -314,8 +314,8 @@ github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZn
 github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hemilabs/heminetwork v0.1.3-0.20240606145309-bbeed8bacbe5 h1:+iRHzXDPPRgmHORU2qAwg5kqKN7AhJsY1glW5rsMF6U=
-github.com/hemilabs/heminetwork v0.1.3-0.20240606145309-bbeed8bacbe5/go.mod h1:96jUxG9rPO+iwOyGg84VkRvejDgqoozFoEF6za7/vzw=
+github.com/hemilabs/heminetwork v0.2.0 h1:aUSIPpti6WvBkVcpTnZCXhu/Av/yCFlSeMw5sBPIqMc=
+github.com/hemilabs/heminetwork v0.2.0/go.mod h1:Te/IuHkkV4hyAzmweCTbNu/7piMeFIIIojbRV2iJ034=
 github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw=
 github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc=
 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
@@ -768,7 +768,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q=
-nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
+nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0=
+nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
 rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
 rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=