From 738621b45d31991c16703f3e7111b01ccabdbee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Sun, 24 Nov 2024 08:46:40 -0500 Subject: [PATCH] add flags for fixes, prep changelog --- CHANGELOG.md | 4 +++- block/fetcher/rpc.go | 18 +++++++++++------- cmd/firesol/rpc/check.go | 2 +- cmd/firesol/rpc/fetcher.go | 13 +++++++++++-- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e89171..1887b0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See [MAINTAINERS.md](./MAINTAINERS.md) for instructions to keep up to date. -## Unreleased +## v1.1.1 +* Fix "unable to fetch block was skipped and should not have been requested" but when an endpoint is a load balancer pointing to different nodes -- this removes an optimization that reduces the number of RPC calls by assuming that no new block will appear below the 'latest slot' of your node. If you are indeed pointing to only single solana nodes, you can use the new `--optimize-single-target` flag to re-enable this optimization. * Fix startup always looking for 'first block' instead of 'cursor block', failing unnecessarily on non-archive nodes +* Add --network flag (default: mainnet) -- this flag is used only to enable a special fix around block 13334464, which you don't want to skip on testnet or devnet. ## v1.1.0 diff --git a/block/fetcher/rpc.go b/block/fetcher/rpc.go index ba9fa11..f137c85 100644 --- a/block/fetcher/rpc.go +++ b/block/fetcher/rpc.go @@ -36,19 +36,23 @@ type fetchBlock func(ctx context.Context, requestedSlot uint64) (slot uint64, ou type RPCFetcher struct { rpcClients *firecoreRPC.Clients[*rpc.Client] + optimizeForSingleTarget bool latestConfirmedSlot uint64 latestFinalizedSlot uint64 latestBlockRetryInterval time.Duration fetchInterval time.Duration lastFetchAt time.Time + isMainnet bool logger *zap.Logger } -func NewRPC(rpcClients *firecoreRPC.Clients[*rpc.Client], fetchInterval time.Duration, latestBlockRetryInterval time.Duration, logger *zap.Logger) *RPCFetcher { +func NewRPC(rpcClients *firecoreRPC.Clients[*rpc.Client], fetchInterval time.Duration, latestBlockRetryInterval time.Duration, optimizeForSingleTarget bool, isMainnet bool, logger *zap.Logger) *RPCFetcher { f := &RPCFetcher{ rpcClients: rpcClients, fetchInterval: fetchInterval, + optimizeForSingleTarget: optimizeForSingleTarget, latestBlockRetryInterval: latestBlockRetryInterval, + isMainnet: isMainnet, logger: logger, } return f @@ -60,8 +64,8 @@ func (f *RPCFetcher) IsBlockAvailable(requestedSlot uint64) bool { } func (f *RPCFetcher) Fetch(ctx context.Context, requestedSlot uint64) (out *pbbstream.Block, skip bool, err error) { - //THIS IS A FKG Ugly hack! - if requestedSlot >= 13334464 && requestedSlot <= 13334475 { + if f.isMainnet && requestedSlot >= 13334464 && requestedSlot <= 13334475 { + // know issue fetching these blocks on mainnet, ugly but works return nil, true, nil } @@ -143,10 +147,10 @@ func (f *RPCFetcher) fetch(ctx context.Context, requestedSlot uint64, lastConfir } if rpcErr.Code == -32004 { - //if currentSlot < lastConfirmBlockNum { - // f.logger.Info("fetcher block was supposedly skipped", zap.Uint64("block_num", currentSlot)) - // return nil, true, nil - //} + if f.optimizeForSingleTarget && currentSlot < lastConfirmBlockNum { + f.logger.Info("fetcher block was supposedly skipped", zap.Uint64("block_num", currentSlot)) + return nil, true, nil + } f.logger.Warn("block not available. trying same block", zap.Uint64("block_num", currentSlot)) continue diff --git a/cmd/firesol/rpc/check.go b/cmd/firesol/rpc/check.go index ccc0b31..0a1788c 100644 --- a/cmd/firesol/rpc/check.go +++ b/cmd/firesol/rpc/check.go @@ -48,7 +48,7 @@ func nextBlockRunE(logger *zap.Logger, _ logging.Tracer) firecore.CommandExecuto rpcClients.Add(client) } - rpcFetcher := fetcher.NewRPC(rpcClients, 1*time.Second, 1*time.Second, logger) + rpcFetcher := fetcher.NewRPC(rpcClients, 1*time.Second, 1*time.Second, false, false, logger) var blockExists bool for !blockExists { diff --git a/cmd/firesol/rpc/fetcher.go b/cmd/firesol/rpc/fetcher.go index add7918..2ac2b8e 100644 --- a/cmd/firesol/rpc/fetcher.go +++ b/cmd/firesol/rpc/fetcher.go @@ -2,10 +2,11 @@ package rpc import ( "fmt" - firecoreRPC "github.com/streamingfast/firehose-core/rpc" "strconv" "time" + firecoreRPC "github.com/streamingfast/firehose-core/rpc" + "github.com/gagliardetto/solana-go/rpc" "github.com/spf13/cobra" "github.com/streamingfast/cli/sflags" @@ -29,6 +30,8 @@ func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch") cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch") cmd.Flags().Int("block-fetch-batch-size", 10, "Number of blocks to fetch in a single batch") + cmd.Flags().Bool("optimize-single-target", false, "Only set this if every endpoint is pointing to a single node (not a cluster). It allows reducing the number of RPC calls by making assumptions about the last block") + cmd.Flags().String("network", "mainnet", "network to fetch from (mainnet, devnet, testnet) -- only used to patch a known issue on some slots") return cmd } @@ -62,9 +65,15 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut } latestBlockRetryInterval := sflags.MustGetDuration(cmd, "latest-block-retry-interval") + optimizeSingleTarget := sflags.MustGetBool(cmd, "optimize-single-target") + var isMainnet bool + switch sflags.MustGetString(cmd, "network") { + case "mainnet", "mainnet-beta": + isMainnet = true + } poller := blockpoller.New( - fetcher.NewRPC(rpcClients, fetchInterval, latestBlockRetryInterval, logger), + fetcher.NewRPC(rpcClients, fetchInterval, latestBlockRetryInterval, optimizeSingleTarget, isMainnet, logger), blockpoller.NewFireBlockHandler("type.googleapis.com/sf.solana.type.v1.Block"), blockpoller.WithStoringState(stateDir), blockpoller.WithLogger(logger),