add flags for fixes, prep changelog

CHANGELOG.md

@@ -4,9 +4,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See [MAINTAINERS.md](./MAINTAINERS.md)
 for instructions to keep up to date.
 
-## Unreleased
+## v1.1.1
 
+* Fix "unable to fetch block was skipped and should not have been requested" but when an endpoint is a load balancer pointing to different nodes -- this removes an optimization that reduces the number of RPC calls by assuming that no new block will appear below the 'latest slot' of your node. If you are indeed pointing to only single solana nodes, you can use the new `--optimize-single-target` flag to re-enable this optimization.
 * Fix startup always looking for 'first block' instead of 'cursor block', failing unnecessarily on non-archive nodes
+* Add --network flag (default: mainnet) -- this flag is used only to enable a special fix around block 13334464, which you don't want to skip on testnet or devnet.
 
 ## v1.1.0
 

block/fetcher/rpc.go

@@ -36,19 +36,23 @@ type fetchBlock func(ctx context.Context, requestedSlot uint64) (slot uint64, ou
 
 type RPCFetcher struct {
 	rpcClients               *firecoreRPC.Clients[*rpc.Client]
+	optimizeForSingleTarget  bool
 	latestConfirmedSlot      uint64
 	latestFinalizedSlot      uint64
 	latestBlockRetryInterval time.Duration
 	fetchInterval            time.Duration
 	lastFetchAt              time.Time
+	isMainnet                bool
 	logger                   *zap.Logger
 }
 
-func NewRPC(rpcClients *firecoreRPC.Clients[*rpc.Client], fetchInterval time.Duration, latestBlockRetryInterval time.Duration, logger *zap.Logger) *RPCFetcher {
+func NewRPC(rpcClients *firecoreRPC.Clients[*rpc.Client], fetchInterval time.Duration, latestBlockRetryInterval time.Duration, optimizeForSingleTarget bool, isMainnet bool, logger *zap.Logger) *RPCFetcher {
 	f := &RPCFetcher{
 		rpcClients:               rpcClients,
 		fetchInterval:            fetchInterval,
+		optimizeForSingleTarget:  optimizeForSingleTarget,
 		latestBlockRetryInterval: latestBlockRetryInterval,
+		isMainnet:                isMainnet,
 		logger:                   logger,
 	}
 	return f
@@ -60,8 +64,8 @@ func (f *RPCFetcher) IsBlockAvailable(requestedSlot uint64) bool {
 }
 
 func (f *RPCFetcher) Fetch(ctx context.Context, requestedSlot uint64) (out *pbbstream.Block, skip bool, err error) {
-	//THIS IS A FKG Ugly hack!
-	if requestedSlot >= 13334464 && requestedSlot <= 13334475 {
+	if f.isMainnet && requestedSlot >= 13334464 && requestedSlot <= 13334475 {
+		// know issue fetching these blocks on mainnet, ugly but works
 		return nil, true, nil
 	}
 
@@ -143,10 +147,10 @@ func (f *RPCFetcher) fetch(ctx context.Context, requestedSlot uint64, lastConfir
 				}
 
 				if rpcErr.Code == -32004 {
-					//if currentSlot < lastConfirmBlockNum {
-					//	f.logger.Info("fetcher block was supposedly skipped", zap.Uint64("block_num", currentSlot))
-					//	return nil, true, nil
-					//}
+					if f.optimizeForSingleTarget && currentSlot < lastConfirmBlockNum {
+						f.logger.Info("fetcher block was supposedly skipped", zap.Uint64("block_num", currentSlot))
+						return nil, true, nil
+					}
 
 					f.logger.Warn("block not available. trying same block", zap.Uint64("block_num", currentSlot))
 					continue

cmd/firesol/rpc/check.go

@@ -48,7 +48,7 @@ func nextBlockRunE(logger *zap.Logger, _ logging.Tracer) firecore.CommandExecuto
 			rpcClients.Add(client)
 		}
 
-		rpcFetcher := fetcher.NewRPC(rpcClients, 1*time.Second, 1*time.Second, logger)
+		rpcFetcher := fetcher.NewRPC(rpcClients, 1*time.Second, 1*time.Second, false, false, logger)
 
 		var blockExists bool
 		for !blockExists {

cmd/firesol/rpc/fetcher.go

@@ -2,10 +2,11 @@ package rpc
 
 import (
 	"fmt"
-	firecoreRPC "github.com/streamingfast/firehose-core/rpc"
 	"strconv"
 	"time"
 
+	firecoreRPC "github.com/streamingfast/firehose-core/rpc"
+
 	"github.com/gagliardetto/solana-go/rpc"
 	"github.com/spf13/cobra"
 	"github.com/streamingfast/cli/sflags"
@@ -29,6 +30,8 @@ func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command {
 	cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch")
 	cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch")
 	cmd.Flags().Int("block-fetch-batch-size", 10, "Number of blocks to fetch in a single batch")
+	cmd.Flags().Bool("optimize-single-target", false, "Only set this if every endpoint is pointing to a single node (not a cluster). It allows reducing the number of RPC calls by making assumptions about the last block")
+	cmd.Flags().String("network", "mainnet", "network to fetch from (mainnet, devnet, testnet) -- only used to patch a known issue on some slots")
 
 	return cmd
 }
@@ -62,9 +65,15 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut
 		}
 
 		latestBlockRetryInterval := sflags.MustGetDuration(cmd, "latest-block-retry-interval")
+		optimizeSingleTarget := sflags.MustGetBool(cmd, "optimize-single-target")
+		var isMainnet bool
+		switch sflags.MustGetString(cmd, "network") {
+		case "mainnet", "mainnet-beta":
+			isMainnet = true
+		}
 
 		poller := blockpoller.New(
-			fetcher.NewRPC(rpcClients, fetchInterval, latestBlockRetryInterval, logger),
+			fetcher.NewRPC(rpcClients, fetchInterval, latestBlockRetryInterval, optimizeSingleTarget, isMainnet, logger),
 			blockpoller.NewFireBlockHandler("type.googleapis.com/sf.solana.type.v1.Block"),
 			blockpoller.WithStoringState(stateDir),
 			blockpoller.WithLogger(logger),