bump substreams: --substreams-block-execution-timeout and module hash…

… computation fix

CHANGELOG.md

@@ -8,6 +8,11 @@ Operators, you should copy/paste content of this content straight to your projec
 
 If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you should copy the content between those 2 version to your own repository, replacing placeholder value `fire{chain}` with your chain's own binary.
 
+## Unreleased
+
+* Substreams: revert module hash calculation from `v1.5.5`, when using a non-zero firstStreamableBlock. Hashes will now be the same even if the chain's first streamable block affects the initialBlock of a module.
+* Substreams: add `--substreams-block-execution-timeout` flag (default 3 minutes) to prevent requests stalling
+
 ## v1.5.7
 
 * Bump substreams to v1.9.3: fix high CPU usage on tier1 caused by a bad error handling

cmd/apps/substreams_common.go

@@ -2,6 +2,7 @@ package apps
 
 import (
 	"sync"
+	"time"
 
 	"github.com/spf13/cobra"
 )
@@ -13,5 +14,6 @@ func registerCommonSubstreamsFlags(cmd *cobra.Command) {
 		cmd.Flags().Uint64("substreams-state-bundle-size", uint64(1_000), "Interval in blocks at which to save store snapshots and output caches")
 		cmd.Flags().String("substreams-state-store-url", "{sf-data-dir}/localdata", "where substreams state data are stored")
 		cmd.Flags().String("substreams-state-store-default-tag", "", "If non-empty, will be appended to {substreams-state-store-url} (ex: 'v1'). Can be overriden per-request with 'X-Sf-Substreams-Cache-Tag' header")
+		cmd.Flags().Duration("substreams-block-execution-timeout", 3*time.Minute, "Maximum execution time for a block before the request is canceled")
 	})
 }

cmd/apps/substreams_tier1.go

@@ -77,6 +77,7 @@ func RegisterSubstreamsTier1App[B firecore.Block](chain *firecore.Chain[B], root
 
 			stateStoreURL := firecore.MustReplaceDataDir(sfDataDir, viper.GetString("substreams-state-store-url"))
 			stateStoreDefaultTag := viper.GetString("substreams-state-store-default-tag")
+			executionTimeout := viper.GetDuration("substreams-block-execution-timeout")
 
 			stateBundleSize := viper.GetUint64("substreams-state-bundle-size")
 
@@ -128,15 +129,16 @@ func RegisterSubstreamsTier1App[B firecore.Block](chain *firecore.Chain[B], root
 					ForkedBlocksStoreURL: forkedBlocksStoreURL,
 					BlockStreamAddr:      blockstreamAddr,
 
-					StateStoreURL:        stateStoreURL,
-					StateStoreDefaultTag: stateStoreDefaultTag,
-					StateBundleSize:      stateBundleSize,
-					MaxSubrequests:       maxSubrequests,
-					SubrequestsEndpoint:  subrequestsEndpoint,
-					SubrequestsInsecure:  subrequestsInsecure,
-					SubrequestsPlaintext: subrequestsPlaintext,
-					BlockType:            blockType,
-					WASMExtensions:       wasmExtensions,
+					StateStoreURL:         stateStoreURL,
+					StateStoreDefaultTag:  stateStoreDefaultTag,
+					StateBundleSize:       stateBundleSize,
+					MaxSubrequests:        maxSubrequests,
+					SubrequestsEndpoint:   subrequestsEndpoint,
+					SubrequestsInsecure:   subrequestsInsecure,
+					SubrequestsPlaintext:  subrequestsPlaintext,
+					BlockType:             blockType,
+					WASMExtensions:        wasmExtensions,
+					BlockExecutionTimeout: executionTimeout,
 
 					Tracing: tracing,
 

cmd/apps/substreams_tier2.go

@@ -55,6 +55,7 @@ func RegisterSubstreamsTier2App[B firecore.Block](chain *firecore.Chain[B], root
 			grpcListenAddr := viper.GetString("substreams-tier2-grpc-listen-addr")
 
 			maximumConcurrentRequests := viper.GetUint64("substreams-tier2-max-concurrent-requests")
+			executionTimeout := viper.GetDuration("substreams-block-execution-timeout")
 
 			tracing := os.Getenv("SUBSTREAMS_TRACING") == "modules_exec"
 
@@ -85,9 +86,10 @@ func RegisterSubstreamsTier2App[B firecore.Block](chain *firecore.Chain[B], root
 				&app.Tier2Config{
 					Tracing: tracing,
 
-					GRPCListenAddr:      grpcListenAddr,
-					ServiceDiscoveryURL: serviceDiscoveryURL,
-					WASMExtensions:      wasmExtensions,
+					GRPCListenAddr:        grpcListenAddr,
+					ServiceDiscoveryURL:   serviceDiscoveryURL,
+					WASMExtensions:        wasmExtensions,
+					BlockExecutionTimeout: executionTimeout,
 
 					MaximumConcurrentRequests: maximumConcurrentRequests,
 				}, &app.Tier2Modules{

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2
 	github.com/streamingfast/pbgo v0.0.6-0.20240430190514-722fe9d82e5d
 	github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0
-	github.com/streamingfast/substreams v1.9.3
+	github.com/streamingfast/substreams v1.9.4-0.20240731180302-278777b83985
 	github.com/stretchr/testify v1.8.4
 	github.com/test-go/testify v1.1.4
 	go.uber.org/multierr v1.10.0

go.sum

@@ -582,8 +582,8 @@ github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAt
 github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8=
 github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 h1:Y15G1Z4fpEdm2b+/70owI7TLuXadlqBtGM7rk4Hxrzk=
 github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0/go.mod h1:/Rnz2TJvaShjUct0scZ9kKV2Jr9/+KBAoWy4UMYxgv4=
-github.com/streamingfast/substreams v1.9.3 h1:3UzSelyAu0tblkZocw5/Wm5FmawwjV+k6qDb77Baj48=
-github.com/streamingfast/substreams v1.9.3/go.mod h1:yPSIRwtAl9/1dB45j82ZsGe8lndRdN8oKy4k/WvHaao=
+github.com/streamingfast/substreams v1.9.4-0.20240731180302-278777b83985 h1:Tcxeb9RyLmgZUybFEdopzt2ZVkncMsTuO0V1zFueaiQ=
+github.com/streamingfast/substreams v1.9.4-0.20240731180302-278777b83985/go.mod h1:yPSIRwtAl9/1dB45j82ZsGe8lndRdN8oKy4k/WvHaao=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=