diff --git a/CHANGELOG.md b/CHANGELOG.md index 4665379..1eb4f6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ Operators, you should copy/paste content of this content straight to your projec If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you should copy the content between those 2 version to your own repository, replacing placeholder value `fire{chain}` with your chain's own binary. +## Unreleased + +* Substreams: revert module hash calculation from `v1.5.5`, when using a non-zero firstStreamableBlock. Hashes will now be the same even if the chain's first streamable block affects the initialBlock of a module. +* Substreams: add `--substreams-block-execution-timeout` flag (default 3 minutes) to prevent requests stalling + ## v1.5.7 * Bump substreams to v1.9.3: fix high CPU usage on tier1 caused by a bad error handling diff --git a/cmd/apps/substreams_common.go b/cmd/apps/substreams_common.go index 2f02f3d..0da9155 100644 --- a/cmd/apps/substreams_common.go +++ b/cmd/apps/substreams_common.go @@ -2,6 +2,7 @@ package apps import ( "sync" + "time" "github.com/spf13/cobra" ) @@ -13,5 +14,6 @@ func registerCommonSubstreamsFlags(cmd *cobra.Command) { cmd.Flags().Uint64("substreams-state-bundle-size", uint64(1_000), "Interval in blocks at which to save store snapshots and output caches") cmd.Flags().String("substreams-state-store-url", "{sf-data-dir}/localdata", "where substreams state data are stored") cmd.Flags().String("substreams-state-store-default-tag", "", "If non-empty, will be appended to {substreams-state-store-url} (ex: 'v1'). Can be overriden per-request with 'X-Sf-Substreams-Cache-Tag' header") + cmd.Flags().Duration("substreams-block-execution-timeout", 3*time.Minute, "Maximum execution time for a block before the request is canceled") }) } diff --git a/cmd/apps/substreams_tier1.go b/cmd/apps/substreams_tier1.go index 393fd3e..0044fb3 100644 --- a/cmd/apps/substreams_tier1.go +++ b/cmd/apps/substreams_tier1.go @@ -77,6 +77,7 @@ func RegisterSubstreamsTier1App[B firecore.Block](chain *firecore.Chain[B], root stateStoreURL := firecore.MustReplaceDataDir(sfDataDir, viper.GetString("substreams-state-store-url")) stateStoreDefaultTag := viper.GetString("substreams-state-store-default-tag") + executionTimeout := viper.GetDuration("substreams-block-execution-timeout") stateBundleSize := viper.GetUint64("substreams-state-bundle-size") @@ -128,15 +129,16 @@ func RegisterSubstreamsTier1App[B firecore.Block](chain *firecore.Chain[B], root ForkedBlocksStoreURL: forkedBlocksStoreURL, BlockStreamAddr: blockstreamAddr, - StateStoreURL: stateStoreURL, - StateStoreDefaultTag: stateStoreDefaultTag, - StateBundleSize: stateBundleSize, - MaxSubrequests: maxSubrequests, - SubrequestsEndpoint: subrequestsEndpoint, - SubrequestsInsecure: subrequestsInsecure, - SubrequestsPlaintext: subrequestsPlaintext, - BlockType: blockType, - WASMExtensions: wasmExtensions, + StateStoreURL: stateStoreURL, + StateStoreDefaultTag: stateStoreDefaultTag, + StateBundleSize: stateBundleSize, + MaxSubrequests: maxSubrequests, + SubrequestsEndpoint: subrequestsEndpoint, + SubrequestsInsecure: subrequestsInsecure, + SubrequestsPlaintext: subrequestsPlaintext, + BlockType: blockType, + WASMExtensions: wasmExtensions, + BlockExecutionTimeout: executionTimeout, Tracing: tracing, diff --git a/cmd/apps/substreams_tier2.go b/cmd/apps/substreams_tier2.go index 4b37508..e9e3876 100644 --- a/cmd/apps/substreams_tier2.go +++ b/cmd/apps/substreams_tier2.go @@ -55,6 +55,7 @@ func RegisterSubstreamsTier2App[B firecore.Block](chain *firecore.Chain[B], root grpcListenAddr := viper.GetString("substreams-tier2-grpc-listen-addr") maximumConcurrentRequests := viper.GetUint64("substreams-tier2-max-concurrent-requests") + executionTimeout := viper.GetDuration("substreams-block-execution-timeout") tracing := os.Getenv("SUBSTREAMS_TRACING") == "modules_exec" @@ -85,9 +86,10 @@ func RegisterSubstreamsTier2App[B firecore.Block](chain *firecore.Chain[B], root &app.Tier2Config{ Tracing: tracing, - GRPCListenAddr: grpcListenAddr, - ServiceDiscoveryURL: serviceDiscoveryURL, - WASMExtensions: wasmExtensions, + GRPCListenAddr: grpcListenAddr, + ServiceDiscoveryURL: serviceDiscoveryURL, + WASMExtensions: wasmExtensions, + BlockExecutionTimeout: executionTimeout, MaximumConcurrentRequests: maximumConcurrentRequests, }, &app.Tier2Modules{ diff --git a/go.mod b/go.mod index dbb7eed..9ad8898 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2 github.com/streamingfast/pbgo v0.0.6-0.20240430190514-722fe9d82e5d github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 - github.com/streamingfast/substreams v1.9.3 + github.com/streamingfast/substreams v1.9.4-0.20240731180302-278777b83985 github.com/stretchr/testify v1.8.4 github.com/test-go/testify v1.1.4 go.uber.org/multierr v1.10.0 diff --git a/go.sum b/go.sum index e9546e1..c43516f 100644 --- a/go.sum +++ b/go.sum @@ -582,8 +582,8 @@ github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAt github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8= github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 h1:Y15G1Z4fpEdm2b+/70owI7TLuXadlqBtGM7rk4Hxrzk= github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0/go.mod h1:/Rnz2TJvaShjUct0scZ9kKV2Jr9/+KBAoWy4UMYxgv4= -github.com/streamingfast/substreams v1.9.3 h1:3UzSelyAu0tblkZocw5/Wm5FmawwjV+k6qDb77Baj48= -github.com/streamingfast/substreams v1.9.3/go.mod h1:yPSIRwtAl9/1dB45j82ZsGe8lndRdN8oKy4k/WvHaao= +github.com/streamingfast/substreams v1.9.4-0.20240731180302-278777b83985 h1:Tcxeb9RyLmgZUybFEdopzt2ZVkncMsTuO0V1zFueaiQ= +github.com/streamingfast/substreams v1.9.4-0.20240731180302-278777b83985/go.mod h1:yPSIRwtAl9/1dB45j82ZsGe8lndRdN8oKy4k/WvHaao= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=