From 66ae2a3406ae2401c6f2abd373b96008c85686d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Mon, 26 Aug 2024 15:58:53 -0400 Subject: [PATCH] bump substreams, add --common-tmp-dir for wazero caching feature --- CHANGELOG.md | 1 + cmd/apps/substreams_tier1.go | 6 ++++++ cmd/apps/substreams_tier2.go | 6 ++++++ cmd/main.go | 1 + constants.go | 1 + go.mod | 4 ++-- go.sum | 8 ++++---- storage.go | 11 +++++++++++ 8 files changed, 32 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cecbd41..62e8464 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s - If the first-streamable-block does not match any known chain, it will require the `advertise-chain-name` to be non-empty - If the first-streamable-block type is unknown (i.e. not ethereum, solana, near, cosmos, bitcoin...), it will require the user to provide `advertise-chain-name` as well as `advertise-block-id-encoding` +* Substreams: add `--common-tmp-dir` flag and activate local caching of pre-compiled WASM modules through wazero feature * Substreams: revert module hash calculation from `v1.5.5`, when using a non-zero firstStreamableBlock. Hashes will now be the same even if the chain's first streamable block affects the initialBlock of a module. * Substreams: add `--substreams-block-execution-timeout` flag (default 3 minutes) to prevent requests stalling diff --git a/cmd/apps/substreams_tier1.go b/cmd/apps/substreams_tier1.go index 7c80028..848a61c 100644 --- a/cmd/apps/substreams_tier1.go +++ b/cmd/apps/substreams_tier1.go @@ -120,6 +120,11 @@ func RegisterSubstreamsTier1App[B firecore.Block](chain *firecore.Chain[B], root wasmExtensions = exts } + tmpDir, err := firecore.GetTmpDir(sfDataDir) + if err != nil { + return nil, fmt.Errorf("getting temporary directory: %w", err) + } + return app.NewTier1(appLogger, &app.Tier1Config{ MeteringConfig: GetCommonMeteringPluginValue(), @@ -128,6 +133,7 @@ func RegisterSubstreamsTier1App[B firecore.Block](chain *firecore.Chain[B], root OneBlocksStoreURL: oneBlocksStoreURL, ForkedBlocksStoreURL: forkedBlocksStoreURL, BlockStreamAddr: blockstreamAddr, + TmpDir: tmpDir, StateStoreURL: stateStoreURL, StateStoreDefaultTag: stateStoreDefaultTag, diff --git a/cmd/apps/substreams_tier2.go b/cmd/apps/substreams_tier2.go index e9e3876..7d4003a 100644 --- a/cmd/apps/substreams_tier2.go +++ b/cmd/apps/substreams_tier2.go @@ -82,6 +82,11 @@ func RegisterSubstreamsTier2App[B firecore.Block](chain *firecore.Chain[B], root wasmExtensions = exts } + tmpDir, err := firecore.GetTmpDir(runtime.AbsDataDir) + if err != nil { + return nil, fmt.Errorf("getting temporary directory: %w", err) + } + return app.NewTier2(appLogger, &app.Tier2Config{ Tracing: tracing, @@ -90,6 +95,7 @@ func RegisterSubstreamsTier2App[B firecore.Block](chain *firecore.Chain[B], root ServiceDiscoveryURL: serviceDiscoveryURL, WASMExtensions: wasmExtensions, BlockExecutionTimeout: executionTimeout, + TmpDir: tmpDir, MaximumConcurrentRequests: maximumConcurrentRequests, }, &app.Tier2Modules{ diff --git a/cmd/main.go b/cmd/main.go index e3baa45..d0774a0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -171,6 +171,7 @@ func registerCommonFlags[B firecore.Block](chain *firecore.Chain[B]) { cmd.Flags().String("common-merged-blocks-store-url", firecore.MergedBlocksStoreURL, "[COMMON] Store URL where to read/write merged blocks.") cmd.Flags().String("common-forked-blocks-store-url", firecore.ForkedBlocksStoreURL, "[COMMON] Store URL where to read/write forked block files that we want to keep.") cmd.Flags().String("common-live-blocks-addr", firecore.RelayerServingAddr, "[COMMON] gRPC endpoint to get real-time blocks.") + cmd.Flags().String("common-tmp-dir", firecore.TmpDir, "[COMMON] Local directory to store temporary files") cmd.Flags().String("advertise-chain-name", "", "[firehose,substreams-tier1] Chain name to advertise in the Info Endpoint. Required but it may be inferred from the genesis blocks.") cmd.Flags().StringSlice("advertise-chain-aliases", nil, "[firehose,substreams-tier1] List of chain name aliases to advertise in the Info Endpoint. If unset, it may be inferred from the genesis blocks.") diff --git a/constants.go b/constants.go index 026e4e1..75e40f7 100644 --- a/constants.go +++ b/constants.go @@ -23,4 +23,5 @@ var ( OneBlockStoreURL string = "file://{data-dir}/storage/one-blocks" ForkedBlocksStoreURL string = "file://{data-dir}/storage/forked-blocks" IndexStoreURL string = "file://{data-dir}/storage/index" + TmpDir string = "file://{data-dir}/tmp" ) diff --git a/go.mod b/go.mod index c2bad1e..96687a0 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2 github.com/streamingfast/pbgo v0.0.6-0.20240823134334-812f6a16c5cb github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 - github.com/streamingfast/substreams v1.9.4-0.20240823175139-fee92bc72fad + github.com/streamingfast/substreams v1.9.4-0.20240826160128-7d7c7b132e06 github.com/stretchr/testify v1.8.4 github.com/test-go/testify v1.1.4 go.uber.org/multierr v1.10.0 @@ -160,7 +160,7 @@ require ( github.com/streamingfast/shutter v1.5.0 github.com/subosito/gotenv v1.4.2 // indirect github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf // indirect - github.com/tetratelabs/wazero v1.7.1 // indirect + github.com/tetratelabs/wazero v1.8.0 // indirect github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869 // indirect go.opencensus.io v0.24.0 go.opentelemetry.io/contrib/detectors/gcp v1.9.0 // indirect diff --git a/go.sum b/go.sum index 9c39e77..e7c34f8 100644 --- a/go.sum +++ b/go.sum @@ -582,8 +582,8 @@ github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAt github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8= github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 h1:Y15G1Z4fpEdm2b+/70owI7TLuXadlqBtGM7rk4Hxrzk= github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0/go.mod h1:/Rnz2TJvaShjUct0scZ9kKV2Jr9/+KBAoWy4UMYxgv4= -github.com/streamingfast/substreams v1.9.4-0.20240823175139-fee92bc72fad h1:Js7hQE7ZwhLaBgy8Hd9/mZDLiqdgY/QbMwlAXBfSf9w= -github.com/streamingfast/substreams v1.9.4-0.20240823175139-fee92bc72fad/go.mod h1:GchLx+0trEb9E9QvPBbWwY2rVsJkpcw66ibpLn0OtVE= +github.com/streamingfast/substreams v1.9.4-0.20240826160128-7d7c7b132e06 h1:tK+N8JBt7/bwYcaxtCNlPan9qkPjedcWglE/KDPe3dA= +github.com/streamingfast/substreams v1.9.4-0.20240826160128-7d7c7b132e06/go.mod h1:8vkSvR4XodacDXHSBpg+L4Jcq5BQpSBhGZOvM76C+H4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -605,8 +605,8 @@ github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf h1:Z2X3Os7oRzpdJ7 github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= -github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8= -github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= +github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g= +github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk= github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869 h1:7v7L5lsfw4w8iqBBXETukHo4IPltmD+mWoLRYUmeGN8= github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869/go.mod h1:Rfzr+sqaDreiCaoQbFCu3sTXxeFq/9kXRuyOoSlGQHE= diff --git a/storage.go b/storage.go index 48e604b..da91437 100644 --- a/storage.go +++ b/storage.go @@ -14,6 +14,17 @@ import ( var commonStoresCreated bool var indexStoreCreated bool +var tmpDirCreated bool + +func GetTmpDir(dataDir string) (tmpDir string, err error) { + if tmpDirCreated { + return + } + + tmpDir = MustReplaceDataDir(dataDir, viperExpandedEnvGetString("common-tmp-dir")) + err = os.MkdirAll(tmpDir, 0755) + return +} func GetCommonStoresURLs(dataDir string) (mergedBlocksStoreURL, oneBlocksStoreURL, forkedBlocksStoreURL string, err error) { mergedBlocksStoreURL = MustReplaceDataDir(dataDir, viperExpandedEnvGetString("common-merged-blocks-store-url"))