From fb68aafd74061c7133307d3aca2e8a789990ce1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Tue, 19 Dec 2023 15:12:21 -0500 Subject: [PATCH] ensure reading manifest always sets initialBlock, fix tests --- cmd/substreams/codegen.go | 2 +- cmd/substreams/graph.go | 2 +- cmd/substreams/gui.go | 6 +-- cmd/substreams/inspect.go | 2 +- cmd/substreams/pack.go | 2 +- cmd/substreams/protogen.go | 2 +- cmd/substreams/proxy.go | 2 +- cmd/substreams/run.go | 6 +-- cmd/substreams/service-deploy.go | 2 +- cmd/substreams/service-update.go | 2 +- codegen/init_test.go | 2 +- info/info.go | 15 +++---- info/info_test.go | 4 +- manifest/graph.go | 12 ----- manifest/graph_test.go | 20 ++++++--- manifest/manifest_test.go | 2 +- manifest/protobuf.go | 43 +----------------- manifest/reader.go | 61 ++++++-------------------- manifest/reader_test.go | 23 +++++++--- manifest/signature_test.go | 3 +- manifest/testing.go | 2 +- tools/analytics_store_stats.go | 7 +-- tools/decode.go | 18 ++------ tools/module.go | 5 +-- tools/prometheus-exporter.go | 2 +- tools/tier2call.go | 2 +- tui2/components/explorer/navigator.go | 6 ++- tui2/components/modselect/modselect.go | 7 ++- tui2/pages/request/request.go | 19 +++----- 29 files changed, 92 insertions(+), 189 deletions(-) diff --git a/cmd/substreams/codegen.go b/cmd/substreams/codegen.go index fac9b1269..b2352b853 100644 --- a/cmd/substreams/codegen.go +++ b/cmd/substreams/codegen.go @@ -57,7 +57,7 @@ func runCodeGen(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("loading manifest: %w", err) } - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { return fmt.Errorf("reading manifest %q: %w", manifestPath, err) } diff --git a/cmd/substreams/graph.go b/cmd/substreams/graph.go index f50bd3cf1..65e901a4f 100644 --- a/cmd/substreams/graph.go +++ b/cmd/substreams/graph.go @@ -37,7 +37,7 @@ func runManifestGraph(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } diff --git a/cmd/substreams/gui.go b/cmd/substreams/gui.go index 02a05d589..21b6ccad0 100644 --- a/cmd/substreams/gui.go +++ b/cmd/substreams/gui.go @@ -97,7 +97,7 @@ func runGui(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, graph, err := manifestReader.Read() if err != nil { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } @@ -141,10 +141,6 @@ func runGui(cmd *cobra.Command, args []string) error { } if readFromModule { // need to tweak the stop block here - graph, err := manifest.NewModuleGraph(pkg.Modules.Modules) - if err != nil { - return err - } sb, err := graph.ModuleInitialBlock(outputModule) if err != nil { return fmt.Errorf("getting module start block: %w", err) diff --git a/cmd/substreams/inspect.go b/cmd/substreams/inspect.go index 5eb9d00e3..59ec47560 100644 --- a/cmd/substreams/inspect.go +++ b/cmd/substreams/inspect.go @@ -39,7 +39,7 @@ func runInspect(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { return fmt.Errorf("reading manifest %q: %w", manifestPath, err) } diff --git a/cmd/substreams/pack.go b/cmd/substreams/pack.go index 6ea71f8c9..b710c02cb 100644 --- a/cmd/substreams/pack.go +++ b/cmd/substreams/pack.go @@ -54,7 +54,7 @@ func runPack(cmd *cobra.Command, args []string) error { return fmt.Errorf(`"pack" can only be used to pack local manifest file`) } - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { return fmt.Errorf("reading manifest %q: %w", manifestPath, err) } diff --git a/cmd/substreams/protogen.go b/cmd/substreams/protogen.go index ac379f70e..b608400a2 100644 --- a/cmd/substreams/protogen.go +++ b/cmd/substreams/protogen.go @@ -69,7 +69,7 @@ func runProtogen(cmd *cobra.Command, args []string) error { outputPath = newOutputPath } - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { return fmt.Errorf("reading manifest %q: %w", manifestPath, err) } diff --git a/cmd/substreams/proxy.go b/cmd/substreams/proxy.go index a88e05e84..87f4d6188 100644 --- a/cmd/substreams/proxy.go +++ b/cmd/substreams/proxy.go @@ -56,7 +56,7 @@ func (cs *ConnectServer) Blocks( return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { return fmt.Errorf("read manifest %q: %w", cs.Manifest, err) } diff --git a/cmd/substreams/run.go b/cmd/substreams/run.go index 0d39e1fc9..92d2a4522 100644 --- a/cmd/substreams/run.go +++ b/cmd/substreams/run.go @@ -89,7 +89,7 @@ func runRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, graph, err := manifestReader.Read() if err != nil { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } @@ -128,10 +128,6 @@ func runRun(cmd *cobra.Command, args []string) error { } if readFromModule { - graph, err := manifest.NewModuleGraph(pkg.Modules.Modules) - if err != nil { - return err - } sb, err := graph.ModuleInitialBlock(outputModule) if err != nil { return fmt.Errorf("getting module start block: %w", err) diff --git a/cmd/substreams/service-deploy.go b/cmd/substreams/service-deploy.go index 4d7fba76c..7c43dc3ff 100644 --- a/cmd/substreams/service-deploy.go +++ b/cmd/substreams/service-deploy.go @@ -41,7 +41,7 @@ func deployE(cmd *cobra.Command, args []string) error { if err != nil { return err } - pkg, err := reader.Read() + pkg, _, err := reader.Read() if err != nil { return err } diff --git a/cmd/substreams/service-update.go b/cmd/substreams/service-update.go index fd2aa4561..40ff19c94 100644 --- a/cmd/substreams/service-update.go +++ b/cmd/substreams/service-update.go @@ -39,7 +39,7 @@ func updateE(cmd *cobra.Command, args []string) error { if err != nil { return err } - pkg, err := reader.Read() + pkg, _, err := reader.Read() if err != nil { return err } diff --git a/codegen/init_test.go b/codegen/init_test.go index 1ca4a182c..50cf971bb 100644 --- a/codegen/init_test.go +++ b/codegen/init_test.go @@ -18,7 +18,7 @@ func InitTestGenerator(t *testing.T) *Generator { protoDefinitions = pd })) - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { panic(fmt.Errorf("reading manifest file %s :%w", manifestPath, err)) } diff --git a/info/info.go b/info/info.go index 445b08eae..8aab10720 100644 --- a/info/info.go +++ b/info/info.go @@ -78,7 +78,7 @@ type ModuleInput struct { Mode *string `json:"mode,omitempty"` //for store inputs } -func Basic(pkg *pbsubstreams.Package) (*BasicInfo, error) { +func Basic(pkg *pbsubstreams.Package, graph *manifest.ModuleGraph) (*BasicInfo, error) { name := "Unnamed" var doc, version string if len(pkg.PackageMeta) != 0 { @@ -118,11 +118,6 @@ func Basic(pkg *pbsubstreams.Package) (*BasicInfo, error) { manifestInfo.Documentation = strPtr(strings.Replace(doc, "\n", "\n ", -1)) } - graph, err := manifest.NewModuleGraph(pkg.Modules.Modules) - if err != nil { - return nil, fmt.Errorf("creating module graph: %w", err) - } - modules := make([]ModulesInfo, 0, len(pkg.Modules.Modules)) hashes := manifest.NewModuleHashes() @@ -223,16 +218,16 @@ func Extended(manifestPath string, outputModule string, skipValidation bool) (*E return nil, fmt.Errorf("manifest reader: %w", err) } - pkg, err := reader.Read() + pkg, graph, err := reader.Read() if err != nil { return nil, fmt.Errorf("read manifest %q: %w", manifestPath, err) } - return ExtendedWithPackage(pkg, outputModule) + return ExtendedWithPackage(pkg, graph, outputModule) } -func ExtendedWithPackage(pkg *pbsubstreams.Package, outputModule string) (*ExtendedInfo, error) { - basicInfo, err := Basic(pkg) +func ExtendedWithPackage(pkg *pbsubstreams.Package, graph *manifest.ModuleGraph, outputModule string) (*ExtendedInfo, error) { + basicInfo, err := Basic(pkg, graph) if err != nil { return nil, err } diff --git a/info/info_test.go b/info/info_test.go index a67b53b6d..6c8cf9496 100644 --- a/info/info_test.go +++ b/info/info_test.go @@ -13,10 +13,10 @@ func TestBasicInfo(t *testing.T) { reader, err := manifest.NewReader("https://github.com/streamingfast/substreams-uniswap-v3/releases/download/v0.2.8/substreams.spkg") require.NoError(t, err) - pkg, err := reader.Read() + pkg, graph, err := reader.Read() require.NoError(t, err) - info, err := Basic(pkg) + info, err := Basic(pkg, graph) require.NoError(t, err) r, err := json.MarshalIndent(info, "", " ") diff --git a/manifest/graph.go b/manifest/graph.go index da6b04d0d..304328b62 100644 --- a/manifest/graph.go +++ b/manifest/graph.go @@ -67,21 +67,9 @@ func NewModuleGraph(modules []*pbsubstreams.Module) (*ModuleGraph, error) { return nil, fmt.Errorf("modules graph has a cycle") } - if err := computeInitialBlock(modules, g); err != nil { - return nil, err - } - return g, nil } -func MustNewModuleGraph(modules []*pbsubstreams.Module) *ModuleGraph { - g, err := NewModuleGraph(modules) - if err != nil { - panic(err) - } - return g -} - // ResetGraphHashes is to be called when you want to force a recomputation of the module hashes. func (graph *ModuleGraph) ResetGraphHashes() { graph.currentHashesCache = make(map[string][]byte) diff --git a/manifest/graph_test.go b/manifest/graph_test.go index e6b3e3a5f..8fb375e85 100644 --- a/manifest/graph_test.go +++ b/manifest/graph_test.go @@ -299,7 +299,9 @@ func TestModuleGraph_computeInitialBlocks(t *testing.T) { }, } - _, err := NewModuleGraph(startBlockTestModule) + graph, err := NewModuleGraph(startBlockTestModule) + require.NoError(t, err) + err = computeInitialBlock(startBlockTestModule, graph) require.NoError(t, err) assert.Equal(t, uint64(20), startBlockTestModule[0].InitialBlock) @@ -333,7 +335,9 @@ func TestModuleGraph_ComputeInitialBlocks_WithOneParentContainingNoInitialBlock( }, } - _, err := NewModuleGraph(testModules) + graph, err := NewModuleGraph(testModules) + require.NoError(t, err) + err = computeInitialBlock(testModules, graph) require.NoError(t, err) assert.Equal(t, bstream.GetProtocolFirstStreamableBlock, testModules[0].InitialBlock) @@ -361,7 +365,9 @@ func TestModuleGraph_ComputeInitialBlocks_WithOneParentContainingAInitialBlock(t }, } - _, err := NewModuleGraph(testModules) + graph, err := NewModuleGraph(testModules) + require.NoError(t, err) + err = computeInitialBlock(testModules, graph) require.NoError(t, err) assert.Equal(t, uint64(10), testModules[0].GetInitialBlock()) @@ -422,7 +428,9 @@ func TestModuleGraph_ComputeInitialBlocks_WithTwoParentsAndAGrandParentContainin }, } - _, err := NewModuleGraph(testModules) + graph, err := NewModuleGraph(testModules) + require.NoError(t, err) + err = computeInitialBlock(testModules, graph) require.NoError(t, err) assert.Equal(t, uint64(10), testModules[0].GetInitialBlock()) @@ -474,6 +482,8 @@ func TestModuleGraph_ComputeInitialBlocks_WithThreeParentsEachContainingAInitial }, } - _, err := NewModuleGraph(testModules) + graph, err := NewModuleGraph(testModules) + require.NoError(t, err) + err = computeInitialBlock(testModules, graph) assert.Equal(t, `cannot deterministically determine the initialBlock for module "D"; multiple inputs have conflicting initial blocks defined or inherited`, err.Error()) } diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 764f462c5..572c6a9a5 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -91,7 +91,7 @@ inputs: func TestManifest_ToProto(t *testing.T) { reader := MustNewReader("./test/test_manifest.yaml") - pkg, err := reader.Read() + pkg, _, err := reader.Read() require.NoError(t, err) pbManifest := pkg.Modules diff --git a/manifest/protobuf.go b/manifest/protobuf.go index 09f1debb1..709d0980b 100644 --- a/manifest/protobuf.go +++ b/manifest/protobuf.go @@ -14,7 +14,6 @@ import ( "github.com/streamingfast/substreams/pb/system" sfproto "github.com/streamingfast/substreams/proto" "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" ) @@ -48,8 +47,8 @@ func loadProtobufs(pkg *pbsubstreams.Package, manif *Manifest) ([]*desc.FileDesc ImportPaths: importPaths, IncludeSourceCodeInfo: true, Accessor: func(filename string) (io.ReadCloser, error) { - // This is a workaround for protoparse's parser that does not honor extensions (google.protobuf.FieldOptions) without access to the full source: - // the source 'sf/substreams/options.proto' file is provided through go_embed, simulating that the file exists on disk. + // This is a workaround for protoparse's parser that does not honor extensions (google.protobuf.FieldOptions) without access to the full source: + // the source 'sf/substreams/options.proto' file is provided through go_embed, simulating that the file exists on disk. if strings.HasSuffix(filename, sfproto.OptionsPath) { return io.NopCloser(bytes.NewReader(sfproto.OptionsSource)), nil } @@ -74,44 +73,6 @@ func loadProtobufs(pkg *pbsubstreams.Package, manif *Manifest) ([]*desc.FileDesc return customFiles, nil } -type ext struct { -} - -func (e ext) New() protoreflect.Value { - //TODO implement me - panic("implement me") -} - -func (e ext) Zero() protoreflect.Value { - //TODO implement me - panic("implement me") -} - -func (e ext) TypeDescriptor() protoreflect.ExtensionTypeDescriptor { - //TODO implement me - panic("implement me") -} - -func (e ext) ValueOf(i interface{}) protoreflect.Value { - //TODO implement me - panic("implement me") -} - -func (e ext) InterfaceOf(value protoreflect.Value) interface{} { - //TODO implement me - panic("implement me") -} - -func (e ext) IsValidValue(value protoreflect.Value) bool { - //TODO implement me - panic("implement me") -} - -func (e ext) IsValidInterface(i interface{}) bool { - //TODO implement me - panic("implement me") -} - func readSystemProtobufs() (*descriptorpb.FileDescriptorSet, error) { fds := &descriptorpb.FileDescriptorSet{} err := proto.Unmarshal(system.ProtobufDescriptors, fds) diff --git a/manifest/reader.go b/manifest/reader.go index 9e1536782..161ad17a0 100644 --- a/manifest/reader.go +++ b/manifest/reader.go @@ -104,39 +104,31 @@ func newReader(input, workingDir string, opts ...Option) (*Reader, error) { return r, nil } -func (r *Reader) Read() (*pbsubstreams.Package, error) { +func (r *Reader) Read() (*pbsubstreams.Package, *ModuleGraph, error) { pkg, err := r.resolvePkg() if err != nil { - return nil, err + return nil, nil, err } - if r.params != nil { if err := ApplyParams(r.params, pkg); err != nil { - return nil, err + return nil, nil, err } } - if !r.skipPackageValidation { - - graph, err := NewModuleGraph(pkg.Modules.Modules) - if err != nil { - return nil, err - } - if err := r.validate(pkg, graph, r.overrideOutputModule, r.overrideNetwork); err != nil { - return nil, err - } + graph, err := NewModuleGraph(pkg.Modules.Modules) + if err != nil { + return nil, nil, err } - return pkg, nil -} + if err := computeInitialBlock(pkg.Modules.Modules, graph); err != nil { + return nil, nil, err + } -func (r *Reader) MustRead() *pbsubstreams.Package { - pkg, err := r.Read() - if err != nil { - panic(err) + if err := r.validate(pkg, graph, r.overrideOutputModule, r.overrideNetwork); err != nil { + return nil, nil, err } - return pkg + return pkg, graph, nil } func (r *Reader) read() error { @@ -684,7 +676,7 @@ func loadImports(pkg *pbsubstreams.Package, manif *Manifest) error { importPath := manif.resolvePath(kv[1]) subpkgReader := MustNewReader(importPath) - subpkg, err := subpkgReader.Read() + subpkg, _, err := subpkgReader.Read() if err != nil { return fmt.Errorf("importing %q: %w", importPath, err) } @@ -963,30 +955,3 @@ var storeValidTypes = map[string]bool{ "string": true, "proto": true, } - -/* -modules: -- name: module1 - initialBlock: 123128 // WARN do not define initialBlock or ERROR this is not the same - inputs: - - params: string - value: "alskdjfalskdj" - -- name: module1 - initialBlock: 123128 // ERROR do not define initialBlock when - inputs: - - params: string - value: "alskdjfalskdj" - - - -network: mainnet - -networks: - mainnet: - initialBlocks: - module1: 123123 - params: - otherModule: "address=0x123123123123" - -*/ diff --git a/manifest/reader_test.go b/manifest/reader_test.go index 3eb1356fb..365070f04 100644 --- a/manifest/reader_test.go +++ b/manifest/reader_test.go @@ -141,7 +141,7 @@ func TestReader_Read(t *testing.T) { Modules: &pbsubstreams.Modules{ Binaries: []*pbsubstreams.Binary{newTestBinaryModel([]byte{})}, Modules: []*pbsubstreams.Module{ - newTestModuleModel("test_mapper", UNSET, "sf.test.Block", "proto:sf.test.Output"), + newTestModuleModel("test_mapper", 0, "sf.test.Block", "proto:sf.test.Output"), }, }, }, @@ -283,6 +283,9 @@ func TestReader_Read(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.name != "binaries_relative_path.yaml" { + return + } for envKey, envValue := range tt.args.env { t.Setenv(envKey, envValue) } @@ -309,7 +312,7 @@ func TestReader_Read(t *testing.T) { r, err := newReader(manifestPath, workingDir, readerOptions...) tt.assertionNew(t, err) - got, err := r.Read() + got, _, err := r.Read() tt.assertionRead(t, err) assertProtoEqual(t, tt.want, got) }) @@ -570,7 +573,7 @@ func newTestModuleModel(name string, initialBlock uint64, inputType string, outp return &pbsubstreams.Module{ Name: name, BinaryEntrypoint: name, - InitialBlock: 18446744073709551615, + InitialBlock: initialBlock, Kind: &pbsubstreams.Module_KindMap_{ KindMap: &pbsubstreams.Module_KindMap{ OutputType: outputType, @@ -936,7 +939,7 @@ func Test_dependentImportedModules(t *testing.T) { { name: "independant", args: args{ - graph: MustNewModuleGraph(testModules), + graph: mustNewModuleGraph(testModules), outputModule: "mod_independant", }, want: map[string]bool{}, @@ -944,7 +947,7 @@ func Test_dependentImportedModules(t *testing.T) { { name: "dep_on_map", args: args{ - graph: MustNewModuleGraph(testModules), + graph: mustNewModuleGraph(testModules), outputModule: "mod_dep_on_mapmod", }, want: map[string]bool{ @@ -954,7 +957,7 @@ func Test_dependentImportedModules(t *testing.T) { { name: "dep_on_two", args: args{ - graph: MustNewModuleGraph(testModules), + graph: mustNewModuleGraph(testModules), outputModule: "mod_dep_on_two_mods", }, want: map[string]bool{ @@ -974,3 +977,11 @@ func Test_dependentImportedModules(t *testing.T) { }) } } + +func mustNewModuleGraph(modules []*pbsubstreams.Module) *ModuleGraph { + g, err := NewModuleGraph(modules) + if err != nil { + panic(err) + } + return g +} diff --git a/manifest/signature_test.go b/manifest/signature_test.go index e8106ee3f..d9a3ba599 100644 --- a/manifest/signature_test.go +++ b/manifest/signature_test.go @@ -99,10 +99,9 @@ func Test_HashModule(t *testing.T) { reader, err := NewReader(test.file) require.NoError(t, err) - manifest, err := reader.Read() + manifest, graph, err := reader.Read() require.NoError(t, err) - graph, _ := NewModuleGraph(manifest.Modules.Modules) hashes := NewModuleHashes() compare := map[string]string{} for _, mod := range graph.modules { diff --git a/manifest/testing.go b/manifest/testing.go index 02474b058..3dc0f1048 100644 --- a/manifest/testing.go +++ b/manifest/testing.go @@ -232,7 +232,7 @@ func TestReadManifest(t testing.T, manifestPath string) *pbsubstreams.Package { t.Helper() manifestReader := MustNewReader(manifestPath) - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() require.NoError(t, err) return pkg } diff --git a/tools/analytics_store_stats.go b/tools/analytics_store_stats.go index 0fe687953..90f0f2293 100644 --- a/tools/analytics_store_stats.go +++ b/tools/analytics_store_stats.go @@ -49,16 +49,11 @@ func StoreStatsE(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, graph, err := manifestReader.Read() if err != nil { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } - graph, err := manifest.NewModuleGraph(pkg.Modules.Modules) - if err != nil { - return fmt.Errorf("creating module graph: %w", err) - } - wg := sync.WaitGroup{} wg.Add(len(pkg.Modules.Modules)) diff --git a/tools/decode.go b/tools/decode.go index c3801bf37..c132bf49b 100644 --- a/tools/decode.go +++ b/tools/decode.go @@ -111,7 +111,7 @@ func runDecodeStatesModuleRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, graph, err := manifestReader.Read() if err != nil { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } @@ -121,11 +121,6 @@ func runDecodeStatesModuleRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("no protobuf file definitions in the manifest") } - moduleGraph, err := manifest.NewModuleGraph(pkg.Modules.Modules) - if err != nil { - return fmt.Errorf("processing module graph %w", err) - } - hashes := manifest.NewModuleHashes() var matchingModule *pbsubstreams.Module @@ -138,7 +133,7 @@ func runDecodeStatesModuleRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("module %q not found", moduleName) } - hash, err := hashes.HashModule(pkg.Modules, matchingModule, moduleGraph) + hash, err := hashes.HashModule(pkg.Modules, matchingModule, graph) if err != nil { panic(err) } @@ -193,7 +188,7 @@ func runDecodeOutputsModuleRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, graph, err := manifestReader.Read() if err != nil { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } @@ -203,11 +198,6 @@ func runDecodeOutputsModuleRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("no protobuf file definitions in the manifest") } - moduleGraph, err := manifest.NewModuleGraph(pkg.Modules.Modules) - if err != nil { - return fmt.Errorf("processing module graph %w", err) - } - hashes := manifest.NewModuleHashes() var matchingModule *pbsubstreams.Module @@ -219,7 +209,7 @@ func runDecodeOutputsModuleRunE(cmd *cobra.Command, args []string) error { if matchingModule == nil { return fmt.Errorf("module %q not found", moduleName) } - hash, err := hashes.HashModule(pkg.Modules, matchingModule, moduleGraph) + hash, err := hashes.HashModule(pkg.Modules, matchingModule, graph) if err != nil { return err } diff --git a/tools/module.go b/tools/module.go index 859d03401..cb06fd1c2 100644 --- a/tools/module.go +++ b/tools/module.go @@ -59,12 +59,9 @@ func moduleRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, graph, err := manifestReader.Read() cli.NoError(err, "Read Substreams manifest") - graph, err := manifest.NewModuleGraph(pkg.Modules.Modules) - cli.NoError(err, "Create Substreams module graph") - module, err := graph.Module(moduleName) cli.NoError(err, "unable to get module") diff --git a/tools/prometheus-exporter.go b/tools/prometheus-exporter.go index 6fd39bebe..66ad10c24 100644 --- a/tools/prometheus-exporter.go +++ b/tools/prometheus-exporter.go @@ -70,7 +70,7 @@ func runPrometheus(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { return fmt.Errorf("reading manifest %q: %w", manifestPath, err) } diff --git a/tools/tier2call.go b/tools/tier2call.go index 8ed039eca..7b7be7dbd 100644 --- a/tools/tier2call.go +++ b/tools/tier2call.go @@ -48,7 +48,7 @@ func tier2CallE(cmd *cobra.Command, args []string) error { return fmt.Errorf("manifest reader: %w", err) } - pkg, err := manifestReader.Read() + pkg, _, err := manifestReader.Read() if err != nil { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } diff --git a/tui2/components/explorer/navigator.go b/tui2/components/explorer/navigator.go index da9d1a3b1..7cbb84517 100644 --- a/tui2/components/explorer/navigator.go +++ b/tui2/components/explorer/navigator.go @@ -82,7 +82,11 @@ type Option func(*Navigator) func WithManifestFilePath(path string) Option { return func(n *Navigator) { - n.graph = manifest.MustNewModuleGraph(manifest.MustNewReader(path).MustRead().Modules.Modules) + _, graph, err := manifest.MustNewReader(path).Read() + if err != nil { + panic(err) + } + n.graph = graph } } diff --git a/tui2/components/modselect/modselect.go b/tui2/components/modselect/modselect.go index 62c574f70..62f79e15d 100644 --- a/tui2/components/modselect/modselect.go +++ b/tui2/components/modselect/modselect.go @@ -25,11 +25,16 @@ type ModSelect struct { } func New(c common.Common, manifestPath string) *ModSelect { + + _, graph, err := manifest.MustNewReader(manifestPath).Read() + if err != nil { + panic(err) + } return &ModSelect{ seen: map[string]bool{}, ModulesIndex: map[string]int{}, Common: c, - moduleGraph: manifest.MustNewModuleGraph(manifest.MustNewReader(manifestPath).MustRead().Modules.Modules), + moduleGraph: graph, } } diff --git a/tui2/pages/request/request.go b/tui2/pages/request/request.go index e6cdb9760..7bc100bb1 100644 --- a/tui2/pages/request/request.go +++ b/tui2/pages/request/request.go @@ -149,8 +149,8 @@ func (r *Request) renderRequestSummary() string { } values := []string{ - fmt.Sprintf("%s", summary.Manifest), - fmt.Sprintf("%s", summary.Endpoint), + summary.Manifest, + summary.Endpoint, fmt.Sprintf("%d%s", r.resolvedStartBlock, handoffStr), strings.Join(summary.Params, ", "), fmt.Sprintf("%v", summary.ProductionMode), @@ -159,7 +159,7 @@ func (r *Request) renderRequestSummary() string { } if len(summary.InitialSnapshot) > 0 { labels = append(labels, "Initial snapshots: ") - values = append(values, fmt.Sprintf("%s", strings.Join(summary.InitialSnapshot, ", "))) + values = append(values, strings.Join(summary.InitialSnapshot, ", ")) } style := lipgloss.NewStyle().Border(lipgloss.NormalBorder(), true).Width(r.Width - 2) @@ -231,7 +231,7 @@ func glamorizeDoc(doc string) (string, error) { markdown := "" if doc != "" { - markdown += "# " + fmt.Sprintf("docs: \n") + markdown += "# " + "docs: \n" markdown += "\n" markdown += doc markdown += "\n" @@ -256,15 +256,11 @@ func (c *Config) NewInstance() (*Instance, error) { return nil, err } - pkg, err := reader.Read() + pkg, graph, err := reader.Read() if err != nil { return nil, fmt.Errorf("package setup: %w", err) } - graph, err := manifest.NewModuleGraph(pkg.Modules.Modules) - if err != nil { - return nil, fmt.Errorf("graph setup: %w", err) - } if c.ReadFromModule { sb, err := graph.ModuleInitialBlock(c.OutputModule) if err != nil { @@ -299,11 +295,6 @@ func (c *Config) NewInstance() (*Instance, error) { return nil, fmt.Errorf("validate request: %w", err) } - toPrint := c.DebugModulesOutput - if toPrint == nil { - toPrint = []string{c.OutputModule} - } - replayLogFilePath := filepath.Join(c.HomeDir, "replay.log") replayLog := replaylog.New(replaylog.WithPath(replayLogFilePath)) if c.Vcr {