From 850c994fc4d8554797d9c44288b75664cc46d2f7 Mon Sep 17 00:00:00 2001 From: colindickson Date: Tue, 19 Sep 2023 19:29:15 -0400 Subject: [PATCH] manifest: add Info method --- manifest/info.go | 137 +++++++++++++++++++++++++++++++++ manifest/info_test.go | 20 +++++ pb/sf/substreams/v1/modules.go | 8 ++ 3 files changed, 165 insertions(+) create mode 100644 manifest/info.go create mode 100644 manifest/info_test.go diff --git a/manifest/info.go b/manifest/info.go new file mode 100644 index 000000000..de713f589 --- /dev/null +++ b/manifest/info.go @@ -0,0 +1,137 @@ +package manifest + +import ( + "fmt" + "strings" + + pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" + "google.golang.org/protobuf/types/descriptorpb" +) + +type ManifestInfo struct { + Name string `json:"name"` + Version string `json:"version"` + Modules []ModulesInfo `json:"modules"` + ProtoFiles []*ProtoFileInfo `json:"proto_files"` +} + +type ProtoFileInfo struct { + Name *string `json:"name,omitempty"` + Package *string `json:"package,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` + PublicDependencies []int32 `json:"public_dependencies,omitempty"` + MessageType []*descriptorpb.DescriptorProto `json:"message_type,omitempty"` + Services []*descriptorpb.ServiceDescriptorProto `json:"services,omitempty"` +} + +type ModulesInfo struct { + Name string `json:"name"` + Kind string `json:"kind"` + Inputs []ModuleInput `json:"inputs"` + OutputType string `json:"output_type"` + UpdatePolicy *string `json:"update_policy,omitempty"` + InitialBlock uint64 `json:"initial_block"` + Documentation string `json:"documentation"` + Hash string `json:"hash"` +} + +type ModuleInput struct { + Type string `json:"type"` + Name string `json:"name"` + Mode *string `json:"mode,omitempty"` //for store inputs +} + +func Info(manifestPath string) (*ManifestInfo, error) { + reader, err := NewReader(manifestPath) + if err != nil { + return nil, fmt.Errorf("manifest reader: %w", err) + } + + pkg, err := reader.Read() + if err != nil { + return nil, fmt.Errorf("read manifest %q: %w", manifestPath, err) + } + + graph, err := NewModuleGraph(pkg.Modules.Modules) + if err != nil { + return nil, fmt.Errorf("creating module graph: %w", err) + } + + modules := make([]ModulesInfo, 0, len(pkg.Modules.Modules)) + + hashes := NewModuleHashes() + for ix, mod := range pkg.Modules.Modules { + modInfo := ModulesInfo{} + + _, _ = hashes.HashModule(pkg.Modules, mod, graph) + modInfo.Hash = hashes.Get(mod.Name) + + modInfo.Name = mod.Name + modInfo.InitialBlock = mod.InitialBlock + + kind := mod.GetKind() + switch v := kind.(type) { + case *pbsubstreams.Module_KindMap_: + modInfo.Kind = "map" + modInfo.OutputType = v.KindMap.OutputType + case *pbsubstreams.Module_KindStore_: + modInfo.Kind = "store" + modInfo.OutputType = v.KindStore.ValueType + modInfo.UpdatePolicy = strPtr(v.KindStore.UpdatePolicy.Pretty()) + default: + modInfo.Kind = "unknown" + } + + modMeta := pkg.ModuleMeta[ix] + if modMeta != nil && modMeta.Doc != "" { + modInfo.Documentation = strings.Replace(modMeta.Doc, "\n", "\n ", -1) + } + + inputs := make([]ModuleInput, 0, len(mod.Inputs)) + for _, input := range mod.Inputs { + inputInfo := ModuleInput{} + + switch v := input.Input.(type) { + case *pbsubstreams.Module_Input_Source_: + inputInfo.Type = "source" + inputInfo.Name = v.Source.Type + case *pbsubstreams.Module_Input_Map_: + inputInfo.Type = "map" + inputInfo.Name = v.Map.ModuleName + case *pbsubstreams.Module_Input_Store_: + inputInfo.Type = "store" + inputInfo.Name = v.Store.ModuleName + if v.Store.Mode > 0 { + inputInfo.Mode = strPtr(v.Store.Mode.Pretty()) + } + } + + inputs = append(inputs, inputInfo) + } + modInfo.Inputs = inputs + + modules = append(modules, modInfo) + } + + protoFiles := make([]*ProtoFileInfo, 0, len(pkg.ProtoFiles)) + for _, protoFile := range pkg.ProtoFiles { + protoFiles = append(protoFiles, &ProtoFileInfo{ + Name: protoFile.Name, + Package: protoFile.Package, + Dependencies: protoFile.Dependency, + PublicDependencies: protoFile.PublicDependency, + MessageType: protoFile.MessageType, + }) + } + + return &ManifestInfo{ + Name: pkg.PackageMeta[0].Name, + Version: pkg.PackageMeta[0].Version, + Modules: modules, + ProtoFiles: protoFiles, + }, nil +} + +func strPtr(s string) *string { + return &s +} diff --git a/manifest/info_test.go b/manifest/info_test.go new file mode 100644 index 000000000..3fd3a4048 --- /dev/null +++ b/manifest/info_test.go @@ -0,0 +1,20 @@ +package manifest + +import ( + "encoding/json" + "fmt" + "github.com/stretchr/testify/require" + "testing" +) + +func TestInfo(t *testing.T) { + info, err := Info("https://github.com/streamingfast/substreams-uniswap-v3/releases/download/v0.2.8/substreams.spkg") + if err != nil { + t.Fatal(err) + } + + r, err := json.MarshalIndent(info, "", " ") + require.NoError(t, err) + + fmt.Println(string(r)) +} diff --git a/pb/sf/substreams/v1/modules.go b/pb/sf/substreams/v1/modules.go index 282bd18dd..ccf63d787 100644 --- a/pb/sf/substreams/v1/modules.go +++ b/pb/sf/substreams/v1/modules.go @@ -36,3 +36,11 @@ func (x *Module_Input) Pretty() string { return strings.TrimSpace(result) } + +func (x Module_KindStore_UpdatePolicy) Pretty() string { + return strings.TrimPrefix(strings.ToLower(x.String()), "update_policy_") +} + +func (x Module_Input_Store_Mode) Pretty() string { + return strings.ToLower(Module_Input_Store_Mode_name[int32(x)]) +}