* add proto sf.substreams.options to allow bundle zip files in sinkCo…

…nfig

* allow `substreams info` to read sinkconfig, add output-sinkconfig-files-path flag
* changelog

cmd/substreams/info.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/streamingfast/cli"
@@ -12,10 +14,10 @@ import (
 	"github.com/streamingfast/substreams/pipeline/outputmodules"
 )
 
-//	var manifestCmd = &cobra.Command{
-//		Use:          "manifest",
-//		SilenceUsage: true,
-//	}
+func init() {
+	infoCmd.Flags().String("output-sinkconfig-files-path", "", "if non-empty, any sinkconfig field of type 'bytes' that was packed from a file will be written to that path")
+}
+
 var infoCmd = &cobra.Command{
 	Use:   "info [<manifest_file> [<output_module>]]",
 	Short: "Display package modules and docs",
@@ -45,6 +47,8 @@ func runInfo(cmd *cobra.Command, args []string) error {
 		outputModule = args[1]
 	}
 
+	outputSinkconfigFilesPath := mustGetString(cmd, "output-sinkconfig-files-path")
+
 	manifestReader, err := manifest.NewReader(manifestPath)
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
@@ -115,5 +119,39 @@ func runInfo(cmd *cobra.Command, args []string) error {
 
 	}
 
+	if pkg.SinkConfig != nil {
+		fmt.Println()
+		fmt.Println("Sink config:")
+		fmt.Println("----")
+		fmt.Println("type:", pkg.SinkConfig.TypeUrl)
+
+		confs, files, err := manifest.DescribeSinkConfigs(pkg)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("configs:")
+		fmt.Println(confs)
+
+		if outputSinkconfigFilesPath != "" && files != nil {
+			if err := os.MkdirAll(outputSinkconfigFilesPath, 0755); err != nil {
+				return err
+			}
+			fmt.Println("output files:")
+			for k, v := range files {
+				filename := filepath.Join(outputSinkconfigFilesPath, k)
+				f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+				if err != nil {
+					return err
+				}
+				if _, err := f.Write(v); err != nil {
+					return err
+				}
+				fmt.Printf("  - %q written to %q\n", k, filename)
+			}
+		}
+
+	}
+
 	return nil
 }

cmd/substreams/logging.go

@@ -2,10 +2,11 @@ package main
 
 import (
 	"github.com/streamingfast/logging"
+	"go.uber.org/zap/zapcore"
 )
 
 var zlog, tracer = logging.RootLogger("substreams", "github.com/streamingfast/substreams/cmd/substreams")
 
 func init() {
-	logging.InstantiateLoggers(logging.WithSwitcherServerAutoStart())
+	logging.InstantiateLoggers(logging.WithLogLevelSwitcherServerAutoStart(), logging.WithDefaultLevel(zapcore.WarnLevel))
 }

docs/release-notes/change-log.md

@@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Highlights
+
+* This release brings enhancements to "sink-config" features, as a step to enable more potent descriptions of "deployable units" inside your Substreams package. The Substreams sinks will be able to leverage the new FieldOptions in upcoming releases.
+
+### Added
+
+* Sink configs can now use protobuf annotations (aka Field Options) to determine how the field will be interpreted in
+  substreams.yaml:
+  * `load_from_file` will put the content of the file directly in the field (string and bytes contents are supported).
+  * `zip_from_folder` will create a zip archive and put its content in the field (field type must be bytes).
+
+  Example: 
+  ```
+  import "sf/substreams/v1/options.proto";
+
+  message HostedPostgresDatabase {
+    bytes schema = 1 [ (sf.substreams.v1.options).load_from_file = true ];
+    bytes extra_config_files = 2 [ (sf.substreams.v1.options).zip_from_folder = true ];
+  }
+  ```
+
+* `substreams info` command now properly displays the content of sink configs, optionally writing the fields that were bundled from files to disk with `--output-sinkconfig-files-path=</some/path>`
+
 ## v1.1.14
 
 ### Bug fixes

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1
 	github.com/streamingfast/dgrpc v0.0.0-20230621153617-bc715cdb9fd1
 	github.com/streamingfast/dstore v0.1.1-0.20230620124109-3924b3b36c77
-	github.com/streamingfast/logging v0.0.0-20220511154537-ce373d264338
+	github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091
 	github.com/streamingfast/pbgo v0.0.6-0.20221020131607-255008258d28
 	github.com/stretchr/testify v1.8.3
 	github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869
@@ -190,7 +190,7 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.9.0 // indirect
-	go.opentelemetry.io/otel/exporters/jaeger v1.15.1 // indirect
+	go.opentelemetry.io/otel/exporters/jaeger v1.16.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1 // indirect

go.sum

@@ -675,8 +675,8 @@ github.com/streamingfast/logging v0.0.0-20210811175431-f3b44b61606a/go.mod h1:4G
 github.com/streamingfast/logging v0.0.0-20210908162127-bdc5856d5341/go.mod h1:4GdqELhZOXj4xwc4IaBmzofzdErGynnaSzuzxy0ZIBo=
 github.com/streamingfast/logging v0.0.0-20220304183711-ddba33d79e27/go.mod h1:4GdqELhZOXj4xwc4IaBmzofzdErGynnaSzuzxy0ZIBo=
 github.com/streamingfast/logging v0.0.0-20220304214715-bc750a74b424/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU=
-github.com/streamingfast/logging v0.0.0-20220511154537-ce373d264338 h1:Mn7i0pN8FASFpCDvtnRhV7L1bdlj1MteLkRWsxFVIhE=
-github.com/streamingfast/logging v0.0.0-20220511154537-ce373d264338/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU=
+github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 h1:RN5mrigyirb8anBEtdjtHFIufXdacyTi6i4KBfeNXeo=
+github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU=
 github.com/streamingfast/opaque v0.0.0-20210811180740-0c01d37ea308 h1:xlWSfi1BoPfsHtPb0VEHGUcAdBF208LUiFCwfaVPfLA=
 github.com/streamingfast/opaque v0.0.0-20210811180740-0c01d37ea308/go.mod h1:K1p8Bj/wG34KJvYzPUqtzpndffmpkrVY11u2hkyxCWQ=
 github.com/streamingfast/pbgo v0.0.6-0.20221020131607-255008258d28 h1:wmQg8T0rIFl/R3dy97OWRi8OSdM3llvRw2p3TPFVKZQ=
@@ -761,8 +761,8 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.3
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4/go.mod h1:05eWWy6ZWzmpeImD3UowLTB3VjDMU1yxQ+ENuVWDM3c=
 go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
 go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
-go.opentelemetry.io/otel/exporters/jaeger v1.15.1 h1:x3SLvwli0OyAJapNcOIzf1xXBRBA+HD3elrMQmFfmXo=
-go.opentelemetry.io/otel/exporters/jaeger v1.15.1/go.mod h1:0Ck9b5oLL/bFZvfAEEqtrb1U0jZXjm5fWXMCOCG3vvM=
+go.opentelemetry.io/otel/exporters/jaeger v1.16.0 h1:YhxxmXZ011C0aDZKoNw+juVWAmEfv/0W2XBOv9aHTaA=
+go.opentelemetry.io/otel/exporters/jaeger v1.16.0/go.mod h1:grYbBo/5afWlPpdPZYhyn78Bk04hnvxn2+hvxQhKIQM=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1 h1:XYDQtNzdb2T4uM1pku2m76eSMDJgqhJ+6KzkqgQBALc=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1/go.mod h1:uOTV75+LOzV+ODmL8ahRLWkFA3eQcSC2aAsbxIu4duk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1 h1:tyoeaUh8REKay72DVYsSEBYV18+fGONe+YYPaOxgLoE=

manifest/logging.go

@@ -2,11 +2,7 @@ package manifest
 
 import (
 	"github.com/streamingfast/logging"
-	"go.uber.org/zap"
 )
 
-var zlog *zap.Logger
+var zlog,  _ = logging.PackageLogger("pipeline", "github.com/streamingfast/substreams/pipeline/manifest")
 
-func init() {
-	zlog, _ = logging.PackageLogger("pipeline", "github.com/streamingfast/substreams/pipeline/manifest")
-}