remove overrides, force validations on Read,

make `skip-package-validation` option only on run/gui/inspect/info

cmd/substreams/codegen.go

@@ -43,11 +43,7 @@ func runCodeGen(cmd *cobra.Command, args []string) error {
 
 	var protoDefinitions []*desc.FileDescriptor
 
-	readerOpts := append(getReaderOpts(cmd), manifest.SkipSourceCodeReader(), manifest.WithCollectProtoDefinitions(func(pd []*desc.FileDescriptor) {
-		protoDefinitions = pd
-	}))
-
-	manifestReader, err := manifest.NewReader(manifestPath, readerOpts...)
+	manifestReader, err := manifest.NewReader(manifestPath, manifest.SkipSourceCodeReader(), manifest.WithCollectProtoDefinitions(func(pd []*desc.FileDescriptor) { protoDefinitions = pd }))
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
 	}

cmd/substreams/graph.go

@@ -32,7 +32,7 @@ func runManifestGraph(cmd *cobra.Command, args []string) error {
 		manifestPath = args[0]
 	}
 
-	manifestReader, err := manifest.NewReader(manifestPath, getReaderOpts(cmd)...)
+	manifestReader, err := manifest.NewReader(manifestPath)
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
 	}

cmd/substreams/gui.go

@@ -10,6 +10,7 @@ import (
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/spf13/cobra"
 	"github.com/streamingfast/cli"
+	"github.com/streamingfast/cli/sflags"
 	"github.com/streamingfast/substreams/client"
 	"github.com/streamingfast/substreams/manifest"
 	"github.com/streamingfast/substreams/tools"
@@ -30,8 +31,10 @@ func init() {
 	guiCmd.Flags().StringSlice("debug-modules-initial-snapshot", nil, "List of 'store' modules from which to print the initial data snapshot (Unavailable in Production Mode")
 	guiCmd.Flags().StringSlice("debug-modules-output", nil, "List of extra modules from which to print outputs, deltas and logs (Unavailable in Production Mode)")
 	guiCmd.Flags().Bool("production-mode", false, "Enable Production Mode, with high-speed parallel processing")
+	guiCmd.Flags().Bool("skip-package-validation", false, "Do not perform any validation when reading substreams package")
 	guiCmd.Flags().StringArrayP("params", "p", nil, "Set a params for parameterizable modules. Can be specified multiple times. Ex: -p module1=valA -p module2=valX&valY")
 	guiCmd.Flags().Bool("replay", false, "Replay saved session into GUI from replay.bin")
+	guiCmd.Flags().Bool("skip-package-validation", false, "Do not perform any validation when reading substreams package")
 	rootCmd.AddCommand(guiCmd)
 }
 
@@ -78,11 +81,23 @@ func runGui(cmd *cobra.Command, args []string) error {
 	debugModulesInitialSnapshot := mustGetStringSlice(cmd, "debug-modules-initial-snapshot")
 
 	outputModule := args[0]
+	network := sflags.MustGetString(cmd, "network")
+	params := sflags.MustGetStringArray(cmd, "params")
 
-	manifestReader, err := manifest.NewReader(manifestPath, getReaderOpts(cmd)...)
+	readerOptions := []manifest.Option{
+		manifest.WithOverrideOutputModule(outputModule),
+		manifest.WithOverrideNetwork(network),
+		manifest.WithParams(params),
+	}
+	if sflags.MustGetBool(cmd, "skip-package-validation") {
+		readerOptions = append(readerOptions, manifest.SkipPackageValidationReader())
+	}
+
+	manifestReader, err := manifest.NewReader(manifestPath, readerOptions...)
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
 	}
+
 	pkg, err := manifestReader.Read()
 	if err != nil {
 		return fmt.Errorf("read manifest %q: %w", manifestPath, err)
@@ -100,11 +115,6 @@ func runGui(cmd *cobra.Command, args []string) error {
 		mustGetBool(cmd, "plaintext"),
 	)
 
-	params := mustGetStringArray(cmd, "params")
-	if err := manifest.ApplyParams(params, pkg); err != nil {
-		return err
-	}
-
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		homeDir = "."
@@ -134,7 +144,7 @@ func runGui(cmd *cobra.Command, args []string) error {
 	if readFromModule { // need to tweak the stop block here
 		graph, err := manifest.NewModuleGraph(pkg.Modules.Modules)
 		if err != nil {
-			return fmt.Errorf("creating module graph: %w", err)
+			return err
 		}
 		sb, err := graph.ModuleInitialBlock(outputModule)
 		if err != nil {
@@ -163,6 +173,8 @@ func runGui(cmd *cobra.Command, args []string) error {
 		StopBlock:                   stopBlock,
 		FinalBlocksOnly:             mustGetBool(cmd, "final-blocks-only"),
 		Params:                      params,
+		ReaderOptions:               readerOptions,
+		OverrideNetwork:             network,
 	}
 
 	ui, err := tui2.New(requestConfig)

cmd/substreams/info.go

@@ -16,6 +16,7 @@ import (
 
 func init() {
 	infoCmd.Flags().String("output-sinkconfig-files-path", "", "if non-empty, any sinkconfig field of type 'bytes' that was packed from a file will be written to that path")
+	infoCmd.Flags().Bool("skip-package-validation", false, "Do not perform any validation when reading substreams package")
 }
 
 var infoCmd = &cobra.Command{

cmd/substreams/inspect.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 
 	"github.com/spf13/cobra"
+	"github.com/streamingfast/cli/sflags"
 	"github.com/streamingfast/substreams/manifest"
 	"google.golang.org/protobuf/proto"
 )
@@ -22,12 +23,18 @@ var inspectCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(inspectCmd)
+	inspectCmd.Flags().Bool("skip-package-validation", false, "Do not perform any validation when reading substreams package")
 }
 
 func runInspect(cmd *cobra.Command, args []string) error {
 	manifestPath := args[0]
 
-	manifestReader, err := manifest.NewReader(manifestPath, getReaderOpts(cmd)...)
+	var readerOptions []manifest.Option
+	if sflags.MustGetBool(cmd, "skip-package-validation") {
+		readerOptions = append(readerOptions, manifest.SkipPackageValidationReader())
+	}
+
+	manifestReader, err := manifest.NewReader(manifestPath, readerOptions...)
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
 	}
@@ -37,10 +44,6 @@ func runInspect(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("reading manifest %q: %w", manifestPath, err)
 	}
 
-	if _, err = manifest.NewModuleGraph(pkg.Modules.Modules); err != nil {
-		return fmt.Errorf("processing module graph %w", err)
-	}
-
 	filename := filepath.Join(os.TempDir(), "package.spkg")
 
 	cnt, err := proto.Marshal(pkg)

cmd/substreams/pack.go

@@ -45,25 +45,7 @@ func runPack(cmd *cobra.Command, args []string) error {
 		manifestPath = args[0]
 	}
 
-	manifestReaderOptions := getReaderOpts(cmd)
-
-	//// Get the value of the -c flag
-	//overridePaths, _ := cmd.Flags().GetStringArray("config")
-	//// If the overridePath is provided, read, decode it and add to manifestReaderOptions
-	//if len(overridePaths) > 0 {
-	//	var overrides []*manifest.ConfigurationOverride
-	//	for _, overridePath := range overridePaths {
-	//		overrideConfig, err := readOverrideConfig(overridePath)
-	//		if err != nil {
-	//			return fmt.Errorf("reading override config %q: %w", overridePath, err)
-	//		}
-	//		overrides = append(overrides, overrideConfig)
-	//	}
-	//	manifestReaderOptions = append(manifestReaderOptions, manifest.WithOverrides(overrides...))
-	//}
-
-	// Use the manifestReaderOptions while creating the manifest reader
-	manifestReader, err := manifest.NewReader(manifestPath, manifestReaderOptions...)
+	manifestReader, err := manifest.NewReader(manifestPath)
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
 	}
@@ -77,10 +59,6 @@ func runPack(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("reading manifest %q: %w", manifestPath, err)
 	}
 
-	if _, err = manifest.NewModuleGraph(pkg.Modules.Modules); err != nil {
-		return fmt.Errorf("processing module graph %w", err)
-	}
-
 	originalOutputFile := maybeGetString(cmd, "output-file")
 	resolvedOutputFile := resolveOutputFile(originalOutputFile, map[string]string{
 		"manifestDir":     filepath.Dir(manifestPath),

cmd/substreams/protogen.go

@@ -53,8 +53,11 @@ func runProtogen(cmd *cobra.Command, args []string) error {
 		manifestPath = args[0]
 	}
 
-	readerOpts := append(getReaderOpts(cmd), manifest.SkipSourceCodeReader(), manifest.SkipModuleOutputTypeValidationReader())
-	manifestReader, err := manifest.NewReader(manifestPath, readerOpts...)
+	readerOptions := []manifest.Option{
+		manifest.SkipSourceCodeReader(),
+		manifest.SkipModuleOutputTypeValidationReader(),
+	}
+	manifestReader, err := manifest.NewReader(manifestPath, readerOptions...)
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
 	}
@@ -71,10 +74,6 @@ func runProtogen(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("reading manifest %q: %w", manifestPath, err)
 	}
 
-	if _, err = manifest.NewModuleGraph(pkg.Modules.Modules); err != nil {
-		return fmt.Errorf("processing module graph %w", err)
-	}
-
 	generator := codegen.NewProtoGenerator(outputPath, excludePaths, generateMod)
 	return generator.GenerateProto(pkg, showGeneratedBufGen)
 }

cmd/substreams/root.go

@@ -4,8 +4,6 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
-	"github.com/streamingfast/cli/sflags"
-	"github.com/streamingfast/substreams/manifest"
 	"go.uber.org/zap/zapcore"
 )
 
@@ -24,13 +22,4 @@ func init() {
 	// From https://thegraph.com/docs/en/operating-graph-node/
 	rootCmd.PersistentFlags().String("ipfs-url", "https://ipfs.network.thegraph.com", "IPFS endpoint to resolve substreams-based subgraphs as manifest")
 	rootCmd.PersistentFlags().Duration("ipfs-timeout", time.Second*10, "IPFS timeout when resolving substreams-based subgraphs as manifest")
-	rootCmd.PersistentFlags().Bool("skip-package-validation", false, "Do not perform any validation when reading substreams package")
-}
-
-func getReaderOpts(cmd *cobra.Command) (out []manifest.Option) {
-	if sflags.MustGetBool(cmd, "skip-package-validation") {
-		out = append(out, manifest.SkipPackageValidationReader())
-	} else {
-	}
-	return
 }

cmd/substreams/run.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/streamingfast/cli"
+	"github.com/streamingfast/cli/sflags"
 	"github.com/streamingfast/substreams/client"
 	"github.com/streamingfast/substreams/manifest"
 	pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2"
@@ -20,6 +21,7 @@ import (
 func init() {
 	runCmd.Flags().StringP("substreams-endpoint", "e", "", "Substreams gRPC endpoint. If empty, will be replaced by the SUBSTREAMS_ENDPOINT_{network_name} environment variable, where `network_name` is determined from the substreams manifest. Some network names have default endpoints.")
 	runCmd.Flags().String("substreams-api-token-envvar", "SUBSTREAMS_API_TOKEN", "name of variable containing Substreams Authentication token")
+	runCmd.Flags().String("network", "", "name of variable containing Substreams Authentication token")
 	runCmd.Flags().StringP("start-block", "s", "", "Start block to stream from. If empty, will be replaced by initialBlock of the first module you are streaming. If negative, will be resolved by the server relative to the chain head")
 	runCmd.Flags().StringP("cursor", "c", "", "Cursor to stream from. Leave blank for no cursor")
 	runCmd.Flags().StringP("stop-block", "t", "0", "Stop block to end stream at, exclusively. If the start-block is positive, a '+' prefix can indicate 'relative to start-block'")
@@ -31,6 +33,7 @@ func init() {
 	runCmd.Flags().StringSlice("debug-modules-output", nil, "List of modules from which to print outputs, deltas and logs (Unavailable in Production Mode)")
 	runCmd.Flags().StringSliceP("header", "H", nil, "Additional headers to be sent in the substreams request")
 	runCmd.Flags().Bool("production-mode", false, "Enable Production Mode, with high-speed parallel processing")
+	runCmd.Flags().Bool("skip-package-validation", false, "Do not perform any validation when reading substreams package")
 	runCmd.Flags().StringArrayP("params", "p", nil, "Set a params for parameterizable modules. Can be specified multiple times. Ex: -p module1=valA -p module2=valX&valY")
 	runCmd.Flags().String("test-file", "", "runs a test file")
 	runCmd.Flags().Bool("test-verbose", false, "print out all the results")
@@ -69,7 +72,19 @@ func runRun(cmd *cobra.Command, args []string) error {
 
 	outputMode := mustGetString(cmd, "output")
 
-	manifestReader, err := manifest.NewReader(manifestPath, getReaderOpts(cmd)...)
+	network := sflags.MustGetString(cmd, "network")
+	params := sflags.MustGetStringArray(cmd, "params")
+
+	readerOptions := []manifest.Option{
+		manifest.WithOverrideOutputModule(outputModule),
+		manifest.WithOverrideNetwork(network),
+		manifest.WithParams(params),
+	}
+	if sflags.MustGetBool(cmd, "skip-package-validation") {
+		readerOptions = append(readerOptions, manifest.SkipPackageValidationReader())
+	}
+
+	manifestReader, err := manifest.NewReader(manifestPath, readerOptions...)
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
 	}
@@ -84,10 +99,6 @@ func runRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("extracting endpoint: %w", err)
 	}
 
-	if err := manifest.ApplyParams(mustGetStringArray(cmd, "params"), pkg); err != nil {
-		return err
-	}
-
 	msgDescs, err := manifest.BuildMessageDescriptors(pkg)
 	if err != nil {
 		return fmt.Errorf("building message descriptors: %w", err)
@@ -111,17 +122,16 @@ func runRun(cmd *cobra.Command, args []string) error {
 
 	debugModulesInitialSnapshot := mustGetStringSlice(cmd, "debug-modules-initial-snapshot")
 
-	graph, err := manifest.NewModuleGraph(pkg.Modules.Modules)
-	if err != nil {
-		return fmt.Errorf("creating module graph: %w", err)
-	}
-
 	startBlock, readFromModule, err := readStartBlockFlag(cmd, "start-block")
 	if err != nil {
 		return fmt.Errorf("stop block: %w", err)
 	}
 
 	if readFromModule {
+		graph, err := manifest.NewModuleGraph(pkg.Modules.Modules)
+		if err != nil {
+			return err
+		}
 		sb, err := graph.ModuleInitialBlock(outputModule)
 		if err != nil {
 			return fmt.Errorf("getting module start block: %w", err)

cmd/substreams/service-deploy.go

@@ -37,7 +37,7 @@ func deployE(cmd *cobra.Command, args []string) error {
 
 	file := args[0]
 
-	reader, err := manifest.NewReader(file, getReaderOpts(cmd)...)
+	reader, err := manifest.NewReader(file)
 	if err != nil {
 		return err
 	}

cmd/substreams/service-update.go

@@ -35,7 +35,7 @@ var updateCmd = &cobra.Command{
 func updateE(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 
-	reader, err := manifest.NewReader(args[0], getReaderOpts(cmd)...)
+	reader, err := manifest.NewReader(args[0])
 	if err != nil {
 		return err
 	}