merge codegen-cmd into develop

README.md

@@ -4,8 +4,6 @@
 
 # Substreams
 
-> Developer preview
-
 Substreams is a powerful blockchain indexing technology, developed for The Graph Network.
 
 Substreams enables developers to write Rust modules, composing data streams alongside the community, and provides extremely high performance indexing by virtue of parallelization, in a streaming-first fashion.

cmd/substreams/auth.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+
+	"github.com/spf13/cobra"
+)
+
+var authCmd = &cobra.Command{
+	Use:          "auth",
+	Short:        "Login command for Substreams development",
+	RunE:         runAuthE,
+	SilenceUsage: true,
+}
+
+func init() {
+	rootCmd.AddCommand(authCmd)
+}
+
+func runAuthE(cmd *cobra.Command, args []string) error {
+	localDevelopment := os.Getenv("LOCAL_DEVELOPMENT")
+
+	fmt.Println("Open this link to authenticate on The Graph Market:")
+	if localDevelopment == "true" {
+		color.Blue("http://localhost:3000/auth/substreams-devenv")
+	} else {
+		color.Blue("https://thegraph.market/auth/substreams-devenv")
+	}
+
+	fmt.Println()
+	fmt.Print("Then paste the token here: ")
+
+	var token string
+	_, err := fmt.Scanln(&token)
+	if err != nil {
+		return fmt.Errorf("error reading token: %w", err)
+	}
+
+	if token == "" {
+		return fmt.Errorf("token cannot be empty")
+	}
+
+	fmt.Println()
+	fmt.Println("Writing `./.substreams.env`")
+	fmt.Println()
+	fmt.Println("Please add `.substreams.env` to your `.gitignore`.")
+	fmt.Println()
+
+	err = os.WriteFile(".substreams.env", []byte(fmt.Sprintf("export SUBSTREAMS_API_TOKEN=%s\n", token)), 0644)
+	if err != nil {
+		return fmt.Errorf("writing .substreams.env file: %w", err)
+	}
+
+	fmt.Println("Load credentials in current terminal with:")
+	color.Blue(" . ./.substreams.env")
+
+	return nil
+}

cmd/substreams/codegen.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/streamingfast/substreams/codegen"
+
+func init() {
+	rootCmd.AddCommand(codegen.Cmd)
+}

cmd/substreams/gui.go

@@ -41,10 +41,10 @@ func init() {
 
 // guiCmd represents the command to run substreams remotely
 var guiCmd = &cobra.Command{
-	Use:   "gui [<manifest>] <module_name>",
+	Use:   "gui [<manifest>] [<module_name>]",
 	Short: "Stream module outputs from a given package on a remote endpoint",
 	Long: cli.Dedent(`
-		Stream module outputs from a given package on a remote endpoint. The manifest is optional as it will try to find a file named
+		Stream module output from a given package on a remote endpoint. The manifest is optional as it will try to find a file named
 		'substreams.yaml' in current working directory if nothing entered. You may enter a directory that contains a 'substreams.yaml'
 		file in place of '<manifest_file>, or a link to a remote .spkg file, using urls gs://, http(s)://, ipfs://, etc.'.
 	`),

cmd/substreams/init.go

@@ -17,7 +17,7 @@ import (
 	"strings"
 	"time"
 
-	connect "connectrpc.com/connect"
+	"connectrpc.com/connect"
 	"github.com/charmbracelet/glamour"
 	"github.com/charmbracelet/huh"
 	"github.com/charmbracelet/huh/spinner"
@@ -49,7 +49,11 @@ var initCmd = &cobra.Command{
 }
 
 func init() {
-	initCmd.Flags().String("discovery-endpoint", "https://codegen.substreams.dev", "Endpoint used to discover code generators")
+	defaultEndpoint := "https://codegen.substreams.dev"
+	if newValue := os.Getenv("SUBSTREAMS_INIT_CODEGEN_ENDPOINT"); newValue != "" {
+		defaultEndpoint = newValue
+	}
+	initCmd.Flags().String("codegen-endpoint", defaultEndpoint, "Endpoint used to discover code generators")
 	initCmd.Flags().String("state-file", "./generator.json", "File to load/save the state of the code generator")
 	rootCmd.AddCommand(initCmd)
 }
@@ -87,7 +91,7 @@ func runSubstreamsInitE(cmd *cobra.Command, args []string) error {
 		connect.WithGRPC(),
 	}
 
-	initConvoURL := sflags.MustGetString(cmd, "discovery-endpoint")
+	initConvoURL := sflags.MustGetString(cmd, "codegen-endpoint")
 	stateFile, stateFileFlagProvided := sflags.MustGetStringProvided(cmd, "state-file")
 	if !strings.HasSuffix(stateFile, ".json") {
 		return fmt.Errorf("state file must have a .json extension")
@@ -424,7 +428,6 @@ func runSubstreamsInitE(cmd *cobra.Command, args []string) error {
 			}
 
 		case *pbconvo.SystemOutput_DownloadFiles_:
-
 			if userState.downloadedFilesfolderPath == "" {
 				savingDest := "output"
 				if projectName := gjson.GetBytes(lastState.State, "name").String(); projectName != "" {
@@ -459,19 +462,20 @@ func runSubstreamsInitE(cmd *cobra.Command, args []string) error {
 				}
 
 				// the multiple \n are not a mistake, it's to have a blank line before the next message
-				fmt.Printf("\nProject will be saved in %s\n", savingDest)
+				fmt.Printf("\nProject will be saved in %s\n\n", savingDest)
 				userState.downloadedFilesfolderPath = savingDest
 			}
 
 			input := msg.DownloadFiles
-			fmt.Println("Files:")
+			fmt.Println(filenameStyle("Files:\n"))
 			for _, file := range input.Files {
-				fmt.Printf("  - %s (%s)\n", filepath.Join(userState.downloadedFilesfolderPath, file.Filename), file.Type)
-				if file.Description != "" {
-					fmt.Println(file.Description)
+				if file.Content == nil {
+					continue
 				}
-			}
 
+				fmt.Printf("%s %s\n", filenameStyle("-"), filenameStyle(file.Filename))
+				fmt.Printf("  %s\n\n", file.Description)
+			}
 			// let the terminal breath a little
 			fmt.Println()
 
@@ -480,19 +484,17 @@ func runSubstreamsInitE(cmd *cobra.Command, args []string) error {
 			}
 
 			overwriteForm := NewOverwriteForm()
+
 			for _, inputFile := range input.Files {
 				switch inputFile.Type {
 				case "application/x-zip+extract": // our custom mime type to always extract the file upon arrival
-					zipRoot := userState.downloadedFilesfolderPath
-
-					sourcePath := filepath.Join(zipRoot, inputFile.Filename)
-					err = saveDownloadFile(sourcePath, overwriteForm, inputFile)
-					if err != nil {
-						return fmt.Errorf("saving zip file: %w", err)
+					if inputFile.Content == nil {
+						continue
 					}
 
+					zipRoot := userState.downloadedFilesfolderPath
+
 					zipContent := inputFile.Content
-					fmt.Printf("Unzipping %s into %s\n", inputFile.Filename, zipRoot)
 					err = unzipFile(overwriteForm, zipContent, zipRoot)
 					if err != nil {
 						return fmt.Errorf("unzipping file: %w", err)
@@ -502,6 +504,10 @@ func runSubstreamsInitE(cmd *cobra.Command, args []string) error {
 					// "application/x-protobuf; messageType=\"sf.substreams.v1.Package\""
 					// "application/zip", "application/x-zip"
 					// "text/plain":
+					if inputFile.Content == nil {
+						continue
+					}
+
 					fullPath := filepath.Join(userState.downloadedFilesfolderPath, inputFile.Filename)
 					err = saveDownloadFile(fullPath, overwriteForm, inputFile)
 					if err != nil {
@@ -580,6 +586,10 @@ func bold(input string) string {
 	return lipgloss.NewStyle().Bold(true).Render(input)
 }
 
+func filenameStyle(input string) string {
+	return lipgloss.NewStyle().Foreground(lipgloss.Color("#9AE3A4")).Render(input)
+}
+
 func unzipFile(overwriteForm *OverwriteForm, zipContent []byte, zipRoot string) error {
 	reader := bytes.NewReader(zipContent)
 	zipReader, err := zip.NewReader(reader, int64(len(zipContent)))

codegen/chainconfig.go

@@ -0,0 +1,94 @@
+package codegen
+
+type ChainConfig struct {
+	ID               string // Public
+	DisplayName      string // Public
+	ExplorerLink     string
+	ApiEndpoint      string
+	FirehoseEndpoint string
+	Network          string
+	SupportsCalls    bool
+}
+
+var ChainConfigByID = map[string]*ChainConfig{
+	"mainnet": {
+		DisplayName:      "Ethereum Mainnet",
+		ExplorerLink:     "https://etherscan.io",
+		ApiEndpoint:      "https://api.etherscan.io",
+		FirehoseEndpoint: "mainnet.eth.streamingfast.io:443",
+		Network:          "mainnet",
+		SupportsCalls:    true,
+	},
+	"bnb": {
+		DisplayName:      "BNB",
+		ExplorerLink:     "https://bscscan.com",
+		ApiEndpoint:      "https://api.bscscan.com",
+		FirehoseEndpoint: "bnb.streamingfast.io:443",
+		Network:          "bsc",
+		SupportsCalls:    true,
+	},
+	"polygon": {
+		DisplayName:      "Polygon",
+		ExplorerLink:     "https://polygonscan.com",
+		ApiEndpoint:      "https://api.polygonscan.com",
+		FirehoseEndpoint: "polygon.streamingfast.io:443",
+		Network:          "polygon",
+		SupportsCalls:    true,
+	},
+	"amoy": {
+		DisplayName:      "Polygon Amoy Testnet",
+		ExplorerLink:     "https://www.okx.com/web3/explorer/amoy",
+		ApiEndpoint:      "",
+		FirehoseEndpoint: "amoy.substreams.pinax.network:443",
+		Network:          "amoy",
+		SupportsCalls:    true,
+	},
+	"arbitrum": {
+		DisplayName:      "Arbitrum",
+		ExplorerLink:     "https://arbiscan.io",
+		ApiEndpoint:      "https://api.arbiscan.io",
+		FirehoseEndpoint: "arb-one.streamingfast.io:443",
+		Network:          "arbitrum",
+		SupportsCalls:    true,
+	},
+	"holesky": {
+		DisplayName:      "Holesky",
+		ExplorerLink:     "https://holesky.etherscan.io/",
+		ApiEndpoint:      "https://api-holesky.etherscan.io",
+		FirehoseEndpoint: "holesky.eth.streamingfast.io:443",
+		Network:          "holesky",
+		SupportsCalls:    true,
+	},
+	"sepolia": {
+		DisplayName:      "Sepolia Testnet",
+		ExplorerLink:     "https://sepolia.etherscan.io",
+		ApiEndpoint:      "https://api-sepolia.etherscan.io",
+		FirehoseEndpoint: "sepolia.streamingfast.io:443",
+		Network:          "sepolia",
+		SupportsCalls:    true,
+	},
+	"optimism": {
+		DisplayName:      "Optimism Mainnet",
+		ExplorerLink:     "https://optimistic.etherscan.io",
+		ApiEndpoint:      "https://api-optimistic.etherscan.io",
+		FirehoseEndpoint: "opt-mainnet.streamingfast.io:443",
+		Network:          "optimism",
+		SupportsCalls:    false,
+	},
+	"avalanche": {
+		DisplayName:      "Avalanche C-chain",
+		ExplorerLink:     "https://subnets.avax.network/c-chain",
+		ApiEndpoint:      "",
+		FirehoseEndpoint: "avalanche-mainnet.streamingfast.io:443",
+		Network:          "avalanche",
+		SupportsCalls:    false,
+	},
+	"chapel": {
+		DisplayName:      "BNB Chapel Testnet",
+		ExplorerLink:     "https://testnet.bscscan.com/",
+		ApiEndpoint:      "",
+		FirehoseEndpoint: "chapel.substreams.pinax.network:443",
+		Network:          "chapel",
+		SupportsCalls:    true,
+	},
+}

codegen/cmd.go

@@ -0,0 +1,14 @@
+package codegen
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var Cmd = &cobra.Command{Use: "codegen", Short: "Code generator for substreams"}
+
+func init() {
+	SubgraphCmd.Flags().Bool("with-dev-env", false, "generate graph node dev environment")
+
+	Cmd.AddCommand(SubgraphCmd)
+	Cmd.AddCommand(SQLCmd)
+}