add graphnode and graphdeploy sample

sink-server/docker/docker.go

@@ -580,27 +580,32 @@ func (e *DockerEngine) createSubgraphManifest(deploymentID string, token string,
 	//		return nil, nil, nil, nil, fmt.Errorf("cannot unmarshal sinkconfig: %w", err)
 	//	}
 
-	//// note: graphnode is the sink
-	//graphnode, graphnodeMotd, err := e.newGraphnode(deploymentID, pkg, pg.Name, ipfs.Name)
-	////pg, pgMotd, err := e.newPostgres(deploymentID, pkg)
-	//if err != nil {
-	//	return nil, nil, nil, nil, fmt.Errorf("creating postgres deployment: %w", err)
-	//}
-	//services[graphnode.Name] = graphnodeMotd
+	graphnode, graphnodeMotd, err := e.newGraphNode(deploymentID, pg.Name, ipfs.Name, pkg)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("creating graphnode deployment: %w", err)
+	}
+	services[graphnode.Name] = graphnodeMotd
+
+	graphdeploy, graphDeployMotd, err := e.newGraphDeploy(deploymentID, ipfs.Name, graphnode.Name, pkg)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("creating graphdeploy deployment: %w", err)
+	}
+	services[graphdeploy.Name] = graphDeployMotd
 
 	config := types.Config{
 		Version: "3",
 		Services: []types.ServiceConfig{
 			pg,
 			ipfs,
-			//graphnode,
+			graphnode,
+			graphdeploy,
 		},
 	}
 
 	//if sqlSvc.PgwebFrontend != nil && sqlSvc.PgwebFrontend.Enabled {
-	//	pgweb, motd := e.newPGWeb(deploymentID, pg.Name)
-	//	services[pgweb.Name] = motd
-	//	config.Services = append(config.Services, pgweb)
+	pgweb, motd := e.newPGWeb(deploymentID, pg.Name)
+	services[pgweb.Name] = motd
+	config.Services = append(config.Services, pgweb)
 	//}
 
 	//if sqlSvc.PostgraphileFrontend != nil && sqlSvc.PostgraphileFrontend.Enabled {

sink-server/docker/graphdeploy.go

@@ -0,0 +1,203 @@
+package docker
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/cli/cli/compose/types"
+	pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
+	"google.golang.org/protobuf/proto"
+	"gopkg.in/yaml.v3"
+)
+
+func (e *DockerEngine) newGraphDeploy(deploymentID string, ipfsService string, graphnodeService string, pkg *pbsubstreams.Package) (conf types.ServiceConfig, motd string, err error) {
+
+	name := graphdeployServiceName(deploymentID)
+
+	configFolder := filepath.Join(e.dir, deploymentID, "config", "graphdeploy")
+	if err := os.MkdirAll(configFolder, 0755); err != nil {
+		return conf, motd, fmt.Errorf("creating folder %q: %w", configFolder, err)
+	}
+
+	dataFolder := filepath.Join(e.dir, deploymentID, "data", "graphdeploy")
+	if err := os.MkdirAll(dataFolder, 0755); err != nil {
+		return conf, motd, fmt.Errorf("creating folder %q: %w", dataFolder, err)
+	}
+
+	conf = types.ServiceConfig{
+		Name:          name,
+		ContainerName: name,
+		Image:         "node:20",
+		Restart:       "on-failure",
+		Entrypoint: []string{
+			"/opt/subservices/config/start.sh",
+		},
+		Volumes: []types.ServiceVolumeConfig{
+			{
+				Type:   "bind",
+				Source: "./data/graphdeploy",
+				Target: "/opt/subservices/data",
+			},
+			{
+				Type:   "bind",
+				Source: "./config/graphdeploy",
+				Target: "/opt/subservices/config",
+			},
+		},
+		Links:     []string{ipfsService + ":ipfs", graphnodeService + ":graphnode"},
+		DependsOn: []string{ipfsService, graphnodeService},
+	}
+
+	motd = fmt.Sprintf("Graph deploy service (no exposed port). Use 'docker logs %s' to see the logs.", name)
+
+	pkgContent, err := proto.Marshal(pkg)
+	if err != nil {
+		return conf, motd, fmt.Errorf("marshalling package: %w", err)
+	}
+
+	pkgName := pkg.PackageMeta[0].Name
+	pkgVersion := pkg.PackageMeta[0].Version
+
+	spkgName := fmt.Sprintf("%s-%s.spkg", pkgName, pkgVersion)
+
+	if err := os.WriteFile(filepath.Join(configFolder, spkgName), pkgContent, 0644); err != nil {
+		return conf, motd, fmt.Errorf("writing file: %w", err)
+	}
+
+	//FIXME
+	schemaGraphql := []byte(`
+type approvals @entity {
+    id: ID!
+    evt_tx_hash: String!
+    evt_index: Int!
+    evt_block_time: String!
+    evt_block_number: Int!
+    approved: String!
+    owner: String!
+    token_id: BigDecimal!
+}
+type approval_for_alls @entity {
+    id: ID!
+    evt_tx_hash: String!
+    evt_index: Int!
+    evt_block_time: String!
+    evt_block_number: Int!
+    approved: Boolean!
+    operator: String!
+    owner: String!
+}
+type mints @entity {
+    id: ID!
+    evt_tx_hash: String!
+    evt_index: Int!
+    evt_block_time: String!
+    evt_block_number: Int!
+    u_project_id: BigDecimal!
+    u_to: String!
+    u_token_id: BigDecimal!
+}
+type transfers @entity {
+    id: ID!
+    evt_tx_hash: String!
+    evt_index: Int!
+    evt_block_time: String!
+    evt_block_number: Int!
+    from: String!
+    to: String!
+    token_id: BigDecimal!
+}
+`)
+	if err := os.WriteFile(filepath.Join(configFolder, "schema.graphql"), schemaGraphql, 0644); err != nil {
+		return conf, motd, fmt.Errorf("writing file: %w", err)
+	}
+
+	//FIXME
+	substreamsYaml := []byte(`specVersion: 0.0.6
+description: Substreams powered art-blocks
+repository: https://github.com/streamingfast/substreams-generated-library
+schema:
+  file: ./schema.graphql
+
+dataSources:
+  - kind: substreams
+    name: art_blocks_graph
+    network: mainnet
+    source:
+      package:
+        moduleName: graph_out
+        file: substreams.spkg
+    mapping:
+      kind: substreams/graph-entities
+      apiVersion: 0.0.5`)
+
+	sgyaml := &yaml.Node{}
+	yaml.Unmarshal(substreamsYaml, sgyaml)
+
+	dataSources := getChild(sgyaml.Content[0], "dataSources")
+	var found bool
+	for _, ds := range dataSources.Content {
+		// modify the yaml to contain the right substreams.spkg file name
+		file := getChild(ds, "source", "package", "file")
+		if file != nil {
+			found = true
+			file.SetString(spkgName)
+		}
+	}
+	if !found {
+		return conf, "", fmt.Errorf("invalid input subgraph.yaml: cannot find the dataSources[].source.package.file to point to the correct file")
+	}
+
+	f, err := os.Create(filepath.Join(configFolder, "subgraph.yaml"))
+	if err != nil {
+		return conf, "", err
+	}
+	defer f.Close()
+	yaml.NewEncoder(f).Encode(sgyaml.Content[0])
+
+	startScript := []byte(fmt.Sprintf(`#!/bin/bash
+set -xeu
+
+if [ ! -f /opt/subservices/data/setup-complete ]; then
+    cd /opt/subservices/config
+    npm install -g @graphprotocol/graph-cli
+    graph create -g http://graphnode:8020 %s
+    graph deploy %s subgraph.yaml --ipfs=http://ipfs:5001 --node=http://graphnode:8020 --version-label=%s
+fi
+
+touch /opt/subservices/data/setup-complete
+sleep 999999
+
+`, pkgName, pkgName, pkgVersion))
+	if err := os.WriteFile(filepath.Join(configFolder, "start.sh"), startScript, 0755); err != nil {
+		fmt.Println("")
+		return conf, motd, fmt.Errorf("writing file: %w", err)
+	}
+
+	return conf, motd, nil
+}
+
+func graphdeployServiceName(deploymentID string) string {
+	return deploymentID + "-graphdeploy"
+}
+
+// getChild only follows the first object of a sequence, it does not thoroughly recurse all branches
+func getChild(parent *yaml.Node, name ...string) *yaml.Node {
+	var foundName bool
+	for _, child := range parent.Content {
+		if foundName {
+			if len(name) == 1 {
+				return child
+			}
+
+			if child.Kind == yaml.SequenceNode {
+				return getChild(child.Content[0], name[1:]...)
+			}
+			return getChild(child, name[1:]...)
+		}
+		if child.Value == name[0] {
+			foundName = true
+		}
+	}
+	return nil
+}

sink-server/docker/graphnode.go

@@ -0,0 +1,100 @@
+package docker
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/cli/cli/compose/types"
+	pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
+	"google.golang.org/protobuf/proto"
+)
+
+func (e *DockerEngine) newGraphNode(deploymentID string, pgService string, ipfsService string, pkg *pbsubstreams.Package) (conf types.ServiceConfig, motd string, err error) {
+
+	name := graphServiceName(deploymentID)
+
+	configFolder := filepath.Join(e.dir, deploymentID, "config", "graphnode")
+	if err := os.MkdirAll(configFolder, 0755); err != nil {
+		return conf, motd, fmt.Errorf("creating folder %q: %w", configFolder, err)
+	}
+
+	conf = types.ServiceConfig{
+		Name:          name,
+		ContainerName: name,
+		Image:         "graphprotocol/graph-node:v0.32.0",
+		Restart:       "on-failure",
+		Entrypoint:    []string{"graph-node", "--ipfs=ipfs:5001", "--node-id=index_node_0"},
+		Ports: []types.ServicePortConfig{
+			{Published: 8000, Target: 8000},
+			{Published: 8001, Target: 8001},
+			{Published: 8020, Target: 8020},
+			{Published: 8030, Target: 8030},
+			{Published: 8040, Target: 8040},
+		},
+
+		Volumes: []types.ServiceVolumeConfig{
+			{
+				Type:   "bind",
+				Source: "./config/graphnode",
+				Target: "/opt/subservices/config",
+			},
+		},
+		Links:     []string{pgService + ":postgres", ipfsService + ":ipfs"},
+		DependsOn: []string{pgService},
+		Environment: map[string]*string{
+			"DSN":                       deref("postgres://dev-node:insecure-change-me-in-prod@postgres:5432/dev-node?sslmode=disable"),
+			"OUTPUT_MODULE":             &pkg.SinkModule,
+			"FIREHOSE_API_TOKEN":        &e.token,
+			"SUBSTREAMS_API_TOKEN":      &e.token,
+			"GRAPH_NODE_CONFIG":         deref("/opt/subservices/config/graph-node.toml"),
+			"GRAPH_LOG":                 deref("debug"),
+			"ipfs":                      deref("http://localhost:5001"),
+			"GRAPH_MAX_GAS_PER_HANDLER": deref("1_000_000_000_000_000"),
+		},
+	}
+
+	motd = fmt.Sprintf("Graph-node service, listening on https://localhost:8000 (see the logs with 'docker logs %s')", name)
+
+	pkgContent, err := proto.Marshal(pkg)
+	if err != nil {
+		return conf, motd, fmt.Errorf("marshalling package: %w", err)
+	}
+
+	if err := os.WriteFile(filepath.Join(configFolder, "substreams.spkg"), pkgContent, 0644); err != nil {
+		return conf, motd, fmt.Errorf("writing file: %w", err)
+	}
+
+	configFile := []byte(`[general]
+
+[store]
+[store.primary]
+weight = 1
+connection = "$DSN"
+pool_size = 10
+
+[deployment]
+[[deployment.rule]]
+store = "primary"
+indexers = [ "index_node_0" ]
+
+[chains]
+ingestor = "index_node_0"
+
+[chains.mainnet]
+shard = "primary"
+provider = [
+    { label = "mainnet-firehose", details = { type = "firehose", url = "https://mainnet.eth.streamingfast.io:443", token = "$FIREHOSE_API_TOKEN", features = ["compression", "filters"], conn_pool_size = 1 }},
+    { label = "mainnet-substreams", details = { type = "substreams", url = "https://mainnet.eth.streamingfast.io:443", token = "$SUBSTREAMS_API_TOKEN", conn_pool_size = 1 }},
+]
+`)
+	if err := os.WriteFile(filepath.Join(configFolder, "graph-node.toml"), configFile, 0644); err != nil {
+		return conf, motd, fmt.Errorf("writing file: %w", err)
+	}
+
+	return conf, motd, nil
+}
+
+func graphServiceName(deploymentID string) string {
+	return deploymentID + "-graphnode"
+}