diff --git a/sink-server/docker/docker.go b/sink-server/docker/docker.go index 4804fd29e..6506bd24b 100644 --- a/sink-server/docker/docker.go +++ b/sink-server/docker/docker.go @@ -580,27 +580,32 @@ func (e *DockerEngine) createSubgraphManifest(deploymentID string, token string, // return nil, nil, nil, nil, fmt.Errorf("cannot unmarshal sinkconfig: %w", err) // } - //// note: graphnode is the sink - //graphnode, graphnodeMotd, err := e.newGraphnode(deploymentID, pkg, pg.Name, ipfs.Name) - ////pg, pgMotd, err := e.newPostgres(deploymentID, pkg) - //if err != nil { - // return nil, nil, nil, nil, fmt.Errorf("creating postgres deployment: %w", err) - //} - //services[graphnode.Name] = graphnodeMotd + graphnode, graphnodeMotd, err := e.newGraphNode(deploymentID, pg.Name, ipfs.Name, pkg) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("creating graphnode deployment: %w", err) + } + services[graphnode.Name] = graphnodeMotd + + graphdeploy, graphDeployMotd, err := e.newGraphDeploy(deploymentID, ipfs.Name, graphnode.Name, pkg) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("creating graphdeploy deployment: %w", err) + } + services[graphdeploy.Name] = graphDeployMotd config := types.Config{ Version: "3", Services: []types.ServiceConfig{ pg, ipfs, - //graphnode, + graphnode, + graphdeploy, }, } //if sqlSvc.PgwebFrontend != nil && sqlSvc.PgwebFrontend.Enabled { - // pgweb, motd := e.newPGWeb(deploymentID, pg.Name) - // services[pgweb.Name] = motd - // config.Services = append(config.Services, pgweb) + pgweb, motd := e.newPGWeb(deploymentID, pg.Name) + services[pgweb.Name] = motd + config.Services = append(config.Services, pgweb) //} //if sqlSvc.PostgraphileFrontend != nil && sqlSvc.PostgraphileFrontend.Enabled { diff --git a/sink-server/docker/graphdeploy.go b/sink-server/docker/graphdeploy.go new file mode 100644 index 000000000..0eca18d6c --- /dev/null +++ b/sink-server/docker/graphdeploy.go @@ -0,0 +1,203 @@ +package docker + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/docker/cli/cli/compose/types" + pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" + "google.golang.org/protobuf/proto" + "gopkg.in/yaml.v3" +) + +func (e *DockerEngine) newGraphDeploy(deploymentID string, ipfsService string, graphnodeService string, pkg *pbsubstreams.Package) (conf types.ServiceConfig, motd string, err error) { + + name := graphdeployServiceName(deploymentID) + + configFolder := filepath.Join(e.dir, deploymentID, "config", "graphdeploy") + if err := os.MkdirAll(configFolder, 0755); err != nil { + return conf, motd, fmt.Errorf("creating folder %q: %w", configFolder, err) + } + + dataFolder := filepath.Join(e.dir, deploymentID, "data", "graphdeploy") + if err := os.MkdirAll(dataFolder, 0755); err != nil { + return conf, motd, fmt.Errorf("creating folder %q: %w", dataFolder, err) + } + + conf = types.ServiceConfig{ + Name: name, + ContainerName: name, + Image: "node:20", + Restart: "on-failure", + Entrypoint: []string{ + "/opt/subservices/config/start.sh", + }, + Volumes: []types.ServiceVolumeConfig{ + { + Type: "bind", + Source: "./data/graphdeploy", + Target: "/opt/subservices/data", + }, + { + Type: "bind", + Source: "./config/graphdeploy", + Target: "/opt/subservices/config", + }, + }, + Links: []string{ipfsService + ":ipfs", graphnodeService + ":graphnode"}, + DependsOn: []string{ipfsService, graphnodeService}, + } + + motd = fmt.Sprintf("Graph deploy service (no exposed port). Use 'docker logs %s' to see the logs.", name) + + pkgContent, err := proto.Marshal(pkg) + if err != nil { + return conf, motd, fmt.Errorf("marshalling package: %w", err) + } + + pkgName := pkg.PackageMeta[0].Name + pkgVersion := pkg.PackageMeta[0].Version + + spkgName := fmt.Sprintf("%s-%s.spkg", pkgName, pkgVersion) + + if err := os.WriteFile(filepath.Join(configFolder, spkgName), pkgContent, 0644); err != nil { + return conf, motd, fmt.Errorf("writing file: %w", err) + } + + //FIXME + schemaGraphql := []byte(` +type approvals @entity { + id: ID! + evt_tx_hash: String! + evt_index: Int! + evt_block_time: String! + evt_block_number: Int! + approved: String! + owner: String! + token_id: BigDecimal! +} +type approval_for_alls @entity { + id: ID! + evt_tx_hash: String! + evt_index: Int! + evt_block_time: String! + evt_block_number: Int! + approved: Boolean! + operator: String! + owner: String! +} +type mints @entity { + id: ID! + evt_tx_hash: String! + evt_index: Int! + evt_block_time: String! + evt_block_number: Int! + u_project_id: BigDecimal! + u_to: String! + u_token_id: BigDecimal! +} +type transfers @entity { + id: ID! + evt_tx_hash: String! + evt_index: Int! + evt_block_time: String! + evt_block_number: Int! + from: String! + to: String! + token_id: BigDecimal! +} +`) + if err := os.WriteFile(filepath.Join(configFolder, "schema.graphql"), schemaGraphql, 0644); err != nil { + return conf, motd, fmt.Errorf("writing file: %w", err) + } + + //FIXME + substreamsYaml := []byte(`specVersion: 0.0.6 +description: Substreams powered art-blocks +repository: https://github.com/streamingfast/substreams-generated-library +schema: + file: ./schema.graphql + +dataSources: + - kind: substreams + name: art_blocks_graph + network: mainnet + source: + package: + moduleName: graph_out + file: substreams.spkg + mapping: + kind: substreams/graph-entities + apiVersion: 0.0.5`) + + sgyaml := &yaml.Node{} + yaml.Unmarshal(substreamsYaml, sgyaml) + + dataSources := getChild(sgyaml.Content[0], "dataSources") + var found bool + for _, ds := range dataSources.Content { + // modify the yaml to contain the right substreams.spkg file name + file := getChild(ds, "source", "package", "file") + if file != nil { + found = true + file.SetString(spkgName) + } + } + if !found { + return conf, "", fmt.Errorf("invalid input subgraph.yaml: cannot find the dataSources[].source.package.file to point to the correct file") + } + + f, err := os.Create(filepath.Join(configFolder, "subgraph.yaml")) + if err != nil { + return conf, "", err + } + defer f.Close() + yaml.NewEncoder(f).Encode(sgyaml.Content[0]) + + startScript := []byte(fmt.Sprintf(`#!/bin/bash +set -xeu + +if [ ! -f /opt/subservices/data/setup-complete ]; then + cd /opt/subservices/config + npm install -g @graphprotocol/graph-cli + graph create -g http://graphnode:8020 %s + graph deploy %s subgraph.yaml --ipfs=http://ipfs:5001 --node=http://graphnode:8020 --version-label=%s +fi + +touch /opt/subservices/data/setup-complete +sleep 999999 + +`, pkgName, pkgName, pkgVersion)) + if err := os.WriteFile(filepath.Join(configFolder, "start.sh"), startScript, 0755); err != nil { + fmt.Println("") + return conf, motd, fmt.Errorf("writing file: %w", err) + } + + return conf, motd, nil +} + +func graphdeployServiceName(deploymentID string) string { + return deploymentID + "-graphdeploy" +} + +// getChild only follows the first object of a sequence, it does not thoroughly recurse all branches +func getChild(parent *yaml.Node, name ...string) *yaml.Node { + var foundName bool + for _, child := range parent.Content { + if foundName { + if len(name) == 1 { + return child + } + + if child.Kind == yaml.SequenceNode { + return getChild(child.Content[0], name[1:]...) + } + return getChild(child, name[1:]...) + } + if child.Value == name[0] { + foundName = true + } + } + return nil +} diff --git a/sink-server/docker/graphnode.go b/sink-server/docker/graphnode.go new file mode 100644 index 000000000..6c4850a44 --- /dev/null +++ b/sink-server/docker/graphnode.go @@ -0,0 +1,100 @@ +package docker + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/docker/cli/cli/compose/types" + pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" + "google.golang.org/protobuf/proto" +) + +func (e *DockerEngine) newGraphNode(deploymentID string, pgService string, ipfsService string, pkg *pbsubstreams.Package) (conf types.ServiceConfig, motd string, err error) { + + name := graphServiceName(deploymentID) + + configFolder := filepath.Join(e.dir, deploymentID, "config", "graphnode") + if err := os.MkdirAll(configFolder, 0755); err != nil { + return conf, motd, fmt.Errorf("creating folder %q: %w", configFolder, err) + } + + conf = types.ServiceConfig{ + Name: name, + ContainerName: name, + Image: "graphprotocol/graph-node:v0.32.0", + Restart: "on-failure", + Entrypoint: []string{"graph-node", "--ipfs=ipfs:5001", "--node-id=index_node_0"}, + Ports: []types.ServicePortConfig{ + {Published: 8000, Target: 8000}, + {Published: 8001, Target: 8001}, + {Published: 8020, Target: 8020}, + {Published: 8030, Target: 8030}, + {Published: 8040, Target: 8040}, + }, + + Volumes: []types.ServiceVolumeConfig{ + { + Type: "bind", + Source: "./config/graphnode", + Target: "/opt/subservices/config", + }, + }, + Links: []string{pgService + ":postgres", ipfsService + ":ipfs"}, + DependsOn: []string{pgService}, + Environment: map[string]*string{ + "DSN": deref("postgres://dev-node:insecure-change-me-in-prod@postgres:5432/dev-node?sslmode=disable"), + "OUTPUT_MODULE": &pkg.SinkModule, + "FIREHOSE_API_TOKEN": &e.token, + "SUBSTREAMS_API_TOKEN": &e.token, + "GRAPH_NODE_CONFIG": deref("/opt/subservices/config/graph-node.toml"), + "GRAPH_LOG": deref("debug"), + "ipfs": deref("http://localhost:5001"), + "GRAPH_MAX_GAS_PER_HANDLER": deref("1_000_000_000_000_000"), + }, + } + + motd = fmt.Sprintf("Graph-node service, listening on https://localhost:8000 (see the logs with 'docker logs %s')", name) + + pkgContent, err := proto.Marshal(pkg) + if err != nil { + return conf, motd, fmt.Errorf("marshalling package: %w", err) + } + + if err := os.WriteFile(filepath.Join(configFolder, "substreams.spkg"), pkgContent, 0644); err != nil { + return conf, motd, fmt.Errorf("writing file: %w", err) + } + + configFile := []byte(`[general] + +[store] +[store.primary] +weight = 1 +connection = "$DSN" +pool_size = 10 + +[deployment] +[[deployment.rule]] +store = "primary" +indexers = [ "index_node_0" ] + +[chains] +ingestor = "index_node_0" + +[chains.mainnet] +shard = "primary" +provider = [ + { label = "mainnet-firehose", details = { type = "firehose", url = "https://mainnet.eth.streamingfast.io:443", token = "$FIREHOSE_API_TOKEN", features = ["compression", "filters"], conn_pool_size = 1 }}, + { label = "mainnet-substreams", details = { type = "substreams", url = "https://mainnet.eth.streamingfast.io:443", token = "$SUBSTREAMS_API_TOKEN", conn_pool_size = 1 }}, +] +`) + if err := os.WriteFile(filepath.Join(configFolder, "graph-node.toml"), configFile, 0644); err != nil { + return conf, motd, fmt.Errorf("writing file: %w", err) + } + + return conf, motd, nil +} + +func graphServiceName(deploymentID string) string { + return deploymentID + "-graphnode" +}