sink-server: added support for rest_frontend for clickhouse sinks

streamingfast · Nov 6, 2023 · dafd69e · dafd69e
1 parent 8f6036b
commit dafd69e
Show file tree

Hide file tree

Showing 7 changed files with 53 additions and 4 deletions.
diff --git a/codegen/templates/ethereum/substreams.clickhouse.yaml b/codegen/templates/ethereum/substreams.clickhouse.yaml
@@ -58,3 +58,5 @@ sink:
       enabled: false
     pgweb_frontend:
       enabled: false
+    rest_frontend:
+      enabled: false
diff --git a/codegen/templates/ethereum/substreams.clickhouse.yaml.gotmpl b/codegen/templates/ethereum/substreams.clickhouse.yaml.gotmpl
@@ -58,3 +58,5 @@ sink:
       enabled: false
     pgweb_frontend:
       enabled: false
+    rest_frontend:
+      enabled: false
diff --git a/codegen/templates/ethereum_project.go b/codegen/templates/ethereum_project.go
@@ -104,7 +104,7 @@ func NewEthereumProject(name string, moduleName string, chain *EthereumChain, co
 		chain:                       chain,
 		ethereumContracts:           contracts,
 		creationBlockNum:            lowestStartBlock,
-		sqlImportVersion:            "1.0.3",
+		sqlImportVersion:            "1.0.4",
 		graphImportVersion:          "0.1.0",
 		databaseChangeImportVersion: "1.2.1",
 		entityChangeImportVersion:   "1.1.0",

diff --git a/go.mod b/go.mod
@@ -60,7 +60,7 @@ require (
 	github.com/streamingfast/eth-go v0.0.0-20230410173454-433bd8803da1
 	github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9
 	github.com/streamingfast/shutter v1.5.0
-	github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231024141225-6af37a8c8567
+	github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3
 	github.com/tetratelabs/wazero v1.1.0
 	github.com/tidwall/pretty v1.2.1
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0

diff --git a/go.sum b/go.sum
@@ -599,8 +599,8 @@ github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9 h1:YRwpVv
 github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9/go.mod h1:ktzt1BUj3GF+SKQHEmn3ShryJ7y87JeCHtaTGaDVATs=
 github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAtyaTOgs=
 github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8=
-github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231024141225-6af37a8c8567 h1:mm8rkuvFMOmBg9lT8S65I5F322ORDa/MdaNLE8THpAQ=
-github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231024141225-6af37a8c8567/go.mod h1:vi9cdOke79ezfUZVAJHJY7Y/nOsmHXkRh0++Ze9hZNk=
+github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3 h1:X+sIFo//ffQKHJbIAkhYa657gYZi/KPWf9phBwUzKrg=
+github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3/go.mod h1:vi9cdOke79ezfUZVAJHJY7Y/nOsmHXkRh0++Ze9hZNk=
 github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14 h1:/HCMj7plThWPFXsj/O8f+zDIAtXciwP4rK6tb+9UvPQ=
 github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14/go.mod h1:t4h97mWfTs6v0zjEFuGDOoW5wLtu9+ttegIx99i7gsM=
 github.com/streamingfast/wasmtime-go/v4 v4.0.0-freemem3 h1:raJHR0JWgYiSyX0vZ3leRK/TkNcn4ZUGTf+d64g48KQ=

diff --git a/sink-server/docker/docker.go b/sink-server/docker/docker.go
@@ -583,6 +583,12 @@ func (e *DockerEngine) createManifest(deploymentID string, token string, pkg *pb
 		}
 	}
 
+	if sinkConfig.RestFrontend != nil && sinkConfig.RestFrontend.Enabled {
+		rest, motd := e.newRestFrontend(deploymentID, dbServiceName)
+		servicesDesc[rest.Name] = motd
+		services = append(services, rest)
+	}
+
 	for _, svc := range services {
 		for _, port := range svc.Ports {
 			usedPorts = append(usedPorts, port.Published)

diff --git a/sink-server/docker/rest.go b/sink-server/docker/rest.go
@@ -0,0 +1,39 @@
+package docker
+
+import (
+	"fmt"
+
+	"github.com/docker/cli/cli/compose/types"
+)
+
+func (e *DockerEngine) newRestFrontend(deploymentID string, dbService string) (conf types.ServiceConfig, motd string) {
+
+	name := fmt.Sprintf("%s-rest", deploymentID)
+	localPort := uint32(3000) // TODO: assign dynamically
+
+	conf = types.ServiceConfig{
+		Name:          name,
+		ContainerName: name,
+		Image:         "docker.io/dfuse/sql-wrapper:latest",
+		Restart:       "on-failure",
+		Ports: []types.ServicePortConfig{
+			{
+				Published: localPort,
+				Target:    3000,
+			},
+		},
+		Links:     []string{dbService + ":clickhouse"},
+		DependsOn: []string{dbService},
+		Environment: map[string]*string{
+			"CLICKHOUSE_URL": deref("tcp://dev-node:insecure-change-me-in-prod@clickhouse:9000/substreams?secure=false&skip_verify=true&connection_timeout=20s"),
+		},
+	}
+
+	motd = fmt.Sprintf("REST frontend service %q available at URL: 'http://localhost:%d'",
+		name,
+		localPort,
+	)
+
+	return conf, motd
+
+}