bump substreams-sink-sql, use protodef-v1.0.5 for new codegens, allow…

…s users to specify run_interval_seconds for dbt configuration

codegen/templates/ethereum_project.go

@@ -104,7 +104,7 @@ func NewEthereumProject(name string, moduleName string, chain *EthereumChain, co
 		chain:                       chain,
 		ethereumContracts:           contracts,
 		creationBlockNum:            lowestStartBlock,
-		sqlImportVersion:            "1.0.4",
+		sqlImportVersion:            "1.0.5",
 		graphImportVersion:          "0.1.0",
 		databaseChangeImportVersion: "1.2.1",
 		entityChangeImportVersion:   "1.1.0",

docs/release-notes/change-log.md

@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
         enabled: true
       dbt:
         files: "./dbt"
+        run_interval_seconds: 60
   ```
   where "./dbt" is a folder containing the dbt project.
 

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/jhump/protoreflect v1.14.0
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/streamingfast/bstream v0.0.2-0.20230731165201-639b4f347707
+	github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a
 	github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80
 	github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1
 	github.com/streamingfast/dgrpc v0.0.0-20230929132851-893fc52687fa
@@ -60,7 +60,7 @@ require (
 	github.com/streamingfast/eth-go v0.0.0-20230410173454-433bd8803da1
 	github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9
 	github.com/streamingfast/shutter v1.5.0
-	github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3
+	github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231107184906-39740d2091cd
 	github.com/tetratelabs/wazero v1.1.0
 	github.com/tidwall/pretty v1.2.1
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0

go.sum

@@ -557,8 +557,8 @@ github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
 github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
 github.com/streamingfast/atm v0.0.0-20220131151839-18c87005e680 h1:fGJnUx0shX9Y312QOlz+/+yLquihXRhNqctJ26jtZZM=
 github.com/streamingfast/atm v0.0.0-20220131151839-18c87005e680/go.mod h1:iISPGAstbUsPgyC3auLLi7PYUTi9lHv5z0COam0OPOY=
-github.com/streamingfast/bstream v0.0.2-0.20230731165201-639b4f347707 h1:hJW+QNNJrR1boQuoEaajlMFjWh0XKt4Fcg33h9hT7Eo=
-github.com/streamingfast/bstream v0.0.2-0.20230731165201-639b4f347707/go.mod h1:Njkx972HcZiz0djWBylxqO/eq686eDGr+egQ1lePj3Q=
+github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a h1:NeCO5JLz38HRK1uaV1Emo9u5gUSRtmtZZGNK8BKyLIE=
+github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a/go.mod h1:Njkx972HcZiz0djWBylxqO/eq686eDGr+egQ1lePj3Q=
 github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80 h1:UxJUTcEVkdZy8N77E3exz0iNlgQuxl4m220GPvzdZ2s=
 github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80/go.mod h1:QxjVH73Lkqk+mP8bndvhMuQDUINfkgsYhdCH/5TJFKI=
 github.com/streamingfast/dauth v0.0.0-20230726175303-fc1d7198cb33 h1:MyOFvos5szhJWpoLYUeJ3UMlXBlL9kOhdD5LgEfDK48=
@@ -599,8 +599,8 @@ github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9 h1:YRwpVv
 github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9/go.mod h1:ktzt1BUj3GF+SKQHEmn3ShryJ7y87JeCHtaTGaDVATs=
 github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAtyaTOgs=
 github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8=
-github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3 h1:X+sIFo//ffQKHJbIAkhYa657gYZi/KPWf9phBwUzKrg=
-github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3/go.mod h1:vi9cdOke79ezfUZVAJHJY7Y/nOsmHXkRh0++Ze9hZNk=
+github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231107184906-39740d2091cd h1:4TXJOCzL4U+NiQbxqleyA/BL+hchpOKaBLwvzM3y/Jc=
+github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231107184906-39740d2091cd/go.mod h1:4Zd5Re1SrhXDnO3VJh/FJcn64SyZPqAv6gFPjbyqMYI=
 github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14 h1:/HCMj7plThWPFXsj/O8f+zDIAtXciwP4rK6tb+9UvPQ=
 github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14/go.mod h1:t4h97mWfTs6v0zjEFuGDOoW5wLtu9+ttegIx99i7gsM=
 github.com/streamingfast/wasmtime-go/v4 v4.0.0-freemem3 h1:raJHR0JWgYiSyX0vZ3leRK/TkNcn4ZUGTf+d64g48KQ=

sink-server/docker/dbt.go

@@ -38,8 +38,13 @@ func (e *DockerEngine) newPostgresDBT(deploymentID string, serviceName string, c
 		return conf, "", fmt.Errorf("unable to create dbt_project.yml file: %w", err)
 	}
 
+	runInterval := config.GetRunIntervalSeconds()
+	if runInterval == 0 {
+		runInterval = 300 //default to 5 minutes
+	}
+
 	//create dbt start script
-	dbtStartScript := createDbtStartScript()
+	dbtStartScript := createDbtStartScript(runInterval)
 
 	//create a volume for the files
 	profileFolder := filepath.Join(e.dir, deploymentID, "data", "profile")
@@ -191,15 +196,15 @@ func createDbtProfileYml(profileName string) ([]byte, error) {
 	return []byte(data), nil
 }
 
-func createDbtStartScript() []byte {
-	return []byte(`#!/bin/bash
+func createDbtStartScript(runInterval int32) []byte {
+	return []byte(fmt.Sprintf(`#!/bin/bash
 
 # Set the working directory
 cd /opt/data/dbt
 
 while true; do
 	dbt run --profiles-dir /opt/data/profile --target dev
-    sleep 60  # Sleep for 60 seconds (1 minute)
+    sleep %d  
 done
-`)
+`, runInterval))
 }