From 3c4a02f3b3febd729a8777d5e9d78304937375fb Mon Sep 17 00:00:00 2001 From: colindickson Date: Tue, 7 Nov 2023 14:23:04 -0500 Subject: [PATCH] bump substreams-sink-sql, use protodef-v1.0.5 for new codegens, allows users to specify run_interval_seconds for dbt configuration --- codegen/templates/ethereum_project.go | 2 +- docs/release-notes/change-log.md | 1 + go.mod | 4 ++-- go.sum | 8 ++++---- sink-server/docker/dbt.go | 15 ++++++++++----- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/codegen/templates/ethereum_project.go b/codegen/templates/ethereum_project.go index 50272ffc2..ee2554df2 100644 --- a/codegen/templates/ethereum_project.go +++ b/codegen/templates/ethereum_project.go @@ -104,7 +104,7 @@ func NewEthereumProject(name string, moduleName string, chain *EthereumChain, co chain: chain, ethereumContracts: contracts, creationBlockNum: lowestStartBlock, - sqlImportVersion: "1.0.4", + sqlImportVersion: "1.0.5", graphImportVersion: "0.1.0", databaseChangeImportVersion: "1.2.1", entityChangeImportVersion: "1.1.0", diff --git a/docs/release-notes/change-log.md b/docs/release-notes/change-log.md index fc308c7e3..4bb7f315f 100644 --- a/docs/release-notes/change-log.md +++ b/docs/release-notes/change-log.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), enabled: true dbt: files: "./dbt" + run_interval_seconds: 60 ``` where "./dbt" is a folder containing the dbt project. diff --git a/go.mod b/go.mod index 337da3695..96a9c9f66 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/jhump/protoreflect v1.14.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/streamingfast/bstream v0.0.2-0.20230731165201-639b4f347707 + github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80 github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1 github.com/streamingfast/dgrpc v0.0.0-20230929132851-893fc52687fa @@ -60,7 +60,7 @@ require ( github.com/streamingfast/eth-go v0.0.0-20230410173454-433bd8803da1 github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9 github.com/streamingfast/shutter v1.5.0 - github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3 + github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231107184906-39740d2091cd github.com/tetratelabs/wazero v1.1.0 github.com/tidwall/pretty v1.2.1 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0 diff --git a/go.sum b/go.sum index 11d98aa8d..e9fa1132a 100644 --- a/go.sum +++ b/go.sum @@ -557,8 +557,8 @@ github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/streamingfast/atm v0.0.0-20220131151839-18c87005e680 h1:fGJnUx0shX9Y312QOlz+/+yLquihXRhNqctJ26jtZZM= github.com/streamingfast/atm v0.0.0-20220131151839-18c87005e680/go.mod h1:iISPGAstbUsPgyC3auLLi7PYUTi9lHv5z0COam0OPOY= -github.com/streamingfast/bstream v0.0.2-0.20230731165201-639b4f347707 h1:hJW+QNNJrR1boQuoEaajlMFjWh0XKt4Fcg33h9hT7Eo= -github.com/streamingfast/bstream v0.0.2-0.20230731165201-639b4f347707/go.mod h1:Njkx972HcZiz0djWBylxqO/eq686eDGr+egQ1lePj3Q= +github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a h1:NeCO5JLz38HRK1uaV1Emo9u5gUSRtmtZZGNK8BKyLIE= +github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a/go.mod h1:Njkx972HcZiz0djWBylxqO/eq686eDGr+egQ1lePj3Q= github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80 h1:UxJUTcEVkdZy8N77E3exz0iNlgQuxl4m220GPvzdZ2s= github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80/go.mod h1:QxjVH73Lkqk+mP8bndvhMuQDUINfkgsYhdCH/5TJFKI= github.com/streamingfast/dauth v0.0.0-20230726175303-fc1d7198cb33 h1:MyOFvos5szhJWpoLYUeJ3UMlXBlL9kOhdD5LgEfDK48= @@ -599,8 +599,8 @@ github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9 h1:YRwpVv github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9/go.mod h1:ktzt1BUj3GF+SKQHEmn3ShryJ7y87JeCHtaTGaDVATs= github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAtyaTOgs= github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8= -github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3 h1:X+sIFo//ffQKHJbIAkhYa657gYZi/KPWf9phBwUzKrg= -github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231102141400-56bec6ccf2e3/go.mod h1:vi9cdOke79ezfUZVAJHJY7Y/nOsmHXkRh0++Ze9hZNk= +github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231107184906-39740d2091cd h1:4TXJOCzL4U+NiQbxqleyA/BL+hchpOKaBLwvzM3y/Jc= +github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231107184906-39740d2091cd/go.mod h1:4Zd5Re1SrhXDnO3VJh/FJcn64SyZPqAv6gFPjbyqMYI= github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14 h1:/HCMj7plThWPFXsj/O8f+zDIAtXciwP4rK6tb+9UvPQ= github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14/go.mod h1:t4h97mWfTs6v0zjEFuGDOoW5wLtu9+ttegIx99i7gsM= github.com/streamingfast/wasmtime-go/v4 v4.0.0-freemem3 h1:raJHR0JWgYiSyX0vZ3leRK/TkNcn4ZUGTf+d64g48KQ= diff --git a/sink-server/docker/dbt.go b/sink-server/docker/dbt.go index e9d029dd9..8f48b1f48 100644 --- a/sink-server/docker/dbt.go +++ b/sink-server/docker/dbt.go @@ -38,8 +38,13 @@ func (e *DockerEngine) newPostgresDBT(deploymentID string, serviceName string, c return conf, "", fmt.Errorf("unable to create dbt_project.yml file: %w", err) } + runInterval := config.GetRunIntervalSeconds() + if runInterval == 0 { + runInterval = 300 //default to 5 minutes + } + //create dbt start script - dbtStartScript := createDbtStartScript() + dbtStartScript := createDbtStartScript(runInterval) //create a volume for the files profileFolder := filepath.Join(e.dir, deploymentID, "data", "profile") @@ -191,15 +196,15 @@ func createDbtProfileYml(profileName string) ([]byte, error) { return []byte(data), nil } -func createDbtStartScript() []byte { - return []byte(`#!/bin/bash +func createDbtStartScript(runInterval int32) []byte { + return []byte(fmt.Sprintf(`#!/bin/bash # Set the working directory cd /opt/data/dbt while true; do dbt run --profiles-dir /opt/data/profile --target dev - sleep 60 # Sleep for 60 seconds (1 minute) + sleep %d done -`) +`, runInterval)) }