From 16ce65923528103ce0864e6d95fe9608327b0fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 15 Nov 2024 18:12:58 +0100 Subject: [PATCH 01/22] structure pkg --- cmd/{cli => conduit/global}/cli.go | 2 +- cmd/{cli => conduit/global}/cli_test.go | 2 +- cmd/{cli => conduit/global}/conduit_init.go | 4 +- cmd/{cli => conduit/global}/pipeline.tmpl | 0 cmd/{cli => conduit/global}/pipelines_init.go | 2 +- cmd/{cli => conduit}/internal/yaml.go | 0 cmd/conduit/main.go | 11 ++- cmd/conduit/root/init.go | 1 + cmd/conduit/root/pipelines/init.go | 15 ++++ cmd/conduit/root/pipelines/pipelines.go | 15 ++++ cmd/conduit/root/root.go | 68 +++++++++++++++++++ cmd/conduit/root/version/version.go | 41 +++++++++++ go.mod | 1 + go.sum | 2 + 14 files changed, 157 insertions(+), 7 deletions(-) rename cmd/{cli => conduit/global}/cli.go (99%) rename cmd/{cli => conduit/global}/cli_test.go (99%) rename cmd/{cli => conduit/global}/conduit_init.go (97%) rename cmd/{cli => conduit/global}/pipeline.tmpl (100%) rename cmd/{cli => conduit/global}/pipelines_init.go (99%) rename cmd/{cli => conduit}/internal/yaml.go (100%) create mode 100644 cmd/conduit/root/init.go create mode 100644 cmd/conduit/root/pipelines/init.go create mode 100644 cmd/conduit/root/pipelines/pipelines.go create mode 100644 cmd/conduit/root/root.go create mode 100644 cmd/conduit/root/version/version.go diff --git a/cmd/cli/cli.go b/cmd/conduit/global/cli.go similarity index 99% rename from cmd/cli/cli.go rename to cmd/conduit/global/cli.go index 90c39509b..0cad70c97 100644 --- a/cmd/cli/cli.go +++ b/cmd/conduit/global/cli.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cli +package global import ( "fmt" diff --git a/cmd/cli/cli_test.go b/cmd/conduit/global/cli_test.go similarity index 99% rename from cmd/cli/cli_test.go rename to cmd/conduit/global/cli_test.go index 7bca0b0d1..0511c3379 100644 --- a/cmd/cli/cli_test.go +++ b/cmd/conduit/global/cli_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cli +package global import ( "bytes" diff --git a/cmd/cli/conduit_init.go b/cmd/conduit/global/conduit_init.go similarity index 97% rename from cmd/cli/conduit_init.go rename to cmd/conduit/global/conduit_init.go index 40ece5351..d4bc0dbb0 100644 --- a/cmd/cli/conduit_init.go +++ b/cmd/conduit/global/conduit_init.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cli +package global import ( "flag" @@ -20,7 +20,7 @@ import ( "os" "path/filepath" - "github.com/conduitio/conduit/cmd/cli/internal" + "github.com/conduitio/conduit/cmd/conduit/internal" "github.com/conduitio/conduit/pkg/conduit" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/yaml/v3" diff --git a/cmd/cli/pipeline.tmpl b/cmd/conduit/global/pipeline.tmpl similarity index 100% rename from cmd/cli/pipeline.tmpl rename to cmd/conduit/global/pipeline.tmpl diff --git a/cmd/cli/pipelines_init.go b/cmd/conduit/global/pipelines_init.go similarity index 99% rename from cmd/cli/pipelines_init.go rename to cmd/conduit/global/pipelines_init.go index 8b27c420b..003ffdd2e 100644 --- a/cmd/cli/pipelines_init.go +++ b/cmd/conduit/global/pipelines_init.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cli +package global import ( _ "embed" diff --git a/cmd/cli/internal/yaml.go b/cmd/conduit/internal/yaml.go similarity index 100% rename from cmd/cli/internal/yaml.go rename to cmd/conduit/internal/yaml.go diff --git a/cmd/conduit/main.go b/cmd/conduit/main.go index a744d7f3e..0af5d9f75 100644 --- a/cmd/conduit/main.go +++ b/cmd/conduit/main.go @@ -15,9 +15,16 @@ package main import ( - "github.com/conduitio/conduit/cmd/cli" + "log" + + "github.com/conduitio/conduit/cmd/conduit/root" + "github.com/conduitio/ecdysis" ) func main() { - cli.New().Run() + e := ecdysis.New() + cmd := e.MustBuildCobraCommand(&root.RootCommand{}) + if err := cmd.Execute(); err != nil { + log.Fatal(err) + } } diff --git a/cmd/conduit/root/init.go b/cmd/conduit/root/init.go new file mode 100644 index 000000000..c8040311d --- /dev/null +++ b/cmd/conduit/root/init.go @@ -0,0 +1 @@ +package root diff --git a/cmd/conduit/root/pipelines/init.go b/cmd/conduit/root/pipelines/init.go new file mode 100644 index 000000000..f834c2959 --- /dev/null +++ b/cmd/conduit/root/pipelines/init.go @@ -0,0 +1,15 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipelines diff --git a/cmd/conduit/root/pipelines/pipelines.go b/cmd/conduit/root/pipelines/pipelines.go new file mode 100644 index 000000000..f834c2959 --- /dev/null +++ b/cmd/conduit/root/pipelines/pipelines.go @@ -0,0 +1,15 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipelines diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go new file mode 100644 index 000000000..182c1c3e5 --- /dev/null +++ b/cmd/conduit/root/root.go @@ -0,0 +1,68 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package root + +import ( + "context" + + "github.com/conduitio/conduit/cmd/conduit/global" + "github.com/conduitio/conduit/cmd/conduit/root/version" + "github.com/conduitio/ecdysis" +) + +type RootFlags struct { + Config string `long:"config" usage:"config file (default is $HOME/.conduit.yaml)" persistent:"true"` + Author string `long:"author" short:"a" usage:"author name for copyright attribution" persistent:"true"` + License string `long:"license" short:"l" usage:"name of license for the project" persistent:"true"` + Viper bool `long:"viper" usage:"use Viper for configuration" persistent:"true"` +} + +type RootCommand struct { + flags RootFlags +} + +func (c *RootCommand) Execute(ctx context.Context) error { + global.New().Run() + return nil +} + +var ( + _ ecdysis.CommandWithFlags = (*RootCommand)(nil) + _ ecdysis.CommandWithDocs = (*RootCommand)(nil) + _ ecdysis.CommandWithSubCommands = (*RootCommand)(nil) + _ ecdysis.CommandWithExecute = (*RootCommand)(nil) +) + +func (c *RootCommand) Usage() string { return "conduit" } +func (c *RootCommand) Flags() []ecdysis.Flag { + flags := ecdysis.BuildFlags(&c.flags) + + flags.SetDefault("author", "YOUR NAME") + flags.SetDefault("viper", true) + return flags +} + +func (c *RootCommand) Docs() ecdysis.Docs { + return ecdysis.Docs{ + Short: "Conduit CLI", + Long: `Conduit CLI is a command-line that helps you interact with and manage Conduit.`, + } +} + +func (c *RootCommand) SubCommands() []ecdysis.Command { + return []ecdysis.Command{ + &version.VersionCommand{}, + } +} diff --git a/cmd/conduit/root/version/version.go b/cmd/conduit/root/version/version.go new file mode 100644 index 000000000..b8fab764b --- /dev/null +++ b/cmd/conduit/root/version/version.go @@ -0,0 +1,41 @@ +// Copyright © 2022 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "context" + "fmt" + + "github.com/conduitio/ecdysis" +) + +type VersionCommand struct{} + +var ( + _ ecdysis.CommandWithExecute = (*VersionCommand)(nil) + _ ecdysis.CommandWithDocs = (*VersionCommand)(nil) +) + +func (c *VersionCommand) Usage() string { return "version" } +func (c *VersionCommand) Docs() ecdysis.Docs { + return ecdysis.Docs{ + Short: "Print the version number of conduit", + } +} + +func (c *VersionCommand) Execute(context.Context) error { + fmt.Println("conduit v0.1.0") + return nil +} diff --git a/go.mod b/go.mod index 0e97e891a..96981360b 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/conduitio/conduit-connector-sdk v0.12.0 github.com/conduitio/conduit-processor-sdk v0.4.0 github.com/conduitio/conduit-schema-registry v0.2.2 + github.com/conduitio/ecdysis v0.0.0-20241104140515-1031f323f080 github.com/conduitio/yaml/v3 v3.3.0 github.com/dop251/goja v0.0.0-20240806095544-3491d4a58fbe github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c diff --git a/go.sum b/go.sum index 4d65306b2..231c77729 100644 --- a/go.sum +++ b/go.sum @@ -244,6 +244,8 @@ github.com/conduitio/conduit-processor-sdk v0.4.0 h1:wF1Fj31aneNixNbW5rJ0/5Q3vwW github.com/conduitio/conduit-processor-sdk v0.4.0/go.mod h1:Jj9ZBTee7nO0XeociDxe9gSvLFN1GbPWP1Aj04DPeZQ= github.com/conduitio/conduit-schema-registry v0.2.2 h1:Q0uL8egRAzJlRV7Ed5nEcqZ1yE/UQeZJad3VmhgTSFE= github.com/conduitio/conduit-schema-registry v0.2.2/go.mod h1:EmT4ylkz15LYddL6qU4wDX52n1Yp0aHvEDRIWOYYzFs= +github.com/conduitio/ecdysis v0.0.0-20241104140515-1031f323f080 h1:Pd5uzNGyPy/Va3rCFp0Ni2xzohrpvgsqbdbAyeFpoZs= +github.com/conduitio/ecdysis v0.0.0-20241104140515-1031f323f080/go.mod h1:JWjm2WhbGExspVAOH+8tb1t9iu+RxKrHiMvWgtPsUI8= github.com/conduitio/yaml/v3 v3.3.0 h1:kbbaOSHcuH39gP4+rgbJGl6DSbLZcJgEaBvkEXJlCsI= github.com/conduitio/yaml/v3 v3.3.0/go.mod h1:JNgFMOX1t8W4YJuRZOh6GggVtSMsgP9XgTw+7dIenpc= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= From d54599f6d89edaed23fe14eb22bf978e63ed41e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 18 Nov 2024 16:16:29 +0100 Subject: [PATCH 02/22] refactor to reutilize global flags --- cmd/conduit/global/cli.go | 158 -------- cmd/conduit/global/conduit_init.go | 122 ------ cmd/conduit/global/pipelines_init.go | 354 ---------------- cmd/conduit/main.go | 13 +- cmd/conduit/root/init.go | 141 +++++++ cmd/conduit/root/pipelines/init.go | 382 ++++++++++++++++++ .../{global => root/pipelines}/pipeline.tmpl | 0 cmd/conduit/root/pipelines/pipelines.go | 15 + cmd/conduit/root/root.go | 92 +++-- .../{global/cli_test.go => root/root_test.go} | 2 +- cmd/conduit/root/version/version.go | 41 -- 11 files changed, 596 insertions(+), 724 deletions(-) delete mode 100644 cmd/conduit/global/cli.go delete mode 100644 cmd/conduit/global/conduit_init.go delete mode 100644 cmd/conduit/global/pipelines_init.go rename cmd/conduit/{global => root/pipelines}/pipeline.tmpl (100%) rename cmd/conduit/{global/cli_test.go => root/root_test.go} (99%) delete mode 100644 cmd/conduit/root/version/version.go diff --git a/cmd/conduit/global/cli.go b/cmd/conduit/global/cli.go deleted file mode 100644 index 0cad70c97..000000000 --- a/cmd/conduit/global/cli.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package global - -import ( - "fmt" - "os" - - "github.com/conduitio/conduit/pkg/conduit" - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -var ( - initArgs InitArgs - pipelinesInitArgs PipelinesInitArgs -) - -type Instance struct { - rootCmd *cobra.Command -} - -// New creates a new CLI Instance. -func New() *Instance { - return &Instance{ - rootCmd: buildRootCmd(), - } -} - -func (i *Instance) Run() { - if err := i.rootCmd.Execute(); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } -} - -func buildRootCmd() *cobra.Command { - cfg := conduit.DefaultConfig() - - cmd := &cobra.Command{ - Use: "conduit", - Short: "Conduit CLI", - Long: "Conduit CLI is a command-line that helps you interact with and manage Conduit.", - Version: conduit.Version(true), - Run: func(cmd *cobra.Command, args []string) { - e := &conduit.Entrypoint{} - e.Serve(cfg) - }, - } - cmd.CompletionOptions.DisableDefaultCmd = true - conduit.Flags(&cfg).VisitAll(cmd.Flags().AddGoFlag) - - // init - cmd.AddCommand(buildInitCmd()) - - // pipelines - cmd.AddGroup(&cobra.Group{ - ID: "pipelines", - Title: "Pipelines", - }) - cmd.AddCommand(buildPipelinesCmd()) - - // mark hidden flags - cmd.Flags().VisitAll(func(f *pflag.Flag) { - if conduit.HiddenFlags[f.Name] { - err := cmd.Flags().MarkHidden(f.Name) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to mark flag %q as hidden: %v", f.Name, err) - } - } - }) - - return cmd -} - -func buildInitCmd() *cobra.Command { - initCmd := &cobra.Command{ - Use: "init", - Short: "Initialize Conduit with a configuration file and directories.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return NewConduitInit(initArgs).Run() - }, - } - initCmd.Flags().StringVar( - &initArgs.Path, - "config.path", - "", - "path where Conduit will be initialized", - ) - - return initCmd -} - -func buildPipelinesCmd() *cobra.Command { - pipelinesCmd := &cobra.Command{ - Use: "pipelines", - Short: "Initialize and manage pipelines", - Args: cobra.NoArgs, - GroupID: "pipelines", - } - - pipelinesCmd.AddCommand(buildPipelinesInitCmd()) - - return pipelinesCmd -} - -func buildPipelinesInitCmd() *cobra.Command { - pipelinesInitCmd := &cobra.Command{ - Use: "init [pipeline-name]", - Short: "Initialize an example pipeline.", - Long: `Initialize a pipeline configuration file, with all of parameters for source and destination connectors -initialized and described. The source and destination connector can be chosen via flags. If no connectors are chosen, then -a simple and runnable generator-to-log pipeline is configured.`, - Args: cobra.MaximumNArgs(1), - Example: " conduit pipelines init awesome-pipeline-name --source postgres --destination kafka --path pipelines/pg-to-kafka.yaml", - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) > 0 { - pipelinesInitArgs.Name = args[0] - } - return NewPipelinesInit(pipelinesInitArgs).Run() - }, - } - - // Add flags to pipelines init command - pipelinesInitCmd.Flags().StringVar( - &pipelinesInitArgs.Source, - "source", - "", - "Source connector (any of the built-in connectors).", - ) - pipelinesInitCmd.Flags().StringVar( - &pipelinesInitArgs.Destination, - "destination", - "", - "Destination connector (any of the built-in connectors).", - ) - pipelinesInitCmd.Flags().StringVar( - &pipelinesInitArgs.Path, - "pipelines.path", - "./pipelines", - "Path where the pipeline will be saved.", - ) - - return pipelinesInitCmd -} diff --git a/cmd/conduit/global/conduit_init.go b/cmd/conduit/global/conduit_init.go deleted file mode 100644 index d4bc0dbb0..000000000 --- a/cmd/conduit/global/conduit_init.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package global - -import ( - "flag" - "fmt" - "os" - "path/filepath" - - "github.com/conduitio/conduit/cmd/conduit/internal" - "github.com/conduitio/conduit/pkg/conduit" - "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/yaml/v3" -) - -type InitArgs struct { - Path string -} - -type ConduitInit struct { - args InitArgs -} - -func NewConduitInit(args InitArgs) *ConduitInit { - return &ConduitInit{args: args} -} - -func (i *ConduitInit) Run() error { - err := i.createDirs() - if err != nil { - return err - } - - err = i.createConfigYAML() - if err != nil { - return fmt.Errorf("failed to create config YAML: %w", err) - } - - fmt.Println(` -Conduit has been initialized! - -To quickly create an example pipeline, run 'conduit pipelines init'. -To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) - - return nil -} - -func (i *ConduitInit) createConfigYAML() error { - cfgYAML := internal.NewYAMLTree() - i.conduitCfgFlags().VisitAll(func(f *flag.Flag) { - if conduit.HiddenFlags[f.Name] { - return // hide flag from output - } - cfgYAML.Insert(f.Name, f.DefValue, f.Usage) - }) - - yamlData, err := yaml.Marshal(cfgYAML.Root) - if err != nil { - return cerrors.Errorf("error marshaling YAML: %w\n", err) - } - - path := filepath.Join(i.path(), "conduit.yaml") - err = os.WriteFile(path, yamlData, 0o600) - if err != nil { - return cerrors.Errorf("error writing conduit.yaml: %w", err) - } - fmt.Printf("Configuration file written to %v\n", path) - - return nil -} - -func (i *ConduitInit) createDirs() error { - dirs := []string{"processors", "connectors", "pipelines"} - - for _, dir := range dirs { - path := filepath.Join(i.path(), dir) - - // Attempt to create the directory, skipping if it already exists - if err := os.Mkdir(path, os.ModePerm); err != nil { - if os.IsExist(err) { - fmt.Printf("Directory '%s' already exists, skipping...\n", path) - continue - } - return fmt.Errorf("failed to create directory '%s': %w", path, err) - } - - fmt.Printf("Created directory: %s\n", path) - } - - return nil -} - -func (i *ConduitInit) conduitCfgFlags() *flag.FlagSet { - cfg := conduit.DefaultConfigWithBasePath(i.path()) - return conduit.Flags(&cfg) -} - -func (i *ConduitInit) path() string { - if i.args.Path != "" { - return i.args.Path - } - - path, err := os.Getwd() - if err != nil { - panic(cerrors.Errorf("failed to get current working directory: %w", err)) - } - - return path -} diff --git a/cmd/conduit/global/pipelines_init.go b/cmd/conduit/global/pipelines_init.go deleted file mode 100644 index 003ffdd2e..000000000 --- a/cmd/conduit/global/pipelines_init.go +++ /dev/null @@ -1,354 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package global - -import ( - _ "embed" - "fmt" - "log" - "os" - "path/filepath" - "strings" - "text/template" - - "github.com/conduitio/conduit-commons/config" - "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/connector/builtin" -) - -//go:embed pipeline.tmpl -var pipelineCfgTmpl string - -var funcMap = template.FuncMap{ - "formatParameterValueTable": formatParameterValueTable, - "formatParameterValueYAML": formatParameterValueYAML, - "formatParameterDescriptionYAML": formatParameterDescriptionYAML, - "formatParameterRequired": formatParameterRequired, -} - -func formatParameterRequired(param config.Parameter) string { - for _, v := range param.Validations { - if v.Type() == config.ValidationTypeRequired { - return "Required" - } - } - - return "Optional" -} - -// formatParameterValue formats the value of a configuration parameter. -func formatParameterValueTable(value string) string { - switch { - case value == "": - return `` - case strings.Contains(value, "\n"): - // specifically used in the javascript processor - return fmt.Sprintf("\n```js\n%s\n```\n", value) - default: - return fmt.Sprintf("`%s`", value) - } -} - -func formatParameterDescriptionYAML(description string) string { - const ( - indentLen = 10 - prefix = "# " - lineLen = 80 - tmpNewLine = "〠" - ) - - // remove markdown new lines - description = strings.ReplaceAll(description, "\n\n", tmpNewLine) - description = strings.ReplaceAll(description, "\n", " ") - description = strings.ReplaceAll(description, tmpNewLine, "\n") - - formattedDescription := formatMultiline(description, strings.Repeat(" ", indentLen)+prefix, lineLen) - // remove first indent and last new line - formattedDescription = formattedDescription[indentLen : len(formattedDescription)-1] - return formattedDescription -} - -func formatMultiline( - input string, - prefix string, - maxLineLen int, -) string { - textLen := maxLineLen - len(prefix) - - // split the input into lines of length textLen - lines := strings.Split(input, "\n") - var formattedLines []string - for _, line := range lines { - if len(line) <= textLen { - formattedLines = append(formattedLines, line) - continue - } - - // split the line into multiple lines, don't break words - words := strings.Fields(line) - var formattedLine string - for _, word := range words { - if len(formattedLine)+len(word) > textLen { - formattedLines = append(formattedLines, formattedLine[1:]) - formattedLine = "" - } - formattedLine += " " + word - } - if formattedLine != "" { - formattedLines = append(formattedLines, formattedLine[1:]) - } - } - - // combine lines including indent and prefix - var formatted string - for _, line := range formattedLines { - formatted += prefix + line + "\n" - } - - return formatted -} - -func formatParameterValueYAML(value string) string { - switch { - case value == "": - return `""` - case strings.Contains(value, "\n"): - // specifically used in the javascript processor - formattedValue := formatMultiline(value, " ", 10000) - return fmt.Sprintf("|\n%s", formattedValue) - default: - return fmt.Sprintf(`'%s'`, value) - } -} - -const ( - defaultDestination = "file" - defaultSource = "generator" -) - -type pipelineTemplate struct { - Name string - SourceSpec connectorTemplate - DestinationSpec connectorTemplate -} - -type connectorTemplate struct { - Name string - Params config.Parameters -} - -type PipelinesInitArgs struct { - Name string - Source string - Destination string - Path string -} - -type PipelinesInit struct { - args PipelinesInitArgs -} - -func NewPipelinesInit(args PipelinesInitArgs) *PipelinesInit { - return &PipelinesInit{args: args} -} - -func (pi *PipelinesInit) Run() error { - var pipeline pipelineTemplate - // if no source/destination arguments are provided, - // we build a runnable example pipeline - if pi.args.Source == "" && pi.args.Destination == "" { - pipeline = pi.buildDemoPipeline() - } else { - p, err := pi.buildTemplatePipeline() - if err != nil { - return err - } - pipeline = p - } - - err := pi.write(pipeline) - if err != nil { - return cerrors.Errorf("could not write pipeline: %w", err) - } - - fmt.Printf(`Your pipeline has been initialized and created at %s. - -To run the pipeline, simply run 'conduit'.`, pi.configFilePath()) - - return nil -} - -func (pi *PipelinesInit) buildTemplatePipeline() (pipelineTemplate, error) { - srcParams, err := pi.getSourceParams() - if err != nil { - return pipelineTemplate{}, cerrors.Errorf("failed getting source params: %w", err) - } - - dstParams, err := pi.getDestinationParams() - if err != nil { - return pipelineTemplate{}, cerrors.Errorf("failed getting destination params: %w", err) - } - - return pipelineTemplate{ - Name: pi.pipelineName(), - SourceSpec: srcParams, - DestinationSpec: dstParams, - }, nil -} - -func (pi *PipelinesInit) buildDemoPipeline() pipelineTemplate { - srcParams, _ := pi.getSourceParams() - dstParams, _ := pi.getDestinationParams() - - return pipelineTemplate{ - Name: pi.pipelineName(), - SourceSpec: connectorTemplate{ - Name: defaultSource, - Params: map[string]config.Parameter{ - "format.type": { - Description: srcParams.Params["format.type"].Description, - Type: srcParams.Params["format.type"].Type, - Default: "structured", - Validations: srcParams.Params["format.type"].Validations, - }, - "format.options.scheduledDeparture": { - Description: "Generate field 'scheduledDeparture' of type 'time'", - Type: config.ParameterTypeString, - Default: "time", - }, - "format.options.airline": { - Description: "Generate field 'airline' of type string", - Type: config.ParameterTypeString, - Default: "string", - }, - "rate": { - Description: srcParams.Params["rate"].Description, - Type: srcParams.Params["rate"].Type, - Default: "1", - }, - }, - }, - DestinationSpec: connectorTemplate{ - Name: defaultDestination, - Params: map[string]config.Parameter{ - "path": { - Description: dstParams.Params["path"].Description, - Type: dstParams.Params["path"].Type, - Default: "./destination.txt", - }, - }, - }, - } -} - -func (pi *PipelinesInit) getOutput() *os.File { - output, err := os.OpenFile(pi.configFilePath(), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) - if err != nil { - log.Fatalf("error: failed to open %s: %v", pi.args.Path, err) - } - - return output -} - -func (pi *PipelinesInit) write(pipeline pipelineTemplate) error { - t, err := template.New("").Funcs(funcMap).Option("missingkey=zero").Parse(pipelineCfgTmpl) - if err != nil { - return cerrors.Errorf("failed parsing template: %w", err) - } - - output := pi.getOutput() - defer output.Close() - - err = t.Execute(output, pipeline) - if err != nil { - return cerrors.Errorf("failed executing template: %w", err) - } - - return nil -} - -func (pi *PipelinesInit) getSourceParams() (connectorTemplate, error) { - for _, conn := range builtin.DefaultBuiltinConnectors { - specs := conn.NewSpecification() - if specs.Name == pi.sourceConnector() || specs.Name == "builtin:"+pi.sourceConnector() { - if conn.NewSource == nil { - return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", pi.sourceConnector()) - } - - return connectorTemplate{ - Name: specs.Name, - Params: conn.NewSource().Parameters(), - }, nil - } - } - - return connectorTemplate{}, cerrors.Errorf("%v: %w", pi.sourceConnector(), plugin.ErrPluginNotFound) -} - -func (pi *PipelinesInit) getDestinationParams() (connectorTemplate, error) { - for _, conn := range builtin.DefaultBuiltinConnectors { - specs := conn.NewSpecification() - if specs.Name == pi.destinationConnector() || specs.Name == "builtin:"+pi.destinationConnector() { - if conn.NewDestination == nil { - return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", pi.destinationConnector()) - } - - return connectorTemplate{ - Name: specs.Name, - Params: conn.NewDestination().Parameters(), - }, nil - } - } - - return connectorTemplate{}, cerrors.Errorf("%v: %w", pi.destinationConnector(), plugin.ErrPluginNotFound) -} - -func (pi *PipelinesInit) configFilePath() string { - path := pi.args.Path - if path == "" { - path = "./pipelines" - } - - return filepath.Join(path, pi.configFileName()) -} - -func (pi *PipelinesInit) configFileName() string { - return fmt.Sprintf("pipeline-%s.yaml", pi.pipelineName()) -} - -func (pi *PipelinesInit) sourceConnector() string { - if pi.args.Source != "" { - return pi.args.Source - } - - return defaultSource -} - -func (pi *PipelinesInit) destinationConnector() string { - if pi.args.Destination != "" { - return pi.args.Destination - } - - return defaultDestination -} - -func (pi *PipelinesInit) pipelineName() string { - if pi.args.Name != "" { - return pi.args.Name - } - - return fmt.Sprintf("%s-to-%s", pi.sourceConnector(), pi.destinationConnector()) -} diff --git a/cmd/conduit/main.go b/cmd/conduit/main.go index 0af5d9f75..3dc10c7e4 100644 --- a/cmd/conduit/main.go +++ b/cmd/conduit/main.go @@ -14,17 +14,8 @@ package main -import ( - "log" - - "github.com/conduitio/conduit/cmd/conduit/root" - "github.com/conduitio/ecdysis" -) +import "github.com/conduitio/conduit/cmd/conduit/root" func main() { - e := ecdysis.New() - cmd := e.MustBuildCobraCommand(&root.RootCommand{}) - if err := cmd.Execute(); err != nil { - log.Fatal(err) - } + root.New().Run() } diff --git a/cmd/conduit/root/init.go b/cmd/conduit/root/init.go index c8040311d..3030501cb 100644 --- a/cmd/conduit/root/init.go +++ b/cmd/conduit/root/init.go @@ -1 +1,142 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package root + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/conduitio/conduit/cmd/conduit/internal" + "github.com/conduitio/conduit/pkg/conduit" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/yaml/v3" + "github.com/spf13/cobra" +) + +type InitArgs struct { + Path string +} + +type ConduitInit struct { + args InitArgs +} + +func NewConduitInit(args InitArgs) *ConduitInit { + return &ConduitInit{args: args} +} + +func (i *ConduitInit) Run() error { + err := i.createDirs() + if err != nil { + return err + } + + err = i.createConfigYAML() + if err != nil { + return fmt.Errorf("failed to create config YAML: %w", err) + } + + fmt.Println(` +Conduit has been initialized! + +To quickly create an example pipeline, run 'conduit pipelines init'. +To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) + + return nil +} + +func (i *ConduitInit) createConfigYAML() error { + cfgYAML := internal.NewYAMLTree() + i.conduitCfgFlags().VisitAll(func(f *flag.Flag) { + if conduit.HiddenFlags[f.Name] { + return // hide flag from output + } + cfgYAML.Insert(f.Name, f.DefValue, f.Usage) + }) + + yamlData, err := yaml.Marshal(cfgYAML.Root) + if err != nil { + return cerrors.Errorf("error marshaling YAML: %w\n", err) + } + + path := filepath.Join(i.path(), "conduit.yaml") + err = os.WriteFile(path, yamlData, 0o600) + if err != nil { + return cerrors.Errorf("error writing conduit.yaml: %w", err) + } + fmt.Printf("Configuration file written to %v\n", path) + + return nil +} + +func (i *ConduitInit) createDirs() error { + dirs := []string{"processors", "connectors", "pipelines"} + + for _, dir := range dirs { + path := filepath.Join(i.path(), dir) + + // Attempt to create the directory, skipping if it already exists + if err := os.Mkdir(path, os.ModePerm); err != nil { + if os.IsExist(err) { + fmt.Printf("Directory '%s' already exists, skipping...\n", path) + continue + } + return fmt.Errorf("failed to create directory '%s': %w", path, err) + } + + fmt.Printf("Created directory: %s\n", path) + } + + return nil +} + +func (i *ConduitInit) conduitCfgFlags() *flag.FlagSet { + cfg := conduit.DefaultConfigWithBasePath(i.path()) + return conduit.Flags(&cfg) +} + +func (i *ConduitInit) path() string { + if i.args.Path != "" { + return i.args.Path + } + + path, err := os.Getwd() + if err != nil { + panic(cerrors.Errorf("failed to get current working directory: %w", err)) + } + + return path +} + +func buildInitCmd() *cobra.Command { + initCmd := &cobra.Command{ + Use: "init", + Short: "Initialize Conduit with a configuration file and directories.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return NewConduitInit(initArgs).Run() + }, + } + initCmd.Flags().StringVar( + &initArgs.Path, + "config.path", + "", + "path where Conduit will be initialized", + ) + + return initCmd +} diff --git a/cmd/conduit/root/pipelines/init.go b/cmd/conduit/root/pipelines/init.go index f834c2959..66e15ea51 100644 --- a/cmd/conduit/root/pipelines/init.go +++ b/cmd/conduit/root/pipelines/init.go @@ -13,3 +13,385 @@ // limitations under the License. package pipelines + +import ( + _ "embed" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin" + "github.com/spf13/cobra" +) + +//go:embed pipeline.tmpl +var pipelineCfgTmpl string + +var pipelinesInitArgs PipelinesInitArgs + +var funcMap = template.FuncMap{ + "formatParameterValueTable": formatParameterValueTable, + "formatParameterValueYAML": formatParameterValueYAML, + "formatParameterDescriptionYAML": formatParameterDescriptionYAML, + "formatParameterRequired": formatParameterRequired, +} + +func formatParameterRequired(param config.Parameter) string { + for _, v := range param.Validations { + if v.Type() == config.ValidationTypeRequired { + return "Required" + } + } + + return "Optional" +} + +// formatParameterValue formats the value of a configuration parameter. +func formatParameterValueTable(value string) string { + switch { + case value == "": + return `` + case strings.Contains(value, "\n"): + // specifically used in the javascript processor + return fmt.Sprintf("\n```js\n%s\n```\n", value) + default: + return fmt.Sprintf("`%s`", value) + } +} + +func formatParameterDescriptionYAML(description string) string { + const ( + indentLen = 10 + prefix = "# " + lineLen = 80 + tmpNewLine = "〠" + ) + + // remove markdown new lines + description = strings.ReplaceAll(description, "\n\n", tmpNewLine) + description = strings.ReplaceAll(description, "\n", " ") + description = strings.ReplaceAll(description, tmpNewLine, "\n") + + formattedDescription := formatMultiline(description, strings.Repeat(" ", indentLen)+prefix, lineLen) + // remove first indent and last new line + formattedDescription = formattedDescription[indentLen : len(formattedDescription)-1] + return formattedDescription +} + +func formatMultiline( + input string, + prefix string, + maxLineLen int, +) string { + textLen := maxLineLen - len(prefix) + + // split the input into lines of length textLen + lines := strings.Split(input, "\n") + var formattedLines []string + for _, line := range lines { + if len(line) <= textLen { + formattedLines = append(formattedLines, line) + continue + } + + // split the line into multiple lines, don't break words + words := strings.Fields(line) + var formattedLine string + for _, word := range words { + if len(formattedLine)+len(word) > textLen { + formattedLines = append(formattedLines, formattedLine[1:]) + formattedLine = "" + } + formattedLine += " " + word + } + if formattedLine != "" { + formattedLines = append(formattedLines, formattedLine[1:]) + } + } + + // combine lines including indent and prefix + var formatted string + for _, line := range formattedLines { + formatted += prefix + line + "\n" + } + + return formatted +} + +func formatParameterValueYAML(value string) string { + switch { + case value == "": + return `""` + case strings.Contains(value, "\n"): + // specifically used in the javascript processor + formattedValue := formatMultiline(value, " ", 10000) + return fmt.Sprintf("|\n%s", formattedValue) + default: + return fmt.Sprintf(`'%s'`, value) + } +} + +const ( + defaultDestination = "file" + defaultSource = "generator" +) + +type pipelineTemplate struct { + Name string + SourceSpec connectorTemplate + DestinationSpec connectorTemplate +} + +type connectorTemplate struct { + Name string + Params config.Parameters +} + +type PipelinesInitArgs struct { + Name string + Source string + Destination string + Path string +} + +type PipelinesInit struct { + args PipelinesInitArgs +} + +func NewPipelinesInit(args PipelinesInitArgs) *PipelinesInit { + return &PipelinesInit{args: args} +} + +func (pi *PipelinesInit) Run() error { + var pipeline pipelineTemplate + // if no source/destination arguments are provided, + // we build a runnable example pipeline + if pi.args.Source == "" && pi.args.Destination == "" { + pipeline = pi.buildDemoPipeline() + } else { + p, err := pi.buildTemplatePipeline() + if err != nil { + return err + } + pipeline = p + } + + err := pi.write(pipeline) + if err != nil { + return cerrors.Errorf("could not write pipeline: %w", err) + } + + fmt.Printf(`Your pipeline has been initialized and created at %s. + +To run the pipeline, simply run 'conduit'.`, pi.configFilePath()) + + return nil +} + +func (pi *PipelinesInit) buildTemplatePipeline() (pipelineTemplate, error) { + srcParams, err := pi.getSourceParams() + if err != nil { + return pipelineTemplate{}, cerrors.Errorf("failed getting source params: %w", err) + } + + dstParams, err := pi.getDestinationParams() + if err != nil { + return pipelineTemplate{}, cerrors.Errorf("failed getting destination params: %w", err) + } + + return pipelineTemplate{ + Name: pi.pipelineName(), + SourceSpec: srcParams, + DestinationSpec: dstParams, + }, nil +} + +func (pi *PipelinesInit) buildDemoPipeline() pipelineTemplate { + srcParams, _ := pi.getSourceParams() + dstParams, _ := pi.getDestinationParams() + + return pipelineTemplate{ + Name: pi.pipelineName(), + SourceSpec: connectorTemplate{ + Name: defaultSource, + Params: map[string]config.Parameter{ + "format.type": { + Description: srcParams.Params["format.type"].Description, + Type: srcParams.Params["format.type"].Type, + Default: "structured", + Validations: srcParams.Params["format.type"].Validations, + }, + "format.options.scheduledDeparture": { + Description: "Generate field 'scheduledDeparture' of type 'time'", + Type: config.ParameterTypeString, + Default: "time", + }, + "format.options.airline": { + Description: "Generate field 'airline' of type string", + Type: config.ParameterTypeString, + Default: "string", + }, + "rate": { + Description: srcParams.Params["rate"].Description, + Type: srcParams.Params["rate"].Type, + Default: "1", + }, + }, + }, + DestinationSpec: connectorTemplate{ + Name: defaultDestination, + Params: map[string]config.Parameter{ + "path": { + Description: dstParams.Params["path"].Description, + Type: dstParams.Params["path"].Type, + Default: "./destination.txt", + }, + }, + }, + } +} + +func (pi *PipelinesInit) getOutput() *os.File { + output, err := os.OpenFile(pi.configFilePath(), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + log.Fatalf("error: failed to open %s: %v", pi.args.Path, err) + } + + return output +} + +func (pi *PipelinesInit) write(pipeline pipelineTemplate) error { + t, err := template.New("").Funcs(funcMap).Option("missingkey=zero").Parse(pipelineCfgTmpl) + if err != nil { + return cerrors.Errorf("failed parsing template: %w", err) + } + + output := pi.getOutput() + defer output.Close() + + err = t.Execute(output, pipeline) + if err != nil { + return cerrors.Errorf("failed executing template: %w", err) + } + + return nil +} + +func (pi *PipelinesInit) getSourceParams() (connectorTemplate, error) { + for _, conn := range builtin.DefaultBuiltinConnectors { + specs := conn.NewSpecification() + if specs.Name == pi.sourceConnector() || specs.Name == "builtin:"+pi.sourceConnector() { + if conn.NewSource == nil { + return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", pi.sourceConnector()) + } + + return connectorTemplate{ + Name: specs.Name, + Params: conn.NewSource().Parameters(), + }, nil + } + } + + return connectorTemplate{}, cerrors.Errorf("%v: %w", pi.sourceConnector(), plugin.ErrPluginNotFound) +} + +func (pi *PipelinesInit) getDestinationParams() (connectorTemplate, error) { + for _, conn := range builtin.DefaultBuiltinConnectors { + specs := conn.NewSpecification() + if specs.Name == pi.destinationConnector() || specs.Name == "builtin:"+pi.destinationConnector() { + if conn.NewDestination == nil { + return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", pi.destinationConnector()) + } + + return connectorTemplate{ + Name: specs.Name, + Params: conn.NewDestination().Parameters(), + }, nil + } + } + + return connectorTemplate{}, cerrors.Errorf("%v: %w", pi.destinationConnector(), plugin.ErrPluginNotFound) +} + +func (pi *PipelinesInit) configFilePath() string { + path := pi.args.Path + if path == "" { + path = "./pipelines" + } + + return filepath.Join(path, pi.configFileName()) +} + +func (pi *PipelinesInit) configFileName() string { + return fmt.Sprintf("pipeline-%s.yaml", pi.pipelineName()) +} + +func (pi *PipelinesInit) sourceConnector() string { + if pi.args.Source != "" { + return pi.args.Source + } + + return defaultSource +} + +func (pi *PipelinesInit) destinationConnector() string { + if pi.args.Destination != "" { + return pi.args.Destination + } + + return defaultDestination +} + +func (pi *PipelinesInit) pipelineName() string { + if pi.args.Name != "" { + return pi.args.Name + } + + return fmt.Sprintf("%s-to-%s", pi.sourceConnector(), pi.destinationConnector()) +} + +func BuildPipelinesInitCmd() *cobra.Command { + pipelinesInitCmd := &cobra.Command{ + Use: "init [pipeline-name]", + Short: "Initialize an example pipeline.", + Long: `Initialize a pipeline configuration file, with all of parameters for source and destination connectors +initialized and described. The source and destination connector can be chosen via flags. If no connectors are chosen, then +a simple and runnable generator-to-log pipeline is configured.`, + Args: cobra.MaximumNArgs(1), + Example: " conduit pipelines init awesome-pipeline-name --source postgres --destination kafka --path pipelines/pg-to-kafka.yaml", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + pipelinesInitArgs.Name = args[0] + } + return NewPipelinesInit(pipelinesInitArgs).Run() + }, + } + + // Add flags to pipelines init command + pipelinesInitCmd.Flags().StringVar( + &pipelinesInitArgs.Source, + "source", + "", + "Source connector (any of the built-in connectors).", + ) + pipelinesInitCmd.Flags().StringVar( + &pipelinesInitArgs.Destination, + "destination", + "", + "Destination connector (any of the built-in connectors).", + ) + pipelinesInitCmd.Flags().StringVar( + &pipelinesInitArgs.Path, + "pipelines.path", + "./pipelines", + "Path where the pipeline will be saved.", + ) + + return pipelinesInitCmd +} diff --git a/cmd/conduit/global/pipeline.tmpl b/cmd/conduit/root/pipelines/pipeline.tmpl similarity index 100% rename from cmd/conduit/global/pipeline.tmpl rename to cmd/conduit/root/pipelines/pipeline.tmpl diff --git a/cmd/conduit/root/pipelines/pipelines.go b/cmd/conduit/root/pipelines/pipelines.go index f834c2959..f130d2eb9 100644 --- a/cmd/conduit/root/pipelines/pipelines.go +++ b/cmd/conduit/root/pipelines/pipelines.go @@ -13,3 +13,18 @@ // limitations under the License. package pipelines + +import "github.com/spf13/cobra" + +func BuildPipelinesCmd() *cobra.Command { + pipelinesCmd := &cobra.Command{ + Use: "pipelines", + Short: "Initialize and manage pipelines", + Args: cobra.NoArgs, + GroupID: "pipelines", + } + + pipelinesCmd.AddCommand(BuildPipelinesInitCmd()) + + return pipelinesCmd +} diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index 182c1c3e5..fb28f4d39 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -15,54 +15,72 @@ package root import ( - "context" + "fmt" + "os" - "github.com/conduitio/conduit/cmd/conduit/global" - "github.com/conduitio/conduit/cmd/conduit/root/version" - "github.com/conduitio/ecdysis" + "github.com/conduitio/conduit/cmd/conduit/root/pipelines" + "github.com/conduitio/conduit/pkg/conduit" + "github.com/spf13/cobra" + "github.com/spf13/pflag" ) -type RootFlags struct { - Config string `long:"config" usage:"config file (default is $HOME/.conduit.yaml)" persistent:"true"` - Author string `long:"author" short:"a" usage:"author name for copyright attribution" persistent:"true"` - License string `long:"license" short:"l" usage:"name of license for the project" persistent:"true"` - Viper bool `long:"viper" usage:"use Viper for configuration" persistent:"true"` -} - -type RootCommand struct { - flags RootFlags -} - -func (c *RootCommand) Execute(ctx context.Context) error { - global.New().Run() - return nil -} - var ( - _ ecdysis.CommandWithFlags = (*RootCommand)(nil) - _ ecdysis.CommandWithDocs = (*RootCommand)(nil) - _ ecdysis.CommandWithSubCommands = (*RootCommand)(nil) - _ ecdysis.CommandWithExecute = (*RootCommand)(nil) + initArgs InitArgs ) -func (c *RootCommand) Usage() string { return "conduit" } -func (c *RootCommand) Flags() []ecdysis.Flag { - flags := ecdysis.BuildFlags(&c.flags) +type Instance struct { + rootCmd *cobra.Command +} - flags.SetDefault("author", "YOUR NAME") - flags.SetDefault("viper", true) - return flags +// New creates a new CLI Instance. +func New() *Instance { + return &Instance{ + rootCmd: buildRootCmd(), + } } -func (c *RootCommand) Docs() ecdysis.Docs { - return ecdysis.Docs{ - Short: "Conduit CLI", - Long: `Conduit CLI is a command-line that helps you interact with and manage Conduit.`, +func (i *Instance) Run() { + if err := i.rootCmd.Execute(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) } } -func (c *RootCommand) SubCommands() []ecdysis.Command { - return []ecdysis.Command{ - &version.VersionCommand{}, +func buildRootCmd() *cobra.Command { + cfg := conduit.DefaultConfig() + + cmd := &cobra.Command{ + Use: "conduit", + Short: "Conduit CLI", + Long: "Conduit CLI is a command-line that helps you interact with and manage Conduit.", + Version: conduit.Version(true), + Run: func(cmd *cobra.Command, args []string) { + e := &conduit.Entrypoint{} + e.Serve(cfg) + }, } + cmd.CompletionOptions.DisableDefaultCmd = true + conduit.Flags(&cfg).VisitAll(cmd.Flags().AddGoFlag) + + // init + cmd.AddCommand(buildInitCmd()) + + // pipelines + cmd.AddGroup(&cobra.Group{ + ID: "pipelines", + Title: "Pipelines", + }) + cmd.AddCommand(pipelines.BuildPipelinesCmd()) + + // mark hidden flags + cmd.Flags().VisitAll(func(f *pflag.Flag) { + if conduit.HiddenFlags[f.Name] { + err := cmd.Flags().MarkHidden(f.Name) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to mark flag %q as hidden: %v", f.Name, err) + } + } + }) + + return cmd } diff --git a/cmd/conduit/global/cli_test.go b/cmd/conduit/root/root_test.go similarity index 99% rename from cmd/conduit/global/cli_test.go rename to cmd/conduit/root/root_test.go index 0511c3379..b399858b4 100644 --- a/cmd/conduit/global/cli_test.go +++ b/cmd/conduit/root/root_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package global +package root import ( "bytes" diff --git a/cmd/conduit/root/version/version.go b/cmd/conduit/root/version/version.go deleted file mode 100644 index b8fab764b..000000000 --- a/cmd/conduit/root/version/version.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright © 2022 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package version - -import ( - "context" - "fmt" - - "github.com/conduitio/ecdysis" -) - -type VersionCommand struct{} - -var ( - _ ecdysis.CommandWithExecute = (*VersionCommand)(nil) - _ ecdysis.CommandWithDocs = (*VersionCommand)(nil) -) - -func (c *VersionCommand) Usage() string { return "version" } -func (c *VersionCommand) Docs() ecdysis.Docs { - return ecdysis.Docs{ - Short: "Print the version number of conduit", - } -} - -func (c *VersionCommand) Execute(context.Context) error { - fmt.Println("conduit v0.1.0") - return nil -} From 34dd04cb545bf291892f27a47b2602486768f934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 19 Nov 2024 15:36:57 +0100 Subject: [PATCH 03/22] update root with global flags --- cmd/conduit/main.go | 15 +++- cmd/conduit/root/init_ecdysis.go | 53 ++++++++++++ cmd/conduit/root/root.go | 4 +- cmd/conduit/root/root_ecdysis.go | 143 +++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 cmd/conduit/root/init_ecdysis.go create mode 100644 cmd/conduit/root/root_ecdysis.go diff --git a/cmd/conduit/main.go b/cmd/conduit/main.go index 3dc10c7e4..e20de4c78 100644 --- a/cmd/conduit/main.go +++ b/cmd/conduit/main.go @@ -14,8 +14,19 @@ package main -import "github.com/conduitio/conduit/cmd/conduit/root" +import ( + "fmt" + "os" + + "github.com/conduitio/conduit/cmd/conduit/root" + "github.com/conduitio/ecdysis" +) func main() { - root.New().Run() + e := ecdysis.New() + cmd := e.MustBuildCobraCommand(&root.RootCommand{}) + if err := cmd.Execute(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } } diff --git a/cmd/conduit/root/init_ecdysis.go b/cmd/conduit/root/init_ecdysis.go new file mode 100644 index 000000000..503e961b2 --- /dev/null +++ b/cmd/conduit/root/init_ecdysis.go @@ -0,0 +1,53 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package root + +import ( + "context" + + "github.com/conduitio/ecdysis" +) + +var ( + _ ecdysis.CommandWithFlags = (*InitCommand)(nil) + _ ecdysis.CommandWithExecute = (*InitCommand)(nil) + _ ecdysis.CommandWithDocs = (*InitCommand)(nil) +) + +type InitFlags struct { + ConfigPath string `flag:"config.path" usage:"path where Conduit will be initialized"` +} + +type InitCommand struct { + flags InitFlags +} + +func (c *InitCommand) Usage() string { return "init" } + +func (c *InitCommand) Docs() ecdysis.Docs { + return ecdysis.Docs{ + Short: `Initialize Conduit with a configuration file and directories.`, + } +} + +func (c *InitCommand) Execute(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *InitCommand) Flags() []ecdysis.Flag { + //TODO implement me + panic("implement me") +} diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index fb28f4d39..e61327218 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -59,11 +59,13 @@ func buildRootCmd() *cobra.Command { e.Serve(cfg) }, } + cmd.CompletionOptions.DisableDefaultCmd = true conduit.Flags(&cfg).VisitAll(cmd.Flags().AddGoFlag) // init - cmd.AddCommand(buildInitCmd()) + initCmd := buildInitCmd() + cmd.AddCommand(initCmd) // pipelines cmd.AddGroup(&cobra.Group{ diff --git a/cmd/conduit/root/root_ecdysis.go b/cmd/conduit/root/root_ecdysis.go new file mode 100644 index 000000000..f040929f7 --- /dev/null +++ b/cmd/conduit/root/root_ecdysis.go @@ -0,0 +1,143 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package root + +import ( + "context" + "fmt" + "time" + + "github.com/conduitio/conduit/pkg/conduit" + "github.com/conduitio/ecdysis" +) + +var ( + _ ecdysis.CommandWithFlags = (*RootCommand)(nil) + _ ecdysis.CommandWithExecute = (*RootCommand)(nil) + _ ecdysis.CommandWithDocs = (*RootCommand)(nil) + _ ecdysis.CommandWithSubCommands = (*RootCommand)(nil) +) + +// TODO: Check which ones are really global from the design document +type RootFlags struct { + // Database configuration + DBType string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite" persistent:"true"` + DBBadgerPath string `long:"db.badger.path" usage:"path to badger DB" persistent:"true"` + DBPostgresConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format" persistent:"true"` + DBPostgresTable string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)" persistent:"true"` + DBSQLitePath string `long:"db.sqlite.path" usage:"path to sqlite3 DB" persistent:"true"` + DBSQLiteTable string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)" persistent:"true"` + + // API configuration + APIEnabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API" persistent:"true"` + APIHTTPAddress string `long:"http.address" usage:"address for serving the HTTP API" persistent:"true"` + APIGRPCAddress string `long:"grpc.address" usage:"address for serving the gRPC API" persistent:"true"` + + // Logging configuration + LogLevel string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace" persistent:"true"` + LogFormat string `long:"log.format" usage:"sets the format of the logging; accepts json, cli" persistent:"true"` + + // Connectors and Processors paths + ConnectorsPath string `long:"connectors.path" usage:"path to standalone connectors' directory" persistent:"true"` + ProcessorsPath string `long:"processors.path" usage:"path to standalone processors' directory" persistent:"true"` + + // Pipeline configuration + PipelinesPath string `long:"pipelines.path" usage:"path to the directory that has the yaml pipeline configuration files, or a single pipeline configuration file" persistent:"true"` + PipelinesExitOnDegraded bool `long:"pipelines.exit-on-degraded" usage:"exit Conduit if a pipeline enters a degraded state" persistent:"true"` + PipelinesErrorRecoveryMinDelay time.Duration `long:"pipelines.error-recovery.min-delay" usage:"minimum delay before restart" persistent:"true"` + PipelinesErrorRecoveryMaxDelay time.Duration `long:"pipelines.error-recovery.max-delay" usage:"maximum delay before restart" persistent:"true"` + PipelinesErrorRecoveryBackoffFactor int `long:"pipelines.error-recovery.backoff-factor" usage:"backoff factor applied to the last delay" persistent:"true"` + PipelinesErrorRecoveryMaxRetries int64 `long:"pipelines.error-recovery.max-retries" usage:"maximum number of retries" persistent:"true"` + PipelinesErrorRecoveryMaxRetriesWindow time.Duration `long:"pipelines.error-recovery.max-retries-window" usage:"amount of time running without any errors after which a pipeline is considered healthy" persistent:"true"` + + // Schema registry configuration + SchemaRegistryType string `long:"schema-registry.type" usage:"schema registry type; accepts builtin,confluent" persistent:"true"` + SchemaRegistryConfluentConnectionString string `long:"schema-registry.confluent.connection-string" usage:"confluent schema registry connection string" persistent:"true"` + + // Preview features + PreviewPipelineArchV2 bool `long:"preview.pipeline-arch-v2" usage:"enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)" persistent:"true"` + + // Development profiling + DevCPUProfile string `long:"dev.cpuprofile" usage:"write CPU profile to file" persistent:"true"` + DevMemProfile string `long:"dev.memprofile" usage:"write memory profile to file" persistent:"true"` + DevBlockProfile string `long:"dev.blockprofile" usage:"write block profile to file" persistent:"true"` + + // Version + Version bool `long:"version" short:"v" usage:"show version" persistent:"true"` +} + +type RootCommand struct { + flags RootFlags + cfg conduit.Config +} + +func (c *RootCommand) Execute(ctx context.Context) error { + if c.flags.Version { + // TODO: use the logger instead + fmt.Print(conduit.Version(true)) + return nil + } + + e := &conduit.Entrypoint{} + e.Serve(c.cfg) + + return nil +} + +func (c *RootCommand) Usage() string { return "conduit" } +func (c *RootCommand) Flags() []ecdysis.Flag { + flags := ecdysis.BuildFlags(&c.flags) + + c.cfg = conduit.DefaultConfig() + + flags.SetDefault("db.type", c.cfg.DB.Type) + flags.SetDefault("db.badger.path", c.cfg.DB.Badger.Path) + flags.SetDefault("db.postgres.connection-string", c.cfg.DB.Postgres.ConnectionString) + flags.SetDefault("db.postgres.table", c.cfg.DB.Postgres.Table) + flags.SetDefault("db.sqlite.path", c.cfg.DB.SQLite.Path) + flags.SetDefault("db.sqlite.table", c.cfg.DB.SQLite.Table) + flags.SetDefault("api.enabled", c.cfg.API.Enabled) + flags.SetDefault("http.address", c.cfg.API.HTTP.Address) + flags.SetDefault("grpc.address", c.cfg.API.GRPC.Address) + flags.SetDefault("log.level", c.cfg.Log.Level) + flags.SetDefault("log.format", c.cfg.Log.Format) + flags.SetDefault("connectors.path", c.cfg.Connectors.Path) + flags.SetDefault("processors.path", c.cfg.Processors.Path) + flags.SetDefault("pipelines.path", c.cfg.Pipelines.Path) + flags.SetDefault("pipelines.exit-on-degraded", c.cfg.Pipelines.ExitOnDegraded) + flags.SetDefault("pipelines.error-recovery.min-delay", c.cfg.Pipelines.ErrorRecovery.MinDelay) + flags.SetDefault("pipelines.error-recovery.max-delay", c.cfg.Pipelines.ErrorRecovery.MaxDelay) + flags.SetDefault("pipelines.error-recovery.backoff-factor", c.cfg.Pipelines.ErrorRecovery.BackoffFactor) + flags.SetDefault("pipelines.error-recovery.max-retries", c.cfg.Pipelines.ErrorRecovery.MaxRetries) + flags.SetDefault("pipelines.error-recovery.max-retries-window", c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow) + flags.SetDefault("schema-registry.type", c.cfg.SchemaRegistry.Type) + flags.SetDefault("schema-registry.confluent.connection-string", c.cfg.SchemaRegistry.Confluent.ConnectionString) + flags.SetDefault("preview.pipeline-arch-v2", c.cfg.Preview.PipelineArchV2) + + return flags +} + +func (c *RootCommand) Docs() ecdysis.Docs { + return ecdysis.Docs{ + Short: "Conduit CLI", + Long: `Conduit CLI is a command-line that helps you interact with and manage Conduit.`, + } +} + +func (c *RootCommand) SubCommands() []ecdysis.Command { + return []ecdysis.Command{ + // inject root flags in sub-command + } +} From 2abe751315b0a4744ca025901af809c53077973b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 20 Nov 2024 16:07:55 +0100 Subject: [PATCH 04/22] update new init --- cmd/conduit/root/init_ecdysis.go | 89 ++++++++++++++++++++++++++++++-- cmd/conduit/root/root_ecdysis.go | 2 +- 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/cmd/conduit/root/init_ecdysis.go b/cmd/conduit/root/init_ecdysis.go index 503e961b2..75d418027 100644 --- a/cmd/conduit/root/init_ecdysis.go +++ b/cmd/conduit/root/init_ecdysis.go @@ -16,8 +16,16 @@ package root import ( "context" + "flag" + "fmt" + "os" + "path/filepath" + "github.com/conduitio/conduit/cmd/conduit/internal" + "github.com/conduitio/conduit/pkg/conduit" + "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/ecdysis" + "github.com/conduitio/yaml/v3" ) var ( @@ -42,12 +50,85 @@ func (c *InitCommand) Docs() ecdysis.Docs { } } +func (c *InitCommand) createDirs() error { + dirs := []string{"processors", "connectors", "pipelines"} + + for _, dir := range dirs { + path := filepath.Join(c.flags.ConfigPath, dir) + + // Attempt to create the directory, skipping if it already exists + if err := os.Mkdir(path, os.ModePerm); err != nil { + if os.IsExist(err) { + fmt.Printf("Directory '%s' already exists, skipping...\n", path) + continue + } + return fmt.Errorf("failed to create directory '%s': %w", path, err) + } + + fmt.Printf("Created directory: %s\n", path) + } + + return nil +} + +func (c *InitCommand) conduitCfgFlags() *flag.FlagSet { + cfg := conduit.DefaultConfigWithBasePath(c.flags.ConfigPath) + return conduit.Flags(&cfg) +} + +func (c *InitCommand) createConfigYAML() error { + cfgYAML := internal.NewYAMLTree() + c.conduitCfgFlags().VisitAll(func(f *flag.Flag) { + cfgYAML.Insert(f.Name, f.DefValue, f.Usage) + }) + + yamlData, err := yaml.Marshal(cfgYAML.Root) + if err != nil { + return cerrors.Errorf("error marshaling YAML: %w\n", err) + } + + path := filepath.Join(c.flags.ConfigPath, "conduit.yaml") + err = os.WriteFile(path, yamlData, 0o600) + if err != nil { + return cerrors.Errorf("error writing conduit.yaml: %w", err) + } + fmt.Printf("Configuration file written to %v\n", path) + + return nil +} + func (c *InitCommand) Execute(ctx context.Context) error { - //TODO implement me - panic("implement me") + err := c.createDirs() + if err != nil { + return err + } + + err = c.createConfigYAML() + if err != nil { + return fmt.Errorf("failed to create config YAML: %w", err) + } + + fmt.Println(` +Conduit has been initialized! + +To quickly create an example pipeline, run 'conduit pipelines init'. +To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) + + return nil + } func (c *InitCommand) Flags() []ecdysis.Flag { - //TODO implement me - panic("implement me") + flags := ecdysis.BuildFlags(&c.flags) + + // Set current working directory as default + currentPath, err := os.Getwd() + if err != nil { + panic(cerrors.Errorf("failed to get current working directory: %w", err)) + } + + flags.SetDefault("config.path", currentPath) + + return flags + } diff --git a/cmd/conduit/root/root_ecdysis.go b/cmd/conduit/root/root_ecdysis.go index f040929f7..9a6af96af 100644 --- a/cmd/conduit/root/root_ecdysis.go +++ b/cmd/conduit/root/root_ecdysis.go @@ -138,6 +138,6 @@ func (c *RootCommand) Docs() ecdysis.Docs { func (c *RootCommand) SubCommands() []ecdysis.Command { return []ecdysis.Command{ - // inject root flags in sub-command + &InitCommand{}, } } From d191bb4beb7dff5d6fcaced93d33bc9f0efc507b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 20 Nov 2024 18:17:50 +0100 Subject: [PATCH 05/22] finish migrating commands --- cmd/conduit/root/init.go | 128 +++--- cmd/conduit/root/init_ecdysis.go | 134 ------ .../root/pipelines/format_parameter.go | 111 +++++ cmd/conduit/root/pipelines/init.go | 383 ++++++------------ cmd/conduit/root/pipelines/pipelines.go | 29 +- cmd/conduit/root/pipelines/template.go | 14 + cmd/conduit/root/root.go | 161 +++++--- cmd/conduit/root/root_ecdysis.go | 143 ------- cmd/conduit/root/root_test.go | 83 ---- 9 files changed, 432 insertions(+), 754 deletions(-) delete mode 100644 cmd/conduit/root/init_ecdysis.go create mode 100644 cmd/conduit/root/pipelines/format_parameter.go create mode 100644 cmd/conduit/root/pipelines/template.go delete mode 100644 cmd/conduit/root/root_ecdysis.go delete mode 100644 cmd/conduit/root/root_test.go diff --git a/cmd/conduit/root/init.go b/cmd/conduit/root/init.go index 3030501cb..75d418027 100644 --- a/cmd/conduit/root/init.go +++ b/cmd/conduit/root/init.go @@ -15,6 +15,7 @@ package root import ( + "context" "flag" "fmt" "os" @@ -23,48 +24,61 @@ import ( "github.com/conduitio/conduit/cmd/conduit/internal" "github.com/conduitio/conduit/pkg/conduit" "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/ecdysis" "github.com/conduitio/yaml/v3" - "github.com/spf13/cobra" ) -type InitArgs struct { - Path string -} +var ( + _ ecdysis.CommandWithFlags = (*InitCommand)(nil) + _ ecdysis.CommandWithExecute = (*InitCommand)(nil) + _ ecdysis.CommandWithDocs = (*InitCommand)(nil) +) -type ConduitInit struct { - args InitArgs +type InitFlags struct { + ConfigPath string `flag:"config.path" usage:"path where Conduit will be initialized"` } -func NewConduitInit(args InitArgs) *ConduitInit { - return &ConduitInit{args: args} +type InitCommand struct { + flags InitFlags } -func (i *ConduitInit) Run() error { - err := i.createDirs() - if err != nil { - return err - } +func (c *InitCommand) Usage() string { return "init" } - err = i.createConfigYAML() - if err != nil { - return fmt.Errorf("failed to create config YAML: %w", err) +func (c *InitCommand) Docs() ecdysis.Docs { + return ecdysis.Docs{ + Short: `Initialize Conduit with a configuration file and directories.`, } +} - fmt.Println(` -Conduit has been initialized! +func (c *InitCommand) createDirs() error { + dirs := []string{"processors", "connectors", "pipelines"} -To quickly create an example pipeline, run 'conduit pipelines init'. -To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) + for _, dir := range dirs { + path := filepath.Join(c.flags.ConfigPath, dir) + + // Attempt to create the directory, skipping if it already exists + if err := os.Mkdir(path, os.ModePerm); err != nil { + if os.IsExist(err) { + fmt.Printf("Directory '%s' already exists, skipping...\n", path) + continue + } + return fmt.Errorf("failed to create directory '%s': %w", path, err) + } + + fmt.Printf("Created directory: %s\n", path) + } return nil } -func (i *ConduitInit) createConfigYAML() error { +func (c *InitCommand) conduitCfgFlags() *flag.FlagSet { + cfg := conduit.DefaultConfigWithBasePath(c.flags.ConfigPath) + return conduit.Flags(&cfg) +} + +func (c *InitCommand) createConfigYAML() error { cfgYAML := internal.NewYAMLTree() - i.conduitCfgFlags().VisitAll(func(f *flag.Flag) { - if conduit.HiddenFlags[f.Name] { - return // hide flag from output - } + c.conduitCfgFlags().VisitAll(func(f *flag.Flag) { cfgYAML.Insert(f.Name, f.DefValue, f.Usage) }) @@ -73,7 +87,7 @@ func (i *ConduitInit) createConfigYAML() error { return cerrors.Errorf("error marshaling YAML: %w\n", err) } - path := filepath.Join(i.path(), "conduit.yaml") + path := filepath.Join(c.flags.ConfigPath, "conduit.yaml") err = os.WriteFile(path, yamlData, 0o600) if err != nil { return cerrors.Errorf("error writing conduit.yaml: %w", err) @@ -83,60 +97,38 @@ func (i *ConduitInit) createConfigYAML() error { return nil } -func (i *ConduitInit) createDirs() error { - dirs := []string{"processors", "connectors", "pipelines"} +func (c *InitCommand) Execute(ctx context.Context) error { + err := c.createDirs() + if err != nil { + return err + } - for _, dir := range dirs { - path := filepath.Join(i.path(), dir) + err = c.createConfigYAML() + if err != nil { + return fmt.Errorf("failed to create config YAML: %w", err) + } - // Attempt to create the directory, skipping if it already exists - if err := os.Mkdir(path, os.ModePerm); err != nil { - if os.IsExist(err) { - fmt.Printf("Directory '%s' already exists, skipping...\n", path) - continue - } - return fmt.Errorf("failed to create directory '%s': %w", path, err) - } + fmt.Println(` +Conduit has been initialized! - fmt.Printf("Created directory: %s\n", path) - } +To quickly create an example pipeline, run 'conduit pipelines init'. +To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) return nil -} -func (i *ConduitInit) conduitCfgFlags() *flag.FlagSet { - cfg := conduit.DefaultConfigWithBasePath(i.path()) - return conduit.Flags(&cfg) } -func (i *ConduitInit) path() string { - if i.args.Path != "" { - return i.args.Path - } +func (c *InitCommand) Flags() []ecdysis.Flag { + flags := ecdysis.BuildFlags(&c.flags) - path, err := os.Getwd() + // Set current working directory as default + currentPath, err := os.Getwd() if err != nil { panic(cerrors.Errorf("failed to get current working directory: %w", err)) } - return path -} + flags.SetDefault("config.path", currentPath) + + return flags -func buildInitCmd() *cobra.Command { - initCmd := &cobra.Command{ - Use: "init", - Short: "Initialize Conduit with a configuration file and directories.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return NewConduitInit(initArgs).Run() - }, - } - initCmd.Flags().StringVar( - &initArgs.Path, - "config.path", - "", - "path where Conduit will be initialized", - ) - - return initCmd } diff --git a/cmd/conduit/root/init_ecdysis.go b/cmd/conduit/root/init_ecdysis.go deleted file mode 100644 index 75d418027..000000000 --- a/cmd/conduit/root/init_ecdysis.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package root - -import ( - "context" - "flag" - "fmt" - "os" - "path/filepath" - - "github.com/conduitio/conduit/cmd/conduit/internal" - "github.com/conduitio/conduit/pkg/conduit" - "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/ecdysis" - "github.com/conduitio/yaml/v3" -) - -var ( - _ ecdysis.CommandWithFlags = (*InitCommand)(nil) - _ ecdysis.CommandWithExecute = (*InitCommand)(nil) - _ ecdysis.CommandWithDocs = (*InitCommand)(nil) -) - -type InitFlags struct { - ConfigPath string `flag:"config.path" usage:"path where Conduit will be initialized"` -} - -type InitCommand struct { - flags InitFlags -} - -func (c *InitCommand) Usage() string { return "init" } - -func (c *InitCommand) Docs() ecdysis.Docs { - return ecdysis.Docs{ - Short: `Initialize Conduit with a configuration file and directories.`, - } -} - -func (c *InitCommand) createDirs() error { - dirs := []string{"processors", "connectors", "pipelines"} - - for _, dir := range dirs { - path := filepath.Join(c.flags.ConfigPath, dir) - - // Attempt to create the directory, skipping if it already exists - if err := os.Mkdir(path, os.ModePerm); err != nil { - if os.IsExist(err) { - fmt.Printf("Directory '%s' already exists, skipping...\n", path) - continue - } - return fmt.Errorf("failed to create directory '%s': %w", path, err) - } - - fmt.Printf("Created directory: %s\n", path) - } - - return nil -} - -func (c *InitCommand) conduitCfgFlags() *flag.FlagSet { - cfg := conduit.DefaultConfigWithBasePath(c.flags.ConfigPath) - return conduit.Flags(&cfg) -} - -func (c *InitCommand) createConfigYAML() error { - cfgYAML := internal.NewYAMLTree() - c.conduitCfgFlags().VisitAll(func(f *flag.Flag) { - cfgYAML.Insert(f.Name, f.DefValue, f.Usage) - }) - - yamlData, err := yaml.Marshal(cfgYAML.Root) - if err != nil { - return cerrors.Errorf("error marshaling YAML: %w\n", err) - } - - path := filepath.Join(c.flags.ConfigPath, "conduit.yaml") - err = os.WriteFile(path, yamlData, 0o600) - if err != nil { - return cerrors.Errorf("error writing conduit.yaml: %w", err) - } - fmt.Printf("Configuration file written to %v\n", path) - - return nil -} - -func (c *InitCommand) Execute(ctx context.Context) error { - err := c.createDirs() - if err != nil { - return err - } - - err = c.createConfigYAML() - if err != nil { - return fmt.Errorf("failed to create config YAML: %w", err) - } - - fmt.Println(` -Conduit has been initialized! - -To quickly create an example pipeline, run 'conduit pipelines init'. -To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) - - return nil - -} - -func (c *InitCommand) Flags() []ecdysis.Flag { - flags := ecdysis.BuildFlags(&c.flags) - - // Set current working directory as default - currentPath, err := os.Getwd() - if err != nil { - panic(cerrors.Errorf("failed to get current working directory: %w", err)) - } - - flags.SetDefault("config.path", currentPath) - - return flags - -} diff --git a/cmd/conduit/root/pipelines/format_parameter.go b/cmd/conduit/root/pipelines/format_parameter.go new file mode 100644 index 000000000..4506db947 --- /dev/null +++ b/cmd/conduit/root/pipelines/format_parameter.go @@ -0,0 +1,111 @@ +package pipelines + +import ( + "fmt" + "strings" + "text/template" + + "github.com/conduitio/conduit-commons/config" +) + +var funcMap = template.FuncMap{ + "formatParameterValueTable": formatParameterValueTable, + "formatParameterValueYAML": formatParameterValueYAML, + "formatParameterDescriptionYAML": formatParameterDescriptionYAML, + "formatParameterRequired": formatParameterRequired, +} + +func formatParameterRequired(param config.Parameter) string { + for _, v := range param.Validations { + if v.Type() == config.ValidationTypeRequired { + return "Required" + } + } + + return "Optional" +} + +// formatParameterValue formats the value of a configuration parameter. +func formatParameterValueTable(value string) string { + switch { + case value == "": + return `` + case strings.Contains(value, "\n"): + // specifically used in the javascript processor + return fmt.Sprintf("\n```js\n%s\n```\n", value) + default: + return fmt.Sprintf("`%s`", value) + } +} + +func formatParameterDescriptionYAML(description string) string { + const ( + indentLen = 10 + prefix = "# " + lineLen = 80 + tmpNewLine = "〠" + ) + + // remove markdown new lines + description = strings.ReplaceAll(description, "\n\n", tmpNewLine) + description = strings.ReplaceAll(description, "\n", " ") + description = strings.ReplaceAll(description, tmpNewLine, "\n") + + formattedDescription := formatMultiline(description, strings.Repeat(" ", indentLen)+prefix, lineLen) + // remove first indent and last new line + formattedDescription = formattedDescription[indentLen : len(formattedDescription)-1] + return formattedDescription +} + +func formatMultiline( + input string, + prefix string, + maxLineLen int, +) string { + textLen := maxLineLen - len(prefix) + + // split the input into lines of length textLen + lines := strings.Split(input, "\n") + var formattedLines []string + for _, line := range lines { + if len(line) <= textLen { + formattedLines = append(formattedLines, line) + continue + } + + // split the line into multiple lines, don't break words + words := strings.Fields(line) + var formattedLine string + for _, word := range words { + if len(formattedLine)+len(word) > textLen { + formattedLines = append(formattedLines, formattedLine[1:]) + formattedLine = "" + } + formattedLine += " " + word + } + if formattedLine != "" { + formattedLines = append(formattedLines, formattedLine[1:]) + } + } + + // combine lines including indent and prefix + var formatted string + for _, line := range formattedLines { + formatted += prefix + line + "\n" + } + + return formatted +} + +func formatParameterValueYAML(value string) string { + switch { + case value == "": + return `""` + case strings.Contains(value, "\n"): + // specifically used in the javascript processor + formattedValue := formatMultiline(value, " ", 10000) + return fmt.Sprintf("|\n%s", formattedValue) + default: + return fmt.Sprintf(`'%s'`, value) + } +} diff --git a/cmd/conduit/root/pipelines/init.go b/cmd/conduit/root/pipelines/init.go index 66e15ea51..ac3052681 100644 --- a/cmd/conduit/root/pipelines/init.go +++ b/cmd/conduit/root/pipelines/init.go @@ -15,209 +15,131 @@ package pipelines import ( - _ "embed" + "context" "fmt" "log" "os" "path/filepath" - "strings" "text/template" "github.com/conduitio/conduit-commons/config" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/plugin" "github.com/conduitio/conduit/pkg/plugin/connector/builtin" - "github.com/spf13/cobra" + "github.com/conduitio/ecdysis" ) -//go:embed pipeline.tmpl -var pipelineCfgTmpl string +const ( + defaultDestination = "file" + defaultSource = "generator" +) -var pipelinesInitArgs PipelinesInitArgs +var ( + _ ecdysis.CommandWithDocs = (*InitCommand)(nil) + _ ecdysis.CommandWithFlags = (*InitCommand)(nil) + _ ecdysis.CommandWithArgs = (*InitCommand)(nil) + _ ecdysis.CommandWithExecute = (*InitCommand)(nil) -var funcMap = template.FuncMap{ - "formatParameterValueTable": formatParameterValueTable, - "formatParameterValueYAML": formatParameterValueYAML, - "formatParameterDescriptionYAML": formatParameterDescriptionYAML, - "formatParameterRequired": formatParameterRequired, -} + pipelineCfgTmpl string +) -func formatParameterRequired(param config.Parameter) string { - for _, v := range param.Validations { - if v.Type() == config.ValidationTypeRequired { - return "Required" - } - } +type InitArgs struct { + name string +} - return "Optional" +type InitFlags struct { + Source string `long:"source" usage:"Source connector (any of the built-in connectors)."` + Destination string `long:"destination" usage:"Destination connector (any of the built-in connectors)."` + Path string `long:"pipelines.path" usage:"Path where the pipeline will be saved." default:"./pipelines"` } -// formatParameterValue formats the value of a configuration parameter. -func formatParameterValueTable(value string) string { - switch { - case value == "": - return `` - case strings.Contains(value, "\n"): - // specifically used in the javascript processor - return fmt.Sprintf("\n```js\n%s\n```\n", value) - default: - return fmt.Sprintf("`%s`", value) - } +type InitCommand struct { + args InitArgs + flags InitFlags } -func formatParameterDescriptionYAML(description string) string { - const ( - indentLen = 10 - prefix = "# " - lineLen = 80 - tmpNewLine = "〠" - ) - - // remove markdown new lines - description = strings.ReplaceAll(description, "\n\n", tmpNewLine) - description = strings.ReplaceAll(description, "\n", " ") - description = strings.ReplaceAll(description, tmpNewLine, "\n") - - formattedDescription := formatMultiline(description, strings.Repeat(" ", indentLen)+prefix, lineLen) - // remove first indent and last new line - formattedDescription = formattedDescription[indentLen : len(formattedDescription)-1] - return formattedDescription +func (c *InitCommand) configFilePath() string { + return filepath.Join(c.flags.Path, c.configFileName()) } -func formatMultiline( - input string, - prefix string, - maxLineLen int, -) string { - textLen := maxLineLen - len(prefix) - - // split the input into lines of length textLen - lines := strings.Split(input, "\n") - var formattedLines []string - for _, line := range lines { - if len(line) <= textLen { - formattedLines = append(formattedLines, line) - continue - } +func (c *InitCommand) configFileName() string { + return fmt.Sprintf("pipeline-%s.yaml", c.args.name) +} - // split the line into multiple lines, don't break words - words := strings.Fields(line) - var formattedLine string - for _, word := range words { - if len(formattedLine)+len(word) > textLen { - formattedLines = append(formattedLines, formattedLine[1:]) - formattedLine = "" - } - formattedLine += " " + word - } - if formattedLine != "" { - formattedLines = append(formattedLines, formattedLine[1:]) - } - } +func (c *InitCommand) Flags() []ecdysis.Flag { + flags := ecdysis.BuildFlags(&InitFlags{}) - // combine lines including indent and prefix - var formatted string - for _, line := range formattedLines { - formatted += prefix + line + "\n" - } + flags.SetDefault("pipelines.path", "./pipelines") - return formatted + return flags } -func formatParameterValueYAML(value string) string { - switch { - case value == "": - return `""` - case strings.Contains(value, "\n"): - // specifically used in the javascript processor - formattedValue := formatMultiline(value, " ", 10000) - return fmt.Sprintf("|\n%s", formattedValue) - default: - return fmt.Sprintf(`'%s'`, value) +func (c *InitCommand) Args(args []string) error { + if len(args) == 0 { + return cerrors.Errorf("requires a pipeline name") } -} -const ( - defaultDestination = "file" - defaultSource = "generator" -) - -type pipelineTemplate struct { - Name string - SourceSpec connectorTemplate - DestinationSpec connectorTemplate -} - -type connectorTemplate struct { - Name string - Params config.Parameters + if len(args) > 1 { + return cerrors.Errorf("too many arguments") + } + c.args.name = args[0] + return nil } -type PipelinesInitArgs struct { - Name string - Source string - Destination string - Path string -} +func (c *InitCommand) Usage() string { return "init" } -type PipelinesInit struct { - args PipelinesInitArgs +func (c *InitCommand) Docs() ecdysis.Docs { + return ecdysis.Docs{ + Short: "Initialize an example pipeline.", + Long: `Initialize a pipeline configuration file, with all of parameters for source and destination connectors +initialized and described. The source and destination connector can be chosen via flags. If no connectors are chosen, then +a simple and runnable generator-to-log pipeline is configured.`, + Example: "conduit pipelines init awesome-pipeline-name --source postgres --destination kafka --path pipelines/pg-to-kafka.yaml", + } } -func NewPipelinesInit(args PipelinesInitArgs) *PipelinesInit { - return &PipelinesInit{args: args} -} +func (c *InitCommand) getSourceParams() (connectorTemplate, error) { + for _, conn := range builtin.DefaultBuiltinConnectors { + specs := conn.NewSpecification() + if specs.Name == c.flags.Source || specs.Name == "builtin:"+c.flags.Source { + if conn.NewSource == nil { + return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", c.flags.Source) + } -func (pi *PipelinesInit) Run() error { - var pipeline pipelineTemplate - // if no source/destination arguments are provided, - // we build a runnable example pipeline - if pi.args.Source == "" && pi.args.Destination == "" { - pipeline = pi.buildDemoPipeline() - } else { - p, err := pi.buildTemplatePipeline() - if err != nil { - return err + return connectorTemplate{ + Name: specs.Name, + Params: conn.NewSource().Parameters(), + }, nil } - pipeline = p } - err := pi.write(pipeline) - if err != nil { - return cerrors.Errorf("could not write pipeline: %w", err) - } - - fmt.Printf(`Your pipeline has been initialized and created at %s. - -To run the pipeline, simply run 'conduit'.`, pi.configFilePath()) - - return nil + return connectorTemplate{}, cerrors.Errorf("%v: %w", c.flags.Source, plugin.ErrPluginNotFound) } -func (pi *PipelinesInit) buildTemplatePipeline() (pipelineTemplate, error) { - srcParams, err := pi.getSourceParams() - if err != nil { - return pipelineTemplate{}, cerrors.Errorf("failed getting source params: %w", err) - } +func (c *InitCommand) getDestinationParams() (connectorTemplate, error) { + for _, conn := range builtin.DefaultBuiltinConnectors { + specs := conn.NewSpecification() + if specs.Name == c.flags.Destination || specs.Name == "builtin:"+c.flags.Destination { + if conn.NewDestination == nil { + return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", c.flags.Destination) + } - dstParams, err := pi.getDestinationParams() - if err != nil { - return pipelineTemplate{}, cerrors.Errorf("failed getting destination params: %w", err) + return connectorTemplate{ + Name: specs.Name, + Params: conn.NewDestination().Parameters(), + }, nil + } } - return pipelineTemplate{ - Name: pi.pipelineName(), - SourceSpec: srcParams, - DestinationSpec: dstParams, - }, nil + return connectorTemplate{}, cerrors.Errorf("%v: %w", c.flags.Destination, plugin.ErrPluginNotFound) } -func (pi *PipelinesInit) buildDemoPipeline() pipelineTemplate { - srcParams, _ := pi.getSourceParams() - dstParams, _ := pi.getDestinationParams() +func (c *InitCommand) buildDemoPipeline() pipelineTemplate { + srcParams, _ := c.getSourceParams() + dstParams, _ := c.getDestinationParams() return pipelineTemplate{ - Name: pi.pipelineName(), + Name: c.args.name, SourceSpec: connectorTemplate{ Name: defaultSource, Params: map[string]config.Parameter{ @@ -257,22 +179,40 @@ func (pi *PipelinesInit) buildDemoPipeline() pipelineTemplate { } } -func (pi *PipelinesInit) getOutput() *os.File { - output, err := os.OpenFile(pi.configFilePath(), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) +func (c *InitCommand) buildTemplatePipeline() (pipelineTemplate, error) { + srcParams, err := c.getSourceParams() + if err != nil { + return pipelineTemplate{}, cerrors.Errorf("failed getting source params: %w", err) + } + + dstParams, err := c.getDestinationParams() if err != nil { - log.Fatalf("error: failed to open %s: %v", pi.args.Path, err) + return pipelineTemplate{}, cerrors.Errorf("failed getting destination params: %w", err) + } + + return pipelineTemplate{ + Name: c.args.name, + SourceSpec: srcParams, + DestinationSpec: dstParams, + }, nil +} + +func (c *InitCommand) getOutput() *os.File { + output, err := os.OpenFile(c.configFilePath(), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + log.Fatalf("error: failed to open %s: %v", c.configFilePath(), err) } return output } -func (pi *PipelinesInit) write(pipeline pipelineTemplate) error { +func (c *InitCommand) write(pipeline pipelineTemplate) error { t, err := template.New("").Funcs(funcMap).Option("missingkey=zero").Parse(pipelineCfgTmpl) if err != nil { return cerrors.Errorf("failed parsing template: %w", err) } - output := pi.getOutput() + output := c.getOutput() defer output.Close() err = t.Execute(output, pipeline) @@ -283,115 +223,28 @@ func (pi *PipelinesInit) write(pipeline pipelineTemplate) error { return nil } -func (pi *PipelinesInit) getSourceParams() (connectorTemplate, error) { - for _, conn := range builtin.DefaultBuiltinConnectors { - specs := conn.NewSpecification() - if specs.Name == pi.sourceConnector() || specs.Name == "builtin:"+pi.sourceConnector() { - if conn.NewSource == nil { - return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", pi.sourceConnector()) - } - - return connectorTemplate{ - Name: specs.Name, - Params: conn.NewSource().Parameters(), - }, nil - } - } - - return connectorTemplate{}, cerrors.Errorf("%v: %w", pi.sourceConnector(), plugin.ErrPluginNotFound) -} - -func (pi *PipelinesInit) getDestinationParams() (connectorTemplate, error) { - for _, conn := range builtin.DefaultBuiltinConnectors { - specs := conn.NewSpecification() - if specs.Name == pi.destinationConnector() || specs.Name == "builtin:"+pi.destinationConnector() { - if conn.NewDestination == nil { - return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", pi.destinationConnector()) - } - - return connectorTemplate{ - Name: specs.Name, - Params: conn.NewDestination().Parameters(), - }, nil +func (c *InitCommand) Execute(_ context.Context) error { + var pipeline pipelineTemplate + // if no source/destination arguments are provided, + // we build a runnable example pipeline + if c.flags.Source == "" && c.flags.Destination == "" { + pipeline = c.buildDemoPipeline() + } else { + p, err := c.buildTemplatePipeline() + if err != nil { + return err } + pipeline = p } - return connectorTemplate{}, cerrors.Errorf("%v: %w", pi.destinationConnector(), plugin.ErrPluginNotFound) -} - -func (pi *PipelinesInit) configFilePath() string { - path := pi.args.Path - if path == "" { - path = "./pipelines" - } - - return filepath.Join(path, pi.configFileName()) -} - -func (pi *PipelinesInit) configFileName() string { - return fmt.Sprintf("pipeline-%s.yaml", pi.pipelineName()) -} - -func (pi *PipelinesInit) sourceConnector() string { - if pi.args.Source != "" { - return pi.args.Source - } - - return defaultSource -} - -func (pi *PipelinesInit) destinationConnector() string { - if pi.args.Destination != "" { - return pi.args.Destination - } - - return defaultDestination -} - -func (pi *PipelinesInit) pipelineName() string { - if pi.args.Name != "" { - return pi.args.Name + err := c.write(pipeline) + if err != nil { + return cerrors.Errorf("could not write pipeline: %w", err) } - return fmt.Sprintf("%s-to-%s", pi.sourceConnector(), pi.destinationConnector()) -} + fmt.Printf(`Your pipeline has been initialized and created at %s. -func BuildPipelinesInitCmd() *cobra.Command { - pipelinesInitCmd := &cobra.Command{ - Use: "init [pipeline-name]", - Short: "Initialize an example pipeline.", - Long: `Initialize a pipeline configuration file, with all of parameters for source and destination connectors -initialized and described. The source and destination connector can be chosen via flags. If no connectors are chosen, then -a simple and runnable generator-to-log pipeline is configured.`, - Args: cobra.MaximumNArgs(1), - Example: " conduit pipelines init awesome-pipeline-name --source postgres --destination kafka --path pipelines/pg-to-kafka.yaml", - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) > 0 { - pipelinesInitArgs.Name = args[0] - } - return NewPipelinesInit(pipelinesInitArgs).Run() - }, - } +To run the pipeline, simply run 'conduit'.`, c.configFilePath()) - // Add flags to pipelines init command - pipelinesInitCmd.Flags().StringVar( - &pipelinesInitArgs.Source, - "source", - "", - "Source connector (any of the built-in connectors).", - ) - pipelinesInitCmd.Flags().StringVar( - &pipelinesInitArgs.Destination, - "destination", - "", - "Destination connector (any of the built-in connectors).", - ) - pipelinesInitCmd.Flags().StringVar( - &pipelinesInitArgs.Path, - "pipelines.path", - "./pipelines", - "Path where the pipeline will be saved.", - ) - - return pipelinesInitCmd + return nil } diff --git a/cmd/conduit/root/pipelines/pipelines.go b/cmd/conduit/root/pipelines/pipelines.go index f130d2eb9..a733258fb 100644 --- a/cmd/conduit/root/pipelines/pipelines.go +++ b/cmd/conduit/root/pipelines/pipelines.go @@ -14,17 +14,28 @@ package pipelines -import "github.com/spf13/cobra" +import ( + "github.com/conduitio/ecdysis" +) -func BuildPipelinesCmd() *cobra.Command { - pipelinesCmd := &cobra.Command{ - Use: "pipelines", - Short: "Initialize and manage pipelines", - Args: cobra.NoArgs, - GroupID: "pipelines", +var ( + _ ecdysis.CommandWithDocs = (*PipelinesCommand)(nil) + _ ecdysis.CommandWithSubCommands = (*PipelinesCommand)(nil) +) + +type PipelinesCommand struct { +} + +func (c *PipelinesCommand) SubCommands() []ecdysis.Command { + return []ecdysis.Command{ + &InitCommand{}, } +} - pipelinesCmd.AddCommand(BuildPipelinesInitCmd()) +func (p *PipelinesCommand) Usage() string { return "pipelines" } - return pipelinesCmd +func (p *PipelinesCommand) Docs() ecdysis.Docs { + return ecdysis.Docs{ + Short: "Initialize and manage pipelines", + } } diff --git a/cmd/conduit/root/pipelines/template.go b/cmd/conduit/root/pipelines/template.go new file mode 100644 index 000000000..b030a2d1c --- /dev/null +++ b/cmd/conduit/root/pipelines/template.go @@ -0,0 +1,14 @@ +package pipelines + +import "github.com/conduitio/conduit-commons/config" + +type connectorTemplate struct { + Name string + Params config.Parameters +} + +type pipelineTemplate struct { + Name string + SourceSpec connectorTemplate + DestinationSpec connectorTemplate +} diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index e61327218..9d26cff9f 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -15,74 +15,131 @@ package root import ( + "context" "fmt" - "os" + "time" "github.com/conduitio/conduit/cmd/conduit/root/pipelines" "github.com/conduitio/conduit/pkg/conduit" - "github.com/spf13/cobra" - "github.com/spf13/pflag" + "github.com/conduitio/ecdysis" ) var ( - initArgs InitArgs + _ ecdysis.CommandWithFlags = (*RootCommand)(nil) + _ ecdysis.CommandWithExecute = (*RootCommand)(nil) + _ ecdysis.CommandWithDocs = (*RootCommand)(nil) + _ ecdysis.CommandWithSubCommands = (*RootCommand)(nil) ) -type Instance struct { - rootCmd *cobra.Command +// TODO: Check which ones are really global from the design document +type RootFlags struct { + // Database configuration + DBType string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite" persistent:"true"` + DBBadgerPath string `long:"db.badger.path" usage:"path to badger DB" persistent:"true"` + DBPostgresConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format" persistent:"true"` + DBPostgresTable string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)" persistent:"true"` + DBSQLitePath string `long:"db.sqlite.path" usage:"path to sqlite3 DB" persistent:"true"` + DBSQLiteTable string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)" persistent:"true"` + + // API configuration + APIEnabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API" persistent:"true"` + APIHTTPAddress string `long:"http.address" usage:"address for serving the HTTP API" persistent:"true"` + APIGRPCAddress string `long:"grpc.address" usage:"address for serving the gRPC API" persistent:"true"` + + // Logging configuration + LogLevel string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace" persistent:"true"` + LogFormat string `long:"log.format" usage:"sets the format of the logging; accepts json, cli" persistent:"true"` + + // Connectors and Processors paths + ConnectorsPath string `long:"connectors.path" usage:"path to standalone connectors' directory" persistent:"true"` + ProcessorsPath string `long:"processors.path" usage:"path to standalone processors' directory" persistent:"true"` + + // Pipeline configuration + PipelinesPath string `long:"pipelines.path" usage:"path to the directory that has the yaml pipeline configuration files, or a single pipeline configuration file" persistent:"true"` + PipelinesExitOnDegraded bool `long:"pipelines.exit-on-degraded" usage:"exit Conduit if a pipeline enters a degraded state" persistent:"true"` + PipelinesErrorRecoveryMinDelay time.Duration `long:"pipelines.error-recovery.min-delay" usage:"minimum delay before restart" persistent:"true"` + PipelinesErrorRecoveryMaxDelay time.Duration `long:"pipelines.error-recovery.max-delay" usage:"maximum delay before restart" persistent:"true"` + PipelinesErrorRecoveryBackoffFactor int `long:"pipelines.error-recovery.backoff-factor" usage:"backoff factor applied to the last delay" persistent:"true"` + PipelinesErrorRecoveryMaxRetries int64 `long:"pipelines.error-recovery.max-retries" usage:"maximum number of retries" persistent:"true"` + PipelinesErrorRecoveryMaxRetriesWindow time.Duration `long:"pipelines.error-recovery.max-retries-window" usage:"amount of time running without any errors after which a pipeline is considered healthy" persistent:"true"` + + // Schema registry configuration + SchemaRegistryType string `long:"schema-registry.type" usage:"schema registry type; accepts builtin,confluent" persistent:"true"` + SchemaRegistryConfluentConnectionString string `long:"schema-registry.confluent.connection-string" usage:"confluent schema registry connection string" persistent:"true"` + + // Preview features + PreviewPipelineArchV2 bool `long:"preview.pipeline-arch-v2" usage:"enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)" persistent:"true"` + + // Development profiling + DevCPUProfile string `long:"dev.cpuprofile" usage:"write CPU profile to file" persistent:"true"` + DevMemProfile string `long:"dev.memprofile" usage:"write memory profile to file" persistent:"true"` + DevBlockProfile string `long:"dev.blockprofile" usage:"write block profile to file" persistent:"true"` + + // Version + Version bool `long:"version" short:"v" usage:"show version" persistent:"true"` } -// New creates a new CLI Instance. -func New() *Instance { - return &Instance{ - rootCmd: buildRootCmd(), - } +type RootCommand struct { + flags RootFlags + cfg conduit.Config } -func (i *Instance) Run() { - if err := i.rootCmd.Execute(); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) +func (c *RootCommand) Execute(ctx context.Context) error { + if c.flags.Version { + // TODO: use the logger instead + fmt.Print(conduit.Version(true)) + return nil } + + e := &conduit.Entrypoint{} + e.Serve(c.cfg) + + return nil } -func buildRootCmd() *cobra.Command { - cfg := conduit.DefaultConfig() - - cmd := &cobra.Command{ - Use: "conduit", - Short: "Conduit CLI", - Long: "Conduit CLI is a command-line that helps you interact with and manage Conduit.", - Version: conduit.Version(true), - Run: func(cmd *cobra.Command, args []string) { - e := &conduit.Entrypoint{} - e.Serve(cfg) - }, +func (c *RootCommand) Usage() string { return "conduit" } +func (c *RootCommand) Flags() []ecdysis.Flag { + flags := ecdysis.BuildFlags(&c.flags) + + c.cfg = conduit.DefaultConfig() + + flags.SetDefault("db.type", c.cfg.DB.Type) + flags.SetDefault("db.badger.path", c.cfg.DB.Badger.Path) + flags.SetDefault("db.postgres.connection-string", c.cfg.DB.Postgres.ConnectionString) + flags.SetDefault("db.postgres.table", c.cfg.DB.Postgres.Table) + flags.SetDefault("db.sqlite.path", c.cfg.DB.SQLite.Path) + flags.SetDefault("db.sqlite.table", c.cfg.DB.SQLite.Table) + flags.SetDefault("api.enabled", c.cfg.API.Enabled) + flags.SetDefault("http.address", c.cfg.API.HTTP.Address) + flags.SetDefault("grpc.address", c.cfg.API.GRPC.Address) + flags.SetDefault("log.level", c.cfg.Log.Level) + flags.SetDefault("log.format", c.cfg.Log.Format) + flags.SetDefault("connectors.path", c.cfg.Connectors.Path) + flags.SetDefault("processors.path", c.cfg.Processors.Path) + flags.SetDefault("pipelines.path", c.cfg.Pipelines.Path) + flags.SetDefault("pipelines.exit-on-degraded", c.cfg.Pipelines.ExitOnDegraded) + flags.SetDefault("pipelines.error-recovery.min-delay", c.cfg.Pipelines.ErrorRecovery.MinDelay) + flags.SetDefault("pipelines.error-recovery.max-delay", c.cfg.Pipelines.ErrorRecovery.MaxDelay) + flags.SetDefault("pipelines.error-recovery.backoff-factor", c.cfg.Pipelines.ErrorRecovery.BackoffFactor) + flags.SetDefault("pipelines.error-recovery.max-retries", c.cfg.Pipelines.ErrorRecovery.MaxRetries) + flags.SetDefault("pipelines.error-recovery.max-retries-window", c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow) + flags.SetDefault("schema-registry.type", c.cfg.SchemaRegistry.Type) + flags.SetDefault("schema-registry.confluent.connection-string", c.cfg.SchemaRegistry.Confluent.ConnectionString) + flags.SetDefault("preview.pipeline-arch-v2", c.cfg.Preview.PipelineArchV2) + + return flags +} + +func (c *RootCommand) Docs() ecdysis.Docs { + return ecdysis.Docs{ + Short: "Conduit CLI", + Long: `Conduit CLI is a command-line that helps you interact with and manage Conduit.`, } +} - cmd.CompletionOptions.DisableDefaultCmd = true - conduit.Flags(&cfg).VisitAll(cmd.Flags().AddGoFlag) - - // init - initCmd := buildInitCmd() - cmd.AddCommand(initCmd) - - // pipelines - cmd.AddGroup(&cobra.Group{ - ID: "pipelines", - Title: "Pipelines", - }) - cmd.AddCommand(pipelines.BuildPipelinesCmd()) - - // mark hidden flags - cmd.Flags().VisitAll(func(f *pflag.Flag) { - if conduit.HiddenFlags[f.Name] { - err := cmd.Flags().MarkHidden(f.Name) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to mark flag %q as hidden: %v", f.Name, err) - } - } - }) - - return cmd +func (c *RootCommand) SubCommands() []ecdysis.Command { + return []ecdysis.Command{ + &InitCommand{}, + &pipelines.PipelinesCommand{}, + } } diff --git a/cmd/conduit/root/root_ecdysis.go b/cmd/conduit/root/root_ecdysis.go deleted file mode 100644 index 9a6af96af..000000000 --- a/cmd/conduit/root/root_ecdysis.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package root - -import ( - "context" - "fmt" - "time" - - "github.com/conduitio/conduit/pkg/conduit" - "github.com/conduitio/ecdysis" -) - -var ( - _ ecdysis.CommandWithFlags = (*RootCommand)(nil) - _ ecdysis.CommandWithExecute = (*RootCommand)(nil) - _ ecdysis.CommandWithDocs = (*RootCommand)(nil) - _ ecdysis.CommandWithSubCommands = (*RootCommand)(nil) -) - -// TODO: Check which ones are really global from the design document -type RootFlags struct { - // Database configuration - DBType string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite" persistent:"true"` - DBBadgerPath string `long:"db.badger.path" usage:"path to badger DB" persistent:"true"` - DBPostgresConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format" persistent:"true"` - DBPostgresTable string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)" persistent:"true"` - DBSQLitePath string `long:"db.sqlite.path" usage:"path to sqlite3 DB" persistent:"true"` - DBSQLiteTable string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)" persistent:"true"` - - // API configuration - APIEnabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API" persistent:"true"` - APIHTTPAddress string `long:"http.address" usage:"address for serving the HTTP API" persistent:"true"` - APIGRPCAddress string `long:"grpc.address" usage:"address for serving the gRPC API" persistent:"true"` - - // Logging configuration - LogLevel string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace" persistent:"true"` - LogFormat string `long:"log.format" usage:"sets the format of the logging; accepts json, cli" persistent:"true"` - - // Connectors and Processors paths - ConnectorsPath string `long:"connectors.path" usage:"path to standalone connectors' directory" persistent:"true"` - ProcessorsPath string `long:"processors.path" usage:"path to standalone processors' directory" persistent:"true"` - - // Pipeline configuration - PipelinesPath string `long:"pipelines.path" usage:"path to the directory that has the yaml pipeline configuration files, or a single pipeline configuration file" persistent:"true"` - PipelinesExitOnDegraded bool `long:"pipelines.exit-on-degraded" usage:"exit Conduit if a pipeline enters a degraded state" persistent:"true"` - PipelinesErrorRecoveryMinDelay time.Duration `long:"pipelines.error-recovery.min-delay" usage:"minimum delay before restart" persistent:"true"` - PipelinesErrorRecoveryMaxDelay time.Duration `long:"pipelines.error-recovery.max-delay" usage:"maximum delay before restart" persistent:"true"` - PipelinesErrorRecoveryBackoffFactor int `long:"pipelines.error-recovery.backoff-factor" usage:"backoff factor applied to the last delay" persistent:"true"` - PipelinesErrorRecoveryMaxRetries int64 `long:"pipelines.error-recovery.max-retries" usage:"maximum number of retries" persistent:"true"` - PipelinesErrorRecoveryMaxRetriesWindow time.Duration `long:"pipelines.error-recovery.max-retries-window" usage:"amount of time running without any errors after which a pipeline is considered healthy" persistent:"true"` - - // Schema registry configuration - SchemaRegistryType string `long:"schema-registry.type" usage:"schema registry type; accepts builtin,confluent" persistent:"true"` - SchemaRegistryConfluentConnectionString string `long:"schema-registry.confluent.connection-string" usage:"confluent schema registry connection string" persistent:"true"` - - // Preview features - PreviewPipelineArchV2 bool `long:"preview.pipeline-arch-v2" usage:"enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)" persistent:"true"` - - // Development profiling - DevCPUProfile string `long:"dev.cpuprofile" usage:"write CPU profile to file" persistent:"true"` - DevMemProfile string `long:"dev.memprofile" usage:"write memory profile to file" persistent:"true"` - DevBlockProfile string `long:"dev.blockprofile" usage:"write block profile to file" persistent:"true"` - - // Version - Version bool `long:"version" short:"v" usage:"show version" persistent:"true"` -} - -type RootCommand struct { - flags RootFlags - cfg conduit.Config -} - -func (c *RootCommand) Execute(ctx context.Context) error { - if c.flags.Version { - // TODO: use the logger instead - fmt.Print(conduit.Version(true)) - return nil - } - - e := &conduit.Entrypoint{} - e.Serve(c.cfg) - - return nil -} - -func (c *RootCommand) Usage() string { return "conduit" } -func (c *RootCommand) Flags() []ecdysis.Flag { - flags := ecdysis.BuildFlags(&c.flags) - - c.cfg = conduit.DefaultConfig() - - flags.SetDefault("db.type", c.cfg.DB.Type) - flags.SetDefault("db.badger.path", c.cfg.DB.Badger.Path) - flags.SetDefault("db.postgres.connection-string", c.cfg.DB.Postgres.ConnectionString) - flags.SetDefault("db.postgres.table", c.cfg.DB.Postgres.Table) - flags.SetDefault("db.sqlite.path", c.cfg.DB.SQLite.Path) - flags.SetDefault("db.sqlite.table", c.cfg.DB.SQLite.Table) - flags.SetDefault("api.enabled", c.cfg.API.Enabled) - flags.SetDefault("http.address", c.cfg.API.HTTP.Address) - flags.SetDefault("grpc.address", c.cfg.API.GRPC.Address) - flags.SetDefault("log.level", c.cfg.Log.Level) - flags.SetDefault("log.format", c.cfg.Log.Format) - flags.SetDefault("connectors.path", c.cfg.Connectors.Path) - flags.SetDefault("processors.path", c.cfg.Processors.Path) - flags.SetDefault("pipelines.path", c.cfg.Pipelines.Path) - flags.SetDefault("pipelines.exit-on-degraded", c.cfg.Pipelines.ExitOnDegraded) - flags.SetDefault("pipelines.error-recovery.min-delay", c.cfg.Pipelines.ErrorRecovery.MinDelay) - flags.SetDefault("pipelines.error-recovery.max-delay", c.cfg.Pipelines.ErrorRecovery.MaxDelay) - flags.SetDefault("pipelines.error-recovery.backoff-factor", c.cfg.Pipelines.ErrorRecovery.BackoffFactor) - flags.SetDefault("pipelines.error-recovery.max-retries", c.cfg.Pipelines.ErrorRecovery.MaxRetries) - flags.SetDefault("pipelines.error-recovery.max-retries-window", c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow) - flags.SetDefault("schema-registry.type", c.cfg.SchemaRegistry.Type) - flags.SetDefault("schema-registry.confluent.connection-string", c.cfg.SchemaRegistry.Confluent.ConnectionString) - flags.SetDefault("preview.pipeline-arch-v2", c.cfg.Preview.PipelineArchV2) - - return flags -} - -func (c *RootCommand) Docs() ecdysis.Docs { - return ecdysis.Docs{ - Short: "Conduit CLI", - Long: `Conduit CLI is a command-line that helps you interact with and manage Conduit.`, - } -} - -func (c *RootCommand) SubCommands() []ecdysis.Command { - return []ecdysis.Command{ - &InitCommand{}, - } -} diff --git a/cmd/conduit/root/root_test.go b/cmd/conduit/root/root_test.go deleted file mode 100644 index b399858b4..000000000 --- a/cmd/conduit/root/root_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package root - -import ( - "bytes" - "strings" - "testing" - - "github.com/conduitio/conduit/pkg/conduit" -) - -func TestBuildRootCmd_HelpOutput(t *testing.T) { - cmd := buildRootCmd() - - var buf bytes.Buffer - cmd.SetOut(&buf) - cmd.SetArgs([]string{"--help"}) - - err := cmd.Execute() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - output := buf.String() - - expectedFlags := []string{ - "db.type", - "db.badger.path", - "db.postgres.connection-string", - "db.postgres.table", - "db.sqlite.path", - "db.sqlite.table", - "dev.blockprofileblockprofile", - "dev.cpuprofile", - "dev.memprofile", - "api.enabled", - "http.address", - "grpc.address", - "log.level", - "log.format", - "connectors.path", - "processors.path", - "pipelines.path", - "pipelines.exit-on-degraded", - "pipelines.error-recovery.min-delay", - "pipelines.error-recovery.max-delay", - "pipelines.error-recovery.backoff-factor", - "pipelines.error-recovery.max-retries", - "pipelines.error-recovery.max-retries-window", - "schema-registry.type", - "schema-registry.confluent.connection-string", - "preview.pipeline-arch-v2", - } - - unexpectedFlags := []string{ - conduit.FlagPipelinesExitOnError, //nolint:staticcheck // this will be completely removed before Conduit 1.0 - } - - for _, flag := range expectedFlags { - if !strings.Contains(output, flag) { - t.Errorf("expected flag %q not found in help output", flag) - } - } - - for _, flag := range unexpectedFlags { - if strings.Contains(output, flag) { - t.Errorf("unexpected flag %q found in help output", flag) - } - } -} From 790e0052ac3c880861de116a01857a5ac8c16a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Thu, 21 Nov 2024 16:13:54 +0100 Subject: [PATCH 06/22] update which ones are general flags --- cmd/conduit/root/root.go | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index 9d26cff9f..c9c0bea9f 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -34,46 +34,46 @@ var ( // TODO: Check which ones are really global from the design document type RootFlags struct { // Database configuration - DBType string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite" persistent:"true"` - DBBadgerPath string `long:"db.badger.path" usage:"path to badger DB" persistent:"true"` - DBPostgresConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format" persistent:"true"` - DBPostgresTable string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)" persistent:"true"` - DBSQLitePath string `long:"db.sqlite.path" usage:"path to sqlite3 DB" persistent:"true"` - DBSQLiteTable string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)" persistent:"true"` + DBType string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite"` + DBBadgerPath string `long:"db.badger.path" usage:"path to badger DB"` + DBPostgresConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format"` + DBPostgresTable string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)"` + DBSQLitePath string `long:"db.sqlite.path" usage:"path to sqlite3 DB"` + DBSQLiteTable string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)"` // API configuration - APIEnabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API" persistent:"true"` - APIHTTPAddress string `long:"http.address" usage:"address for serving the HTTP API" persistent:"true"` - APIGRPCAddress string `long:"grpc.address" usage:"address for serving the gRPC API" persistent:"true"` + APIEnabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API"` + APIHTTPAddress string `long:"http.address" usage:"address for serving the HTTP API"` + APIGRPCAddress string `long:"grpc.address" usage:"address for serving the gRPC API"` // Logging configuration - LogLevel string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace" persistent:"true"` - LogFormat string `long:"log.format" usage:"sets the format of the logging; accepts json, cli" persistent:"true"` + LogLevel string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace"` + LogFormat string `long:"log.format" usage:"sets the format of the logging; accepts json, cli"` // Connectors and Processors paths - ConnectorsPath string `long:"connectors.path" usage:"path to standalone connectors' directory" persistent:"true"` - ProcessorsPath string `long:"processors.path" usage:"path to standalone processors' directory" persistent:"true"` + ConnectorsPath string `long:"connectors.path" usage:"path to standalone connectors' directory"` + ProcessorsPath string `long:"processors.path" usage:"path to standalone processors' directory"` // Pipeline configuration - PipelinesPath string `long:"pipelines.path" usage:"path to the directory that has the yaml pipeline configuration files, or a single pipeline configuration file" persistent:"true"` - PipelinesExitOnDegraded bool `long:"pipelines.exit-on-degraded" usage:"exit Conduit if a pipeline enters a degraded state" persistent:"true"` - PipelinesErrorRecoveryMinDelay time.Duration `long:"pipelines.error-recovery.min-delay" usage:"minimum delay before restart" persistent:"true"` - PipelinesErrorRecoveryMaxDelay time.Duration `long:"pipelines.error-recovery.max-delay" usage:"maximum delay before restart" persistent:"true"` - PipelinesErrorRecoveryBackoffFactor int `long:"pipelines.error-recovery.backoff-factor" usage:"backoff factor applied to the last delay" persistent:"true"` - PipelinesErrorRecoveryMaxRetries int64 `long:"pipelines.error-recovery.max-retries" usage:"maximum number of retries" persistent:"true"` - PipelinesErrorRecoveryMaxRetriesWindow time.Duration `long:"pipelines.error-recovery.max-retries-window" usage:"amount of time running without any errors after which a pipeline is considered healthy" persistent:"true"` + PipelinesPath string `long:"pipelines.path" usage:"path to the directory that has the yaml pipeline configuration files, or a single pipeline configuration file"` + PipelinesExitOnDegraded bool `long:"pipelines.exit-on-degraded" usage:"exit Conduit if a pipeline enters a degraded state"` + PipelinesErrorRecoveryMinDelay time.Duration `long:"pipelines.error-recovery.min-delay" usage:"minimum delay before restart"` + PipelinesErrorRecoveryMaxDelay time.Duration `long:"pipelines.error-recovery.max-delay" usage:"maximum delay before restart"` + PipelinesErrorRecoveryBackoffFactor int `long:"pipelines.error-recovery.backoff-factor" usage:"backoff factor applied to the last delay"` + PipelinesErrorRecoveryMaxRetries int64 `long:"pipelines.error-recovery.max-retries" usage:"maximum number of retries"` + PipelinesErrorRecoveryMaxRetriesWindow time.Duration `long:"pipelines.error-recovery.max-retries-window" usage:"amount of time running without any errors after which a pipeline is considered healthy"` // Schema registry configuration - SchemaRegistryType string `long:"schema-registry.type" usage:"schema registry type; accepts builtin,confluent" persistent:"true"` - SchemaRegistryConfluentConnectionString string `long:"schema-registry.confluent.connection-string" usage:"confluent schema registry connection string" persistent:"true"` + SchemaRegistryType string `long:"schema-registry.type" usage:"schema registry type; accepts builtin,confluent"` + SchemaRegistryConfluentConnectionString string `long:"schema-registry.confluent.connection-string" usage:"confluent schema registry connection string"` // Preview features - PreviewPipelineArchV2 bool `long:"preview.pipeline-arch-v2" usage:"enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)" persistent:"true"` + PreviewPipelineArchV2 bool `long:"preview.pipeline-arch-v2" usage:"enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)"` // Development profiling - DevCPUProfile string `long:"dev.cpuprofile" usage:"write CPU profile to file" persistent:"true"` - DevMemProfile string `long:"dev.memprofile" usage:"write memory profile to file" persistent:"true"` - DevBlockProfile string `long:"dev.blockprofile" usage:"write block profile to file" persistent:"true"` + DevCPUProfile string `long:"dev.cpuprofile" usage:"write CPU profile to file"` + DevMemProfile string `long:"dev.memprofile" usage:"write memory profile to file"` + DevBlockProfile string `long:"dev.blockprofile" usage:"write block profile to file"` // Version Version bool `long:"version" short:"v" usage:"show version" persistent:"true"` From 077ea7563af85bece130b24ad484c123527ffaf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Thu, 21 Nov 2024 16:26:54 +0100 Subject: [PATCH 07/22] fix lint --- cmd/conduit/root/init.go | 2 -- cmd/conduit/root/pipelines/format_parameter.go | 14 ++++++++++++++ cmd/conduit/root/pipelines/pipelines.go | 7 +++---- cmd/conduit/root/pipelines/template.go | 14 ++++++++++++++ go.mod | 4 ++-- 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cmd/conduit/root/init.go b/cmd/conduit/root/init.go index 75d418027..2c717764b 100644 --- a/cmd/conduit/root/init.go +++ b/cmd/conduit/root/init.go @@ -115,7 +115,6 @@ To quickly create an example pipeline, run 'conduit pipelines init'. To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) return nil - } func (c *InitCommand) Flags() []ecdysis.Flag { @@ -130,5 +129,4 @@ func (c *InitCommand) Flags() []ecdysis.Flag { flags.SetDefault("config.path", currentPath) return flags - } diff --git a/cmd/conduit/root/pipelines/format_parameter.go b/cmd/conduit/root/pipelines/format_parameter.go index 4506db947..38c5e7d33 100644 --- a/cmd/conduit/root/pipelines/format_parameter.go +++ b/cmd/conduit/root/pipelines/format_parameter.go @@ -1,3 +1,17 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pipelines import ( diff --git a/cmd/conduit/root/pipelines/pipelines.go b/cmd/conduit/root/pipelines/pipelines.go index a733258fb..62dbcc91d 100644 --- a/cmd/conduit/root/pipelines/pipelines.go +++ b/cmd/conduit/root/pipelines/pipelines.go @@ -23,8 +23,7 @@ var ( _ ecdysis.CommandWithSubCommands = (*PipelinesCommand)(nil) ) -type PipelinesCommand struct { -} +type PipelinesCommand struct{} func (c *PipelinesCommand) SubCommands() []ecdysis.Command { return []ecdysis.Command{ @@ -32,9 +31,9 @@ func (c *PipelinesCommand) SubCommands() []ecdysis.Command { } } -func (p *PipelinesCommand) Usage() string { return "pipelines" } +func (c *PipelinesCommand) Usage() string { return "pipelines" } -func (p *PipelinesCommand) Docs() ecdysis.Docs { +func (c *PipelinesCommand) Docs() ecdysis.Docs { return ecdysis.Docs{ Short: "Initialize and manage pipelines", } diff --git a/cmd/conduit/root/pipelines/template.go b/cmd/conduit/root/pipelines/template.go index b030a2d1c..fa3e0f005 100644 --- a/cmd/conduit/root/pipelines/template.go +++ b/cmd/conduit/root/pipelines/template.go @@ -1,3 +1,17 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pipelines import "github.com/conduitio/conduit-commons/config" diff --git a/go.mod b/go.mod index 13f55cf47..88856d710 100644 --- a/go.mod +++ b/go.mod @@ -42,8 +42,6 @@ require ( github.com/prometheus/common v0.60.1 github.com/rs/zerolog v1.33.0 github.com/sourcegraph/conc v0.3.0 - github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 github.com/stealthrocket/wazergo v0.19.1 github.com/tetratelabs/wazero v1.8.1 github.com/twmb/franz-go/pkg/sr v1.2.0 @@ -321,6 +319,8 @@ require ( github.com/sourcegraph/go-diff v0.7.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.19.0 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect From 4fc6f5bf00133c29fc376b0e075ba1abea13a20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Thu, 21 Nov 2024 16:35:56 +0100 Subject: [PATCH 08/22] remove TODO --- cmd/conduit/root/root.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index c9c0bea9f..5a0ccb110 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -31,7 +31,6 @@ var ( _ ecdysis.CommandWithSubCommands = (*RootCommand)(nil) ) -// TODO: Check which ones are really global from the design document type RootFlags struct { // Database configuration DBType string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite"` From c00924bf7028839897d9023da69367cc16db2d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Thu, 21 Nov 2024 16:41:32 +0100 Subject: [PATCH 09/22] hide completion for now --- cmd/conduit/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/conduit/main.go b/cmd/conduit/main.go index e20de4c78..a48bbd1e7 100644 --- a/cmd/conduit/main.go +++ b/cmd/conduit/main.go @@ -25,6 +25,9 @@ import ( func main() { e := ecdysis.New() cmd := e.MustBuildCobraCommand(&root.RootCommand{}) + + cmd.CompletionOptions.DisableDefaultCmd = true + if err := cmd.Execute(); err != nil { _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) From 00f153ba898e88fbb2adc930f5c2f23f32c8ac9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 25 Nov 2024 18:15:40 +0100 Subject: [PATCH 10/22] feature complete --- cmd/conduit/internal/config.go | 71 +++++++++++++++ cmd/conduit/main.go | 3 +- cmd/conduit/root/init.go | 59 +++++-------- cmd/conduit/root/root.go | 134 ++++++++++++++++++++++++++-- pkg/conduit/config.go | 8 +- pkg/conduit/entrypoint.go | 154 --------------------------------- pkg/conduit/entrypoint_test.go | 76 ---------------- pkg/conduit/runtime.go | 12 +-- 8 files changed, 230 insertions(+), 287 deletions(-) create mode 100644 cmd/conduit/internal/config.go delete mode 100644 pkg/conduit/entrypoint_test.go diff --git a/cmd/conduit/internal/config.go b/cmd/conduit/internal/config.go new file mode 100644 index 000000000..4266451f0 --- /dev/null +++ b/cmd/conduit/internal/config.go @@ -0,0 +1,71 @@ +package internal + +import ( + "fmt" + "os" + "strings" + + "github.com/conduitio/conduit/pkg/conduit" + "github.com/spf13/viper" +) + +const ( + CONDUIT_PREFIX = "CONDUIT" +) + +// LoadConfigFromFile loads on cfg, the configuration from the file at path. +func LoadConfigFromFile(filePath string, cfg *conduit.Config) error { + v := viper.New() + + // Set the file name and path + v.SetConfigFile(filePath) + + // Attempt to read the configuration file + if err := v.ReadInConfig(); err != nil { + // here we could simply log conduit.yaml file doesn't exist since this is optional + //return fmt.Errorf("error reading config file: %w", err) + return nil + } + + // Unmarshal the config into the cfg struct + if err := v.Unmarshal(&cfg); err != nil { + return fmt.Errorf("unable to decode into struct: %w", err) + } + + return nil +} + +// TODO: check if logger is correct +func LoadConfigFromEnv(cfg *conduit.Config) error { + v := viper.New() + + // Set environment variable prefix + v.SetEnvPrefix(CONDUIT_PREFIX) + + // Automatically map environment variables + v.AutomaticEnv() + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + + for _, env := range os.Environ() { + pair := strings.SplitN(env, "=", 2) + key := pair[0] + value := pair[1] + + // Check if the environment variable has the desired prefix + if strings.HasPrefix(key, fmt.Sprintf("%s_", CONDUIT_PREFIX)) { + // Strip the prefix and replace underscores with dots + strippedKey := strings.ToLower(strings.TrimPrefix(key, fmt.Sprintf("%s_", CONDUIT_PREFIX))) + strippedKey = strings.ReplaceAll(strippedKey, "_", ".") + + // Set the value in Viper + v.Set(strippedKey, value) + } + } + + // Unmarshal the environment variables into the config struct + err := v.Unmarshal(cfg) + if err != nil { + return fmt.Errorf("error unmarshalling config from environment variables: %w", err) + } + return nil +} diff --git a/cmd/conduit/main.go b/cmd/conduit/main.go index a48bbd1e7..0a28295a6 100644 --- a/cmd/conduit/main.go +++ b/cmd/conduit/main.go @@ -24,12 +24,13 @@ import ( func main() { e := ecdysis.New() - cmd := e.MustBuildCobraCommand(&root.RootCommand{}) + cmd := e.MustBuildCobraCommand(&root.RootCommand{}) cmd.CompletionOptions.DisableDefaultCmd = true if err := cmd.Execute(); err != nil { _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } + os.Exit(0) } diff --git a/cmd/conduit/root/init.go b/cmd/conduit/root/init.go index 2c717764b..c2d6b506e 100644 --- a/cmd/conduit/root/init.go +++ b/cmd/conduit/root/init.go @@ -16,30 +16,24 @@ package root import ( "context" - "flag" "fmt" "os" "path/filepath" + "reflect" "github.com/conduitio/conduit/cmd/conduit/internal" - "github.com/conduitio/conduit/pkg/conduit" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/ecdysis" "github.com/conduitio/yaml/v3" ) var ( - _ ecdysis.CommandWithFlags = (*InitCommand)(nil) _ ecdysis.CommandWithExecute = (*InitCommand)(nil) _ ecdysis.CommandWithDocs = (*InitCommand)(nil) ) -type InitFlags struct { - ConfigPath string `flag:"config.path" usage:"path where Conduit will be initialized"` -} - type InitCommand struct { - flags InitFlags + rootFlags *RootFlags } func (c *InitCommand) Usage() string { return "init" } @@ -51,10 +45,12 @@ func (c *InitCommand) Docs() ecdysis.Docs { } func (c *InitCommand) createDirs() error { + // These could be used based on the root flags if those were global dirs := []string{"processors", "connectors", "pipelines"} + conduitPath := filepath.Dir(c.rootFlags.ConduitConfigPath) for _, dir := range dirs { - path := filepath.Join(c.flags.ConfigPath, dir) + path := filepath.Join(conduitPath, dir) // Attempt to create the directory, skipping if it already exists if err := os.Mkdir(path, os.ModePerm); err != nil { @@ -71,28 +67,29 @@ func (c *InitCommand) createDirs() error { return nil } -func (c *InitCommand) conduitCfgFlags() *flag.FlagSet { - cfg := conduit.DefaultConfigWithBasePath(c.flags.ConfigPath) - return conduit.Flags(&cfg) -} - func (c *InitCommand) createConfigYAML() error { cfgYAML := internal.NewYAMLTree() - c.conduitCfgFlags().VisitAll(func(f *flag.Flag) { - cfgYAML.Insert(f.Name, f.DefValue, f.Usage) - }) + v := reflect.Indirect(reflect.ValueOf(c.rootFlags)) + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + field := t.Field(i) + value := fmt.Sprintf("%v", v.Field(i).Interface()) + usage := field.Tag.Get("usage") + longName := field.Tag.Get("long") + cfgYAML.Insert(longName, value, usage) + } yamlData, err := yaml.Marshal(cfgYAML.Root) if err != nil { return cerrors.Errorf("error marshaling YAML: %w\n", err) } - path := filepath.Join(c.flags.ConfigPath, "conduit.yaml") - err = os.WriteFile(path, yamlData, 0o600) + err = os.WriteFile(c.rootFlags.ConduitConfigPath, yamlData, 0o600) if err != nil { return cerrors.Errorf("error writing conduit.yaml: %w", err) } - fmt.Printf("Configuration file written to %v\n", path) + fmt.Printf("Configuration file written to %v\n", c.rootFlags.ConduitConfigPath) return nil } @@ -109,24 +106,10 @@ func (c *InitCommand) Execute(ctx context.Context) error { } fmt.Println(` -Conduit has been initialized! - -To quickly create an example pipeline, run 'conduit pipelines init'. -To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) + Conduit has been initialized! + + To quickly create an example pipeline, run 'conduit pipelines init'. + To see how you can customize your first pipeline, run 'conduit pipelines init --help'.`) return nil } - -func (c *InitCommand) Flags() []ecdysis.Flag { - flags := ecdysis.BuildFlags(&c.flags) - - // Set current working directory as default - currentPath, err := os.Getwd() - if err != nil { - panic(cerrors.Errorf("failed to get current working directory: %w", err)) - } - - flags.SetDefault("config.path", currentPath) - - return flags -} diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index 5a0ccb110..40e6e92d6 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -17,10 +17,14 @@ package root import ( "context" "fmt" + "os" + "path/filepath" "time" + "github.com/conduitio/conduit/cmd/conduit/internal" "github.com/conduitio/conduit/cmd/conduit/root/pipelines" "github.com/conduitio/conduit/pkg/conduit" + "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/ecdysis" ) @@ -32,6 +36,16 @@ var ( ) type RootFlags struct { + // Global flags ----------------------------------------------------------- + + // Conduit configuration file + ConduitConfigPath string `long:"config" usage:"global conduit configuration file" persistent:"true" default:"./conduit.yaml"` + + // Version + Version bool `long:"version" short:"v" usage:"show current Conduit version" persistent:"true"` + + // Root flags ------------------------------------------------------------- + // Database configuration DBType string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite"` DBBadgerPath string `long:"db.badger.path" usage:"path to badger DB"` @@ -73,9 +87,6 @@ type RootFlags struct { DevCPUProfile string `long:"dev.cpuprofile" usage:"write CPU profile to file"` DevMemProfile string `long:"dev.memprofile" usage:"write memory profile to file"` DevBlockProfile string `long:"dev.blockprofile" usage:"write block profile to file"` - - // Version - Version bool `long:"version" short:"v" usage:"show version" persistent:"true"` } type RootCommand struct { @@ -83,16 +94,117 @@ type RootCommand struct { cfg conduit.Config } +func (c *RootCommand) updateConfigFromFlags() { + c.cfg.DB.Type = c.flags.DBType + c.cfg.DB.Postgres.ConnectionString = c.flags.DBPostgresConnectionString + c.cfg.DB.Postgres.Table = c.flags.DBPostgresTable + c.cfg.DB.SQLite.Table = c.flags.DBSQLiteTable + + // Map API configuration + c.cfg.API.Enabled = c.flags.APIEnabled + c.cfg.API.HTTP.Address = c.flags.APIHTTPAddress + c.cfg.API.GRPC.Address = c.flags.APIGRPCAddress + + // Map logging configuration + c.cfg.Log.Level = c.flags.LogLevel + c.cfg.Log.Format = c.flags.LogFormat + + // Map pipeline configuration + c.cfg.Pipelines.ExitOnDegraded = c.flags.PipelinesExitOnDegraded + c.cfg.Pipelines.ErrorRecovery.MinDelay = c.flags.PipelinesErrorRecoveryMinDelay + c.cfg.Pipelines.ErrorRecovery.MaxDelay = c.flags.PipelinesErrorRecoveryMaxDelay + c.cfg.Pipelines.ErrorRecovery.BackoffFactor = c.flags.PipelinesErrorRecoveryBackoffFactor + c.cfg.Pipelines.ErrorRecovery.MaxRetries = c.flags.PipelinesErrorRecoveryMaxRetries + c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow = c.flags.PipelinesErrorRecoveryMaxRetriesWindow + + // Map schema registry configuration + c.cfg.SchemaRegistry.Type = c.flags.SchemaRegistryType + c.cfg.SchemaRegistry.Confluent.ConnectionString = c.flags.SchemaRegistryConfluentConnectionString + + // Map preview features + c.cfg.Preview.PipelineArchV2 = c.flags.PreviewPipelineArchV2 + + // Map development profiling + c.cfg.Dev.CPUProfile = c.flags.DevCPUProfile + c.cfg.Dev.MemProfile = c.flags.DevMemProfile + c.cfg.Dev.BlockProfile = c.flags.DevBlockProfile + + // Update paths + c.cfg.DB.SQLite.Path = c.flags.DBSQLitePath + c.cfg.DB.Badger.Path = c.flags.DBBadgerPath + c.cfg.Pipelines.Path = c.flags.PipelinesPath + c.cfg.Connectors.Path = c.flags.ConnectorsPath + c.cfg.Processors.Path = c.flags.ProcessorsPath +} + +func (c *RootCommand) updateFlagValuesFromConfig() { + // Map database configuration + c.flags.DBType = c.cfg.DB.Type + c.flags.DBPostgresConnectionString = c.cfg.DB.Postgres.ConnectionString + c.flags.DBPostgresTable = c.cfg.DB.Postgres.Table + c.flags.DBSQLiteTable = c.cfg.DB.SQLite.Table + + // Map API configuration + c.flags.APIEnabled = c.cfg.API.Enabled + c.flags.APIHTTPAddress = c.cfg.API.HTTP.Address + c.flags.APIGRPCAddress = c.cfg.API.GRPC.Address + + // Map logging configuration + c.flags.LogLevel = c.cfg.Log.Level + c.flags.LogFormat = c.cfg.Log.Format + + // Map pipeline configuration + c.flags.PipelinesExitOnDegraded = c.cfg.Pipelines.ExitOnDegraded + c.flags.PipelinesErrorRecoveryMinDelay = c.cfg.Pipelines.ErrorRecovery.MinDelay + c.flags.PipelinesErrorRecoveryMaxDelay = c.cfg.Pipelines.ErrorRecovery.MaxDelay + c.flags.PipelinesErrorRecoveryBackoffFactor = c.cfg.Pipelines.ErrorRecovery.BackoffFactor + c.flags.PipelinesErrorRecoveryMaxRetries = c.cfg.Pipelines.ErrorRecovery.MaxRetries + c.flags.PipelinesErrorRecoveryMaxRetriesWindow = c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow + + // Map schema registry configuration + c.flags.SchemaRegistryType = c.cfg.SchemaRegistry.Type + c.flags.SchemaRegistryConfluentConnectionString = c.cfg.SchemaRegistry.Confluent.ConnectionString + + // Map preview features + c.flags.PreviewPipelineArchV2 = c.cfg.Preview.PipelineArchV2 + + // Map development profiling + c.flags.DevCPUProfile = c.cfg.Dev.CPUProfile + c.flags.DevMemProfile = c.cfg.Dev.MemProfile + c.flags.DevBlockProfile = c.cfg.Dev.BlockProfile + + // Update paths + c.flags.DBSQLitePath = c.cfg.DB.SQLite.Path + c.flags.DBBadgerPath = c.cfg.DB.Badger.Path + c.flags.PipelinesPath = c.cfg.Pipelines.Path + c.flags.ConnectorsPath = c.cfg.Connectors.Path + c.flags.ProcessorsPath = c.cfg.Processors.Path +} + func (c *RootCommand) Execute(ctx context.Context) error { if c.flags.Version { - // TODO: use the logger instead - fmt.Print(conduit.Version(true)) + _, _ = fmt.Fprintf(os.Stdout, "%s\n", conduit.Version(true)) return nil } + // 1. Load conduit configuration file and update general config. + if err := internal.LoadConfigFromFile(c.flags.ConduitConfigPath, &c.cfg); err != nil { + return err + } + + // 2. Load environment variables and update general config. + if err := internal.LoadConfigFromEnv(&c.cfg); err != nil { + return err + } + + // 3. Update the general config from flags. + c.updateConfigFromFlags() + + // 4. Update flags from global configuration (this will be needed for conduit init) + c.updateFlagValuesFromConfig() + e := &conduit.Entrypoint{} e.Serve(c.cfg) - return nil } @@ -100,8 +212,14 @@ func (c *RootCommand) Usage() string { return "conduit" } func (c *RootCommand) Flags() []ecdysis.Flag { flags := ecdysis.BuildFlags(&c.flags) - c.cfg = conduit.DefaultConfig() + currentPath, err := os.Getwd() + if err != nil { + panic(cerrors.Errorf("failed to get current working directory: %w", err)) + } + c.cfg = conduit.DefaultConfigWithBasePath(currentPath) + conduitConfigPath := filepath.Join(currentPath, "conduit.yaml") + flags.SetDefault("config.path", conduitConfigPath) flags.SetDefault("db.type", c.cfg.DB.Type) flags.SetDefault("db.badger.path", c.cfg.DB.Badger.Path) flags.SetDefault("db.postgres.connection-string", c.cfg.DB.Postgres.ConnectionString) @@ -138,7 +256,7 @@ func (c *RootCommand) Docs() ecdysis.Docs { func (c *RootCommand) SubCommands() []ecdysis.Command { return []ecdysis.Command{ - &InitCommand{}, + &InitCommand{rootFlags: &c.flags}, &pipelines.PipelinesCommand{}, } } diff --git a/pkg/conduit/config.go b/pkg/conduit/config.go index aba89f144..1ddeb7aa7 100644 --- a/pkg/conduit/config.go +++ b/pkg/conduit/config.go @@ -117,10 +117,10 @@ type Config struct { PipelineArchV2 bool } - dev struct { - cpuprofile string - memprofile string - blockprofile string + Dev struct { + CPUProfile string + MemProfile string + BlockProfile string } } diff --git a/pkg/conduit/entrypoint.go b/pkg/conduit/entrypoint.go index 259dd3fdf..e9cce35c1 100644 --- a/pkg/conduit/entrypoint.go +++ b/pkg/conduit/entrypoint.go @@ -16,35 +16,18 @@ package conduit import ( "context" - "flag" "fmt" "os" "os/signal" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/peterbourgon/ff/v3" - "github.com/peterbourgon/ff/v3/ffyaml" ) const ( exitCodeErr = 1 exitCodeInterrupt = 2 - - // Deprecated: Use `pipelines.error-recovery.exit-on-degraded` instead. - FlagPipelinesExitOnError = "pipelines.exit-on-error" ) -// HiddenFlags is a map of flags that should not be shown in the help output. -var HiddenFlags = map[string]bool{ - FlagPipelinesExitOnError: true, -} - -// Serve is a shortcut for Entrypoint.Serve. -func Serve(cfg Config) { - e := &Entrypoint{} - e.Serve(cfg) -} - // Entrypoint provides methods related to the Conduit entrypoint (parsing // config, managing interrupt signals etc.). type Entrypoint struct{} @@ -59,9 +42,6 @@ type Entrypoint struct{} // - environment variables // - config file (lowest priority) func (e *Entrypoint) Serve(cfg Config) { - flags := Flags(&cfg) - e.ParseConfig(flags) - if cfg.Log.Format == "cli" { _, _ = fmt.Fprintf(os.Stdout, "%s\n", e.Splash()) } @@ -79,140 +59,6 @@ func (e *Entrypoint) Serve(cfg Config) { } } -// Flags returns a flag set that, when parsed, stores the values in the provided -// config struct. -func Flags(cfg *Config) *flag.FlagSet { - // TODO extract flags from config struct rather than defining flags manually - flags := flag.NewFlagSet("conduit", flag.ExitOnError) - - flags.StringVar(&cfg.DB.Type, "db.type", cfg.DB.Type, "database type; accepts badger,postgres,inmemory,sqlite") - flags.StringVar(&cfg.DB.Badger.Path, "db.badger.path", cfg.DB.Badger.Path, "path to badger DB") - flags.StringVar( - &cfg.DB.Postgres.ConnectionString, - "db.postgres.connection-string", - cfg.DB.Postgres.ConnectionString, - "postgres connection string, may be a database URL or in PostgreSQL keyword/value format", - ) - flags.StringVar(&cfg.DB.Postgres.Table, "db.postgres.table", cfg.DB.Postgres.Table, "postgres table in which to store data (will be created if it does not exist)") - flags.StringVar(&cfg.DB.SQLite.Path, "db.sqlite.path", cfg.DB.SQLite.Path, "path to sqlite3 DB") - flags.StringVar(&cfg.DB.SQLite.Table, "db.sqlite.table", cfg.DB.SQLite.Table, "sqlite3 table in which to store data (will be created if it does not exist)") - flags.BoolVar(&cfg.API.Enabled, "api.enabled", cfg.API.Enabled, "enable HTTP and gRPC API") - flags.StringVar(&cfg.API.HTTP.Address, "http.address", cfg.API.HTTP.Address, "address for serving the HTTP API") - flags.StringVar(&cfg.API.GRPC.Address, "grpc.address", cfg.API.GRPC.Address, "address for serving the gRPC API") - - flags.StringVar(&cfg.Log.Level, "log.level", cfg.Log.Level, "sets logging level; accepts debug, info, warn, error, trace") - flags.StringVar(&cfg.Log.Format, "log.format", cfg.Log.Format, "sets the format of the logging; accepts json, cli") - - flags.StringVar(&cfg.Connectors.Path, "connectors.path", cfg.Connectors.Path, "path to standalone connectors' directory") - flags.StringVar(&cfg.Processors.Path, "processors.path", cfg.Processors.Path, "path to standalone processors' directory") - - // Pipeline configuration - flags.StringVar( - &cfg.Pipelines.Path, - "pipelines.path", - cfg.Pipelines.Path, - "path to the directory that has the yaml pipeline configuration files, or a single pipeline configuration file", - ) - - // Deprecated: use `pipelines.exit-on-degraded` instead - // Note: If both `pipeline.exit-on-error` and `pipeline.exit-on-degraded` are set, `pipeline.exit-on-degraded` will take precedence - flags.BoolVar( - &cfg.Pipelines.ExitOnDegraded, - FlagPipelinesExitOnError, - cfg.Pipelines.ExitOnDegraded, - "Deprecated: use `exit-on-degraded` instead.\nexit Conduit if a pipeline experiences an error while running", - ) - - flags.BoolVar( - &cfg.Pipelines.ExitOnDegraded, - "pipelines.exit-on-degraded", - cfg.Pipelines.ExitOnDegraded, - "exit Conduit if a pipeline enters a degraded state", - ) - - flags.DurationVar( - &cfg.Pipelines.ErrorRecovery.MinDelay, - "pipelines.error-recovery.min-delay", - cfg.Pipelines.ErrorRecovery.MinDelay, - "minimum delay before restart", - ) - flags.DurationVar( - &cfg.Pipelines.ErrorRecovery.MaxDelay, - "pipelines.error-recovery.max-delay", - cfg.Pipelines.ErrorRecovery.MaxDelay, - "maximum delay before restart", - ) - flags.IntVar( - &cfg.Pipelines.ErrorRecovery.BackoffFactor, - "pipelines.error-recovery.backoff-factor", - cfg.Pipelines.ErrorRecovery.BackoffFactor, - "backoff factor applied to the last delay", - ) - flags.Int64Var( - &cfg.Pipelines.ErrorRecovery.MaxRetries, - "pipelines.error-recovery.max-retries", - cfg.Pipelines.ErrorRecovery.MaxRetries, - "maximum number of retries", - ) - flags.DurationVar( - &cfg.Pipelines.ErrorRecovery.MaxRetriesWindow, - "pipelines.error-recovery.max-retries-window", - cfg.Pipelines.ErrorRecovery.MaxRetriesWindow, - "amount of time running without any errors after which a pipeline is considered healthy", - ) - - flags.StringVar(&cfg.SchemaRegistry.Type, "schema-registry.type", cfg.SchemaRegistry.Type, "schema registry type; accepts builtin,confluent") - flags.StringVar(&cfg.SchemaRegistry.Confluent.ConnectionString, "schema-registry.confluent.connection-string", cfg.SchemaRegistry.Confluent.ConnectionString, "confluent schema registry connection string") - - flags.BoolVar(&cfg.Preview.PipelineArchV2, "preview.pipeline-arch-v2", cfg.Preview.PipelineArchV2, "enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)") - - flags.StringVar(&cfg.dev.cpuprofile, "dev.cpuprofile", "", "write cpu profile to file") - flags.StringVar(&cfg.dev.memprofile, "dev.memprofile", "", "write memory profile to file") - flags.StringVar(&cfg.dev.blockprofile, "dev.blockprofile", "", "write block profile to file") - - // show user or dev flags - flags.Usage = func() { - tmpFlags := flag.NewFlagSet("conduit", flag.ExitOnError) - - // preserve original flag's output to the same writer - tmpFlags.SetOutput(flags.Output()) - - flags.VisitAll(func(f *flag.Flag) { - if HiddenFlags[f.Name] { - return - } - // reset value to its default, to ensure default is shown correctly - _ = f.Value.Set(f.DefValue) - tmpFlags.Var(f.Value, f.Name, f.Usage) - }) - tmpFlags.Usage() - } - - return flags -} - -func (e *Entrypoint) ParseConfig(flags *flag.FlagSet) { - _ = flags.String("config", "conduit.yaml", "global config file") - version := flags.Bool("version", false, "prints current Conduit version") - - // flags is set up to exit on error, we can safely ignore the error - err := ff.Parse(flags, os.Args[1:], - ff.WithEnvVarPrefix("CONDUIT"), - ff.WithConfigFileFlag("config"), - ff.WithConfigFileParser(ffyaml.Parser), - ff.WithAllowMissingConfigFile(true), - ) - if err != nil { - e.exitWithError(err) - } - - // check if the -version flag is set - if *version { - _, _ = fmt.Fprintf(os.Stdout, "%s\n", Version(true)) - os.Exit(0) - } -} - // CancelOnInterrupt returns a context that is canceled when the interrupt // signal is received. // * After the first signal the function will continue to listen diff --git a/pkg/conduit/entrypoint_test.go b/pkg/conduit/entrypoint_test.go deleted file mode 100644 index 21c56bdad..000000000 --- a/pkg/conduit/entrypoint_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package conduit - -import ( - "bytes" - "strings" - "testing" -) - -func TestFlags_HelpOutput(t *testing.T) { - var buf bytes.Buffer - - flags := Flags(&Config{}) - flags.SetOutput(&buf) - - flags.Usage() - output := buf.String() - - expectedFlags := []string{ - "db.type", - "db.badger.path", - "db.postgres.connection-string", - "db.postgres.table", - "db.sqlite.path", - "db.sqlite.table", - "dev.blockprofileblockprofile", - "dev.cpuprofile", - "dev.memprofile", - "api.enabled", - "http.address", - "grpc.address", - "log.level", - "log.format", - "connectors.path", - "processors.path", - "pipelines.path", - "pipelines.exit-on-degraded", - "pipelines.error-recovery.min-delay", - "pipelines.error-recovery.max-delay", - "pipelines.error-recovery.backoff-factor", - "pipelines.error-recovery.max-retries", - "pipelines.error-recovery.max-retries-window", - "schema-registry.type", - "schema-registry.confluent.connection-string", - "preview.pipeline-arch-v2", - } - - unexpectedFlags := []string{ - FlagPipelinesExitOnError, - } - - for _, flag := range expectedFlags { - if !strings.Contains(output, flag) { - t.Errorf("expected flag %q not found in help output", flag) - } - } - - for _, flag := range unexpectedFlags { - if strings.Contains(output, flag) { - t.Errorf("unexpected flag %q found in help output", flag) - } - } -} diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index 74df8dcb2..c9c94d2c6 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -388,8 +388,8 @@ func (r *Runtime) initProfiling(ctx context.Context) (deferred func(), err error } }() - if r.Config.dev.cpuprofile != "" { - f, err := os.Create(r.Config.dev.cpuprofile) + if r.Config.Dev.CPUProfile != "" { + f, err := os.Create(r.Config.Dev.CPUProfile) if err != nil { return deferred, cerrors.Errorf("could not create CPU profile: %w", err) } @@ -399,9 +399,9 @@ func (r *Runtime) initProfiling(ctx context.Context) (deferred func(), err error } deferFunc(pprof.StopCPUProfile) } - if r.Config.dev.memprofile != "" { + if r.Config.Dev.MemProfile != "" { deferFunc(func() { - f, err := os.Create(r.Config.dev.memprofile) + f, err := os.Create(r.Config.Dev.MemProfile) if err != nil { r.logger.Err(ctx, err).Msg("could not create memory profile") return @@ -413,10 +413,10 @@ func (r *Runtime) initProfiling(ctx context.Context) (deferred func(), err error } }) } - if r.Config.dev.blockprofile != "" { + if r.Config.Dev.BlockProfile != "" { runtime.SetBlockProfileRate(1) deferFunc(func() { - f, err := os.Create(r.Config.dev.blockprofile) + f, err := os.Create(r.Config.Dev.BlockProfile) if err != nil { r.logger.Err(ctx, err).Msg("could not create block profile") return From 29793c283f7fe9aa6d269d4eb3345f7fb616af12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 25 Nov 2024 19:23:14 +0100 Subject: [PATCH 11/22] fix CI --- cmd/conduit/internal/config.go | 26 +++++++++++++++++++------- go.mod | 3 +-- go.sum | 2 -- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/conduit/internal/config.go b/cmd/conduit/internal/config.go index 4266451f0..d83567a3c 100644 --- a/cmd/conduit/internal/config.go +++ b/cmd/conduit/internal/config.go @@ -1,3 +1,17 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package internal import ( @@ -10,7 +24,7 @@ import ( ) const ( - CONDUIT_PREFIX = "CONDUIT" + ConduitPrefix = "CONDUIT" ) // LoadConfigFromFile loads on cfg, the configuration from the file at path. @@ -20,10 +34,8 @@ func LoadConfigFromFile(filePath string, cfg *conduit.Config) error { // Set the file name and path v.SetConfigFile(filePath) - // Attempt to read the configuration file + // Attempt to read the configuration file. if err := v.ReadInConfig(); err != nil { - // here we could simply log conduit.yaml file doesn't exist since this is optional - //return fmt.Errorf("error reading config file: %w", err) return nil } @@ -40,7 +52,7 @@ func LoadConfigFromEnv(cfg *conduit.Config) error { v := viper.New() // Set environment variable prefix - v.SetEnvPrefix(CONDUIT_PREFIX) + v.SetEnvPrefix(ConduitPrefix) // Automatically map environment variables v.AutomaticEnv() @@ -52,9 +64,9 @@ func LoadConfigFromEnv(cfg *conduit.Config) error { value := pair[1] // Check if the environment variable has the desired prefix - if strings.HasPrefix(key, fmt.Sprintf("%s_", CONDUIT_PREFIX)) { + if strings.HasPrefix(key, fmt.Sprintf("%s_", ConduitPrefix)) { // Strip the prefix and replace underscores with dots - strippedKey := strings.ToLower(strings.TrimPrefix(key, fmt.Sprintf("%s_", CONDUIT_PREFIX))) + strippedKey := strings.ToLower(strings.TrimPrefix(key, fmt.Sprintf("%s_", ConduitPrefix))) strippedKey = strings.ReplaceAll(strippedKey, "_", ".") // Set the value in Viper diff --git a/go.mod b/go.mod index e9fa8c5ee..4cce2fd13 100644 --- a/go.mod +++ b/go.mod @@ -35,13 +35,13 @@ require ( github.com/jpillora/backoff v1.0.0 github.com/matryer/is v1.4.1 github.com/neilotoole/slogt v1.1.0 - github.com/peterbourgon/ff/v3 v3.4.0 github.com/piotrkowalczuk/promgrpc/v4 v4.1.4 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.60.1 github.com/rs/zerolog v1.33.0 github.com/sourcegraph/conc v0.3.0 + github.com/spf13/viper v1.19.0 github.com/stealthrocket/wazergo v0.19.1 github.com/tetratelabs/wazero v1.8.1 github.com/twmb/franz-go/pkg/sr v1.2.0 @@ -321,7 +321,6 @@ require ( github.com/spf13/cast v1.7.0 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.19.0 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect diff --git a/go.sum b/go.sum index 3458d408e..c8ca44aee 100644 --- a/go.sum +++ b/go.sum @@ -720,8 +720,6 @@ github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= -github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= -github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= From 2e16578a1eab4a69095e06df04eb6476c8aa6c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 25 Nov 2024 19:42:35 +0100 Subject: [PATCH 12/22] fix conduit init --- cmd/conduit/root/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index 40e6e92d6..4ec17bee2 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -39,7 +39,7 @@ type RootFlags struct { // Global flags ----------------------------------------------------------- // Conduit configuration file - ConduitConfigPath string `long:"config" usage:"global conduit configuration file" persistent:"true" default:"./conduit.yaml"` + ConduitConfigPath string `long:"config.path" usage:"global conduit configuration file" persistent:"true" default:"./conduit.yaml"` // Version Version bool `long:"version" short:"v" usage:"show current Conduit version" persistent:"true"` From f6717b31bfbc5e098c1702427bc2fb8b81d9fdb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 26 Nov 2024 12:27:51 +0100 Subject: [PATCH 13/22] fix flags on pipelines init --- cmd/conduit/root/pipelines/init.go | 34 +++++++++++++++++------------- cmd/conduit/root/root.go | 2 +- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cmd/conduit/root/pipelines/init.go b/cmd/conduit/root/pipelines/init.go index ac3052681..bbcbe2f61 100644 --- a/cmd/conduit/root/pipelines/init.go +++ b/cmd/conduit/root/pipelines/init.go @@ -29,11 +29,6 @@ import ( "github.com/conduitio/ecdysis" ) -const ( - defaultDestination = "file" - defaultSource = "generator" -) - var ( _ ecdysis.CommandWithDocs = (*InitCommand)(nil) _ ecdysis.CommandWithFlags = (*InitCommand)(nil) @@ -48,9 +43,9 @@ type InitArgs struct { } type InitFlags struct { - Source string `long:"source" usage:"Source connector (any of the built-in connectors)."` - Destination string `long:"destination" usage:"Destination connector (any of the built-in connectors)."` - Path string `long:"pipelines.path" usage:"Path where the pipeline will be saved." default:"./pipelines"` + Source string `long:"source" usage:"Source connector (any of the built-in connectors)." default:"generator"` + Destination string `long:"destination" usage:"Destination connector (any of the built-in connectors)." default:"file"` + PipelinesPath string `long:"pipelines.path" usage:"Path where the pipeline will be saved." default:"./pipelines"` } type InitCommand struct { @@ -59,7 +54,7 @@ type InitCommand struct { } func (c *InitCommand) configFilePath() string { - return filepath.Join(c.flags.Path, c.configFileName()) + return filepath.Join(c.flags.PipelinesPath, c.configFileName()) } func (c *InitCommand) configFileName() string { @@ -67,9 +62,16 @@ func (c *InitCommand) configFileName() string { } func (c *InitCommand) Flags() []ecdysis.Flag { - flags := ecdysis.BuildFlags(&InitFlags{}) + flags := ecdysis.BuildFlags(&c.flags) + + currentPath, err := os.Getwd() + if err != nil { + panic(cerrors.Errorf("failed to get current working directory: %w", err)) + } - flags.SetDefault("pipelines.path", "./pipelines") + flags.SetDefault("pipelines.path", filepath.Join(currentPath, "./pipelines")) + flags.SetDefault("source", "generator") + flags.SetDefault("destination", "file") return flags } @@ -94,7 +96,7 @@ func (c *InitCommand) Docs() ecdysis.Docs { Long: `Initialize a pipeline configuration file, with all of parameters for source and destination connectors initialized and described. The source and destination connector can be chosen via flags. If no connectors are chosen, then a simple and runnable generator-to-log pipeline is configured.`, - Example: "conduit pipelines init awesome-pipeline-name --source postgres --destination kafka --path pipelines/pg-to-kafka.yaml", + Example: "conduit pipelines init awesome-pipeline-name --source postgres --destination kafka --pipelines.path pipelines/pg-to-kafka.yaml", } } @@ -141,7 +143,7 @@ func (c *InitCommand) buildDemoPipeline() pipelineTemplate { return pipelineTemplate{ Name: c.args.name, SourceSpec: connectorTemplate{ - Name: defaultSource, + Name: c.flags.Source, Params: map[string]config.Parameter{ "format.type": { Description: srcParams.Params["format.type"].Description, @@ -167,7 +169,7 @@ func (c *InitCommand) buildDemoPipeline() pipelineTemplate { }, }, DestinationSpec: connectorTemplate{ - Name: defaultDestination, + Name: c.flags.Destination, Params: map[string]config.Parameter{ "path": { Description: dstParams.Params["path"].Description, @@ -227,7 +229,9 @@ func (c *InitCommand) Execute(_ context.Context) error { var pipeline pipelineTemplate // if no source/destination arguments are provided, // we build a runnable example pipeline - if c.flags.Source == "" && c.flags.Destination == "" { + fmt.Printf("pipelines path %s\n", c.flags.PipelinesPath) + // TODO: validate presence of either or. + if c.flags.Source == "" || c.flags.Destination == "" { pipeline = c.buildDemoPipeline() } else { p, err := c.buildTemplatePipeline() diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index 4ec17bee2..5d2ac2d29 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -181,7 +181,7 @@ func (c *RootCommand) updateFlagValuesFromConfig() { c.flags.ProcessorsPath = c.cfg.Processors.Path } -func (c *RootCommand) Execute(ctx context.Context) error { +func (c *RootCommand) Execute(_ context.Context) error { if c.flags.Version { _, _ = fmt.Fprintf(os.Stdout, "%s\n", conduit.Version(true)) return nil From f727f42adbd493badf5fbaa842837e42da478f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 26 Nov 2024 12:49:31 +0100 Subject: [PATCH 14/22] fix pipelines init --- cmd/conduit/root/pipelines/init.go | 57 +++++++++++++----------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/cmd/conduit/root/pipelines/init.go b/cmd/conduit/root/pipelines/init.go index bbcbe2f61..4b35bffa3 100644 --- a/cmd/conduit/root/pipelines/init.go +++ b/cmd/conduit/root/pipelines/init.go @@ -16,6 +16,7 @@ package pipelines import ( "context" + _ "embed" "fmt" "log" "os" @@ -35,9 +36,15 @@ var ( _ ecdysis.CommandWithArgs = (*InitCommand)(nil) _ ecdysis.CommandWithExecute = (*InitCommand)(nil) + //go:embed pipeline.tmpl pipelineCfgTmpl string ) +const ( + defaultSource = "generator" + defaultDestination = "file" +) + type InitArgs struct { name string } @@ -49,16 +56,9 @@ type InitFlags struct { } type InitCommand struct { - args InitArgs - flags InitFlags -} - -func (c *InitCommand) configFilePath() string { - return filepath.Join(c.flags.PipelinesPath, c.configFileName()) -} - -func (c *InitCommand) configFileName() string { - return fmt.Sprintf("pipeline-%s.yaml", c.args.name) + args InitArgs + flags InitFlags + configFilePath string } func (c *InitCommand) Flags() []ecdysis.Flag { @@ -70,8 +70,8 @@ func (c *InitCommand) Flags() []ecdysis.Flag { } flags.SetDefault("pipelines.path", filepath.Join(currentPath, "./pipelines")) - flags.SetDefault("source", "generator") - flags.SetDefault("destination", "file") + flags.SetDefault("source", defaultSource) + flags.SetDefault("destination", defaultDestination) return flags } @@ -143,7 +143,7 @@ func (c *InitCommand) buildDemoPipeline() pipelineTemplate { return pipelineTemplate{ Name: c.args.name, SourceSpec: connectorTemplate{ - Name: c.flags.Source, + Name: defaultSource, Params: map[string]config.Parameter{ "format.type": { Description: srcParams.Params["format.type"].Description, @@ -169,7 +169,7 @@ func (c *InitCommand) buildDemoPipeline() pipelineTemplate { }, }, DestinationSpec: connectorTemplate{ - Name: c.flags.Destination, + Name: defaultDestination, Params: map[string]config.Parameter{ "path": { Description: dstParams.Params["path"].Description, @@ -200,9 +200,9 @@ func (c *InitCommand) buildTemplatePipeline() (pipelineTemplate, error) { } func (c *InitCommand) getOutput() *os.File { - output, err := os.OpenFile(c.configFilePath(), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + output, err := os.OpenFile(c.configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) if err != nil { - log.Fatalf("error: failed to open %s: %v", c.configFilePath(), err) + log.Fatalf("error: failed to open %s: %v", c.configFilePath, err) } return output @@ -226,29 +226,22 @@ func (c *InitCommand) write(pipeline pipelineTemplate) error { } func (c *InitCommand) Execute(_ context.Context) error { - var pipeline pipelineTemplate - // if no source/destination arguments are provided, - // we build a runnable example pipeline - fmt.Printf("pipelines path %s\n", c.flags.PipelinesPath) - // TODO: validate presence of either or. - if c.flags.Source == "" || c.flags.Destination == "" { - pipeline = c.buildDemoPipeline() - } else { - p, err := c.buildTemplatePipeline() - if err != nil { - return err - } - pipeline = p - } + c.configFilePath = filepath.Join(c.flags.PipelinesPath, fmt.Sprintf("pipeline-%s.yaml", c.args.name)) + + // TODO: utilize buildDemoPipeline if source and destination are the default ones - err := c.write(pipeline) + pipeline, err := c.buildTemplatePipeline() if err != nil { + return err + } + + if err := c.write(pipeline); err != nil { return cerrors.Errorf("could not write pipeline: %w", err) } fmt.Printf(`Your pipeline has been initialized and created at %s. -To run the pipeline, simply run 'conduit'.`, c.configFilePath()) +To run the pipeline, simply run 'conduit'.`, c.configFilePath) return nil } From f6061b6a791ef97207fb2bae52886cf8607f5677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 26 Nov 2024 13:03:25 +0100 Subject: [PATCH 15/22] uses simplified version of generator and file --- cmd/conduit/root/pipelines/init.go | 112 +++++++++++++------------ cmd/conduit/root/pipelines/template.go | 6 +- 2 files changed, 62 insertions(+), 56 deletions(-) diff --git a/cmd/conduit/root/pipelines/init.go b/cmd/conduit/root/pipelines/init.go index 4b35bffa3..9c59be9a8 100644 --- a/cmd/conduit/root/pipelines/init.go +++ b/cmd/conduit/root/pipelines/init.go @@ -100,102 +100,110 @@ a simple and runnable generator-to-log pipeline is configured.`, } } -func (c *InitCommand) getSourceParams() (connectorTemplate, error) { +func (c *InitCommand) getSourceSpec() (connectorSpec, error) { for _, conn := range builtin.DefaultBuiltinConnectors { specs := conn.NewSpecification() if specs.Name == c.flags.Source || specs.Name == "builtin:"+c.flags.Source { if conn.NewSource == nil { - return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", c.flags.Source) + return connectorSpec{}, cerrors.Errorf("plugin %v has no source", c.flags.Source) } - return connectorTemplate{ + return connectorSpec{ Name: specs.Name, Params: conn.NewSource().Parameters(), }, nil } } - return connectorTemplate{}, cerrors.Errorf("%v: %w", c.flags.Source, plugin.ErrPluginNotFound) + return connectorSpec{}, cerrors.Errorf("%v: %w", c.flags.Source, plugin.ErrPluginNotFound) } -func (c *InitCommand) getDestinationParams() (connectorTemplate, error) { +func (c *InitCommand) getDestinationSpec() (connectorSpec, error) { for _, conn := range builtin.DefaultBuiltinConnectors { specs := conn.NewSpecification() if specs.Name == c.flags.Destination || specs.Name == "builtin:"+c.flags.Destination { if conn.NewDestination == nil { - return connectorTemplate{}, cerrors.Errorf("plugin %v has no source", c.flags.Destination) + return connectorSpec{}, cerrors.Errorf("plugin %v has no source", c.flags.Destination) } - return connectorTemplate{ + return connectorSpec{ Name: specs.Name, Params: conn.NewDestination().Parameters(), }, nil } } - - return connectorTemplate{}, cerrors.Errorf("%v: %w", c.flags.Destination, plugin.ErrPluginNotFound) + return connectorSpec{}, cerrors.Errorf("%v: %w", c.flags.Destination, plugin.ErrPluginNotFound) } -func (c *InitCommand) buildDemoPipeline() pipelineTemplate { - srcParams, _ := c.getSourceParams() - dstParams, _ := c.getDestinationParams() - - return pipelineTemplate{ - Name: c.args.name, - SourceSpec: connectorTemplate{ - Name: defaultSource, - Params: map[string]config.Parameter{ - "format.type": { - Description: srcParams.Params["format.type"].Description, - Type: srcParams.Params["format.type"].Type, - Default: "structured", - Validations: srcParams.Params["format.type"].Validations, - }, - "format.options.scheduledDeparture": { - Description: "Generate field 'scheduledDeparture' of type 'time'", - Type: config.ParameterTypeString, - Default: "time", - }, - "format.options.airline": { - Description: "Generate field 'airline' of type string", - Type: config.ParameterTypeString, - Default: "string", - }, - "rate": { - Description: srcParams.Params["rate"].Description, - Type: srcParams.Params["rate"].Type, - Default: "1", - }, +// getDemoSourceGeneratorSpec returns a simplified version of the source generator connector. +func (c *InitCommand) getDemoSourceGeneratorSpec(spec connectorSpec) connectorSpec { + return connectorSpec{ + Name: defaultSource, + Params: map[string]config.Parameter{ + "format.type": { + Description: spec.Params["format.type"].Description, + Type: spec.Params["format.type"].Type, + Default: "structured", + Validations: spec.Params["format.type"].Validations, + }, + "format.options.scheduledDeparture": { + Description: "Generate field 'scheduledDeparture' of type 'time'", + Type: config.ParameterTypeString, + Default: "time", + }, + "format.options.airline": { + Description: "Generate field 'airline' of type string", + Type: config.ParameterTypeString, + Default: "string", + }, + "rate": { + Description: spec.Params["rate"].Description, + Type: spec.Params["rate"].Type, + Default: "1", }, }, - DestinationSpec: connectorTemplate{ - Name: defaultDestination, - Params: map[string]config.Parameter{ - "path": { - Description: dstParams.Params["path"].Description, - Type: dstParams.Params["path"].Type, - Default: "./destination.txt", - }, + } +} + +// getDemoDestinationFileSpec returns a simplified version of the destination file connector. +func (c *InitCommand) getDemoDestinationFileSpec(spec connectorSpec) connectorSpec { + return connectorSpec{ + Name: defaultDestination, + Params: map[string]config.Parameter{ + "path": { + Description: spec.Params["path"].Description, + Type: spec.Params["path"].Type, + Default: "./destination.txt", }, }, } } func (c *InitCommand) buildTemplatePipeline() (pipelineTemplate, error) { - srcParams, err := c.getSourceParams() + srcSpec, err := c.getSourceSpec() if err != nil { return pipelineTemplate{}, cerrors.Errorf("failed getting source params: %w", err) } - dstParams, err := c.getDestinationParams() + // provide a simplified version + if c.flags.Source == defaultSource { + srcSpec = c.getDemoSourceGeneratorSpec(srcSpec) + } + + dstSpec, err := c.getDestinationSpec() if err != nil { return pipelineTemplate{}, cerrors.Errorf("failed getting destination params: %w", err) } + // provide a simplified version + if c.flags.Destination == defaultDestination { + dstSpec = c.getDemoDestinationFileSpec(dstSpec) + } + return pipelineTemplate{ Name: c.args.name, - SourceSpec: srcParams, - DestinationSpec: dstParams, + SourceSpec: srcSpec, + DestinationSpec: dstSpec, }, nil } @@ -228,8 +236,6 @@ func (c *InitCommand) write(pipeline pipelineTemplate) error { func (c *InitCommand) Execute(_ context.Context) error { c.configFilePath = filepath.Join(c.flags.PipelinesPath, fmt.Sprintf("pipeline-%s.yaml", c.args.name)) - // TODO: utilize buildDemoPipeline if source and destination are the default ones - pipeline, err := c.buildTemplatePipeline() if err != nil { return err diff --git a/cmd/conduit/root/pipelines/template.go b/cmd/conduit/root/pipelines/template.go index fa3e0f005..d7a707309 100644 --- a/cmd/conduit/root/pipelines/template.go +++ b/cmd/conduit/root/pipelines/template.go @@ -16,13 +16,13 @@ package pipelines import "github.com/conduitio/conduit-commons/config" -type connectorTemplate struct { +type connectorSpec struct { Name string Params config.Parameters } type pipelineTemplate struct { Name string - SourceSpec connectorTemplate - DestinationSpec connectorTemplate + SourceSpec connectorSpec + DestinationSpec connectorSpec } From b13f469b0b73fe865ee48fbadf9640d763f397fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 26 Nov 2024 13:39:32 +0100 Subject: [PATCH 16/22] fix init --- cmd/conduit/root/init.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cmd/conduit/root/init.go b/cmd/conduit/root/init.go index c2d6b506e..c974f07d3 100644 --- a/cmd/conduit/root/init.go +++ b/cmd/conduit/root/init.go @@ -73,7 +73,20 @@ func (c *InitCommand) createConfigYAML() error { v := reflect.Indirect(reflect.ValueOf(c.rootFlags)) t := v.Type() + ignoreKeys := map[string]bool{ + "ConduitConfigPath": true, + "DBType": true, + "Version": true, + "DevCPUProfile": true, + "DevMemProfile": true, + "DevBlockProfile": true, + } + for i := 0; i < v.NumField(); i++ { + if ignoreKeys[t.Field(i).Name] { + continue + } + field := t.Field(i) value := fmt.Sprintf("%v", v.Field(i).Interface()) usage := field.Tag.Get("usage") From 16a5656454cabc9bdfcadf1e497806094454cd23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 26 Nov 2024 14:00:17 +0100 Subject: [PATCH 17/22] fix removed config --- cmd/conduit/internal/config.go | 1 - cmd/conduit/root/init.go | 1 - 2 files changed, 2 deletions(-) diff --git a/cmd/conduit/internal/config.go b/cmd/conduit/internal/config.go index d83567a3c..895a46b9d 100644 --- a/cmd/conduit/internal/config.go +++ b/cmd/conduit/internal/config.go @@ -47,7 +47,6 @@ func LoadConfigFromFile(filePath string, cfg *conduit.Config) error { return nil } -// TODO: check if logger is correct func LoadConfigFromEnv(cfg *conduit.Config) error { v := viper.New() diff --git a/cmd/conduit/root/init.go b/cmd/conduit/root/init.go index c974f07d3..097bc8c42 100644 --- a/cmd/conduit/root/init.go +++ b/cmd/conduit/root/init.go @@ -75,7 +75,6 @@ func (c *InitCommand) createConfigYAML() error { ignoreKeys := map[string]bool{ "ConduitConfigPath": true, - "DBType": true, "Version": true, "DevCPUProfile": true, "DevMemProfile": true, From ba72008db3af68fab75e51d5687bf53c2600ef17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 27 Nov 2024 15:34:36 +0100 Subject: [PATCH 18/22] test root flags --- cmd/conduit/root/root_test.go | 85 +++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 cmd/conduit/root/root_test.go diff --git a/cmd/conduit/root/root_test.go b/cmd/conduit/root/root_test.go new file mode 100644 index 000000000..8be1c2d3f --- /dev/null +++ b/cmd/conduit/root/root_test.go @@ -0,0 +1,85 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package root + +import ( + "testing" + + "github.com/conduitio/ecdysis" + "github.com/matryer/is" +) + +func TestRootCommandFlags(t *testing.T) { + is := is.New(t) + + expectedFlags := []struct { + longName string + shortName string + required bool + persistent bool + hidden bool + }{ + {longName: "config.path", persistent: true}, + {longName: "version", shortName: "v", persistent: true}, + {longName: "db.type"}, + {longName: "db.badger.path"}, + {longName: "db.postgres.connection-string"}, + {longName: "db.postgres.table"}, + {longName: "db.sqlite.path"}, + {longName: "db.sqlite.table"}, + {longName: "api.enabled"}, + {longName: "http.address"}, + {longName: "grpc.address"}, + {longName: "log.level"}, + {longName: "log.format"}, + {longName: "connectors.path"}, + {longName: "processors.path"}, + {longName: "pipelines.path"}, + {longName: "pipelines.exit-on-degraded"}, + {longName: "pipelines.error-recovery.min-delay"}, + {longName: "pipelines.error-recovery.max-delay"}, + {longName: "pipelines.error-recovery.backoff-factor"}, + {longName: "pipelines.error-recovery.max-retries"}, + {longName: "pipelines.error-recovery.max-retries-window"}, + {longName: "schema-registry.type"}, + {longName: "schema-registry.confluent.connection-string"}, + {longName: "preview.pipeline-arch-v2"}, + {longName: "dev.cpuprofile"}, + {longName: "dev.memprofile"}, + {longName: "dev.blockprofile"}, + } + + c := &RootCommand{} + flags := c.Flags() + + for _, ef := range expectedFlags { + var foundFlag *ecdysis.Flag + for _, f := range flags { + if f.Long == ef.longName { + foundFlag = &f + break + } + } + + is.True(foundFlag != nil) + + if foundFlag != nil { + is.Equal(ef.shortName, foundFlag.Short) + is.Equal(ef.required, foundFlag.Required) + is.Equal(ef.persistent, foundFlag.Persistent) + is.Equal(ef.hidden, foundFlag.Hidden) + } + } +} From b6abfb6908e058f1244db3ee1d42e16e60e17afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 27 Nov 2024 16:55:10 +0100 Subject: [PATCH 19/22] simpler version --- cmd/conduit/internal/config.go | 82 ------------ cmd/conduit/root/root.go | 233 ++++++++++++++++----------------- go.mod | 2 + go.sum | 2 - pkg/conduit/config.go | 51 ++++---- 5 files changed, 138 insertions(+), 232 deletions(-) delete mode 100644 cmd/conduit/internal/config.go diff --git a/cmd/conduit/internal/config.go b/cmd/conduit/internal/config.go deleted file mode 100644 index 895a46b9d..000000000 --- a/cmd/conduit/internal/config.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "fmt" - "os" - "strings" - - "github.com/conduitio/conduit/pkg/conduit" - "github.com/spf13/viper" -) - -const ( - ConduitPrefix = "CONDUIT" -) - -// LoadConfigFromFile loads on cfg, the configuration from the file at path. -func LoadConfigFromFile(filePath string, cfg *conduit.Config) error { - v := viper.New() - - // Set the file name and path - v.SetConfigFile(filePath) - - // Attempt to read the configuration file. - if err := v.ReadInConfig(); err != nil { - return nil - } - - // Unmarshal the config into the cfg struct - if err := v.Unmarshal(&cfg); err != nil { - return fmt.Errorf("unable to decode into struct: %w", err) - } - - return nil -} - -func LoadConfigFromEnv(cfg *conduit.Config) error { - v := viper.New() - - // Set environment variable prefix - v.SetEnvPrefix(ConduitPrefix) - - // Automatically map environment variables - v.AutomaticEnv() - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - - for _, env := range os.Environ() { - pair := strings.SplitN(env, "=", 2) - key := pair[0] - value := pair[1] - - // Check if the environment variable has the desired prefix - if strings.HasPrefix(key, fmt.Sprintf("%s_", ConduitPrefix)) { - // Strip the prefix and replace underscores with dots - strippedKey := strings.ToLower(strings.TrimPrefix(key, fmt.Sprintf("%s_", ConduitPrefix))) - strippedKey = strings.ReplaceAll(strippedKey, "_", ".") - - // Set the value in Viper - v.Set(strippedKey, value) - } - } - - // Unmarshal the environment variables into the config struct - err := v.Unmarshal(cfg) - if err != nil { - return fmt.Errorf("error unmarshalling config from environment variables: %w", err) - } - return nil -} diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index 5d2ac2d29..ba0a96be5 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -19,13 +19,13 @@ import ( "fmt" "os" "path/filepath" - "time" + "strings" - "github.com/conduitio/conduit/cmd/conduit/internal" "github.com/conduitio/conduit/cmd/conduit/root/pipelines" "github.com/conduitio/conduit/pkg/conduit" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/ecdysis" + "github.com/spf13/viper" ) var ( @@ -35,6 +35,8 @@ var ( _ ecdysis.CommandWithSubCommands = (*RootCommand)(nil) ) +const ConduitPrefix = "CONDUIT" + type RootFlags struct { // Global flags ----------------------------------------------------------- @@ -44,49 +46,7 @@ type RootFlags struct { // Version Version bool `long:"version" short:"v" usage:"show current Conduit version" persistent:"true"` - // Root flags ------------------------------------------------------------- - - // Database configuration - DBType string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite"` - DBBadgerPath string `long:"db.badger.path" usage:"path to badger DB"` - DBPostgresConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format"` - DBPostgresTable string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)"` - DBSQLitePath string `long:"db.sqlite.path" usage:"path to sqlite3 DB"` - DBSQLiteTable string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)"` - - // API configuration - APIEnabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API"` - APIHTTPAddress string `long:"http.address" usage:"address for serving the HTTP API"` - APIGRPCAddress string `long:"grpc.address" usage:"address for serving the gRPC API"` - - // Logging configuration - LogLevel string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace"` - LogFormat string `long:"log.format" usage:"sets the format of the logging; accepts json, cli"` - - // Connectors and Processors paths - ConnectorsPath string `long:"connectors.path" usage:"path to standalone connectors' directory"` - ProcessorsPath string `long:"processors.path" usage:"path to standalone processors' directory"` - - // Pipeline configuration - PipelinesPath string `long:"pipelines.path" usage:"path to the directory that has the yaml pipeline configuration files, or a single pipeline configuration file"` - PipelinesExitOnDegraded bool `long:"pipelines.exit-on-degraded" usage:"exit Conduit if a pipeline enters a degraded state"` - PipelinesErrorRecoveryMinDelay time.Duration `long:"pipelines.error-recovery.min-delay" usage:"minimum delay before restart"` - PipelinesErrorRecoveryMaxDelay time.Duration `long:"pipelines.error-recovery.max-delay" usage:"maximum delay before restart"` - PipelinesErrorRecoveryBackoffFactor int `long:"pipelines.error-recovery.backoff-factor" usage:"backoff factor applied to the last delay"` - PipelinesErrorRecoveryMaxRetries int64 `long:"pipelines.error-recovery.max-retries" usage:"maximum number of retries"` - PipelinesErrorRecoveryMaxRetriesWindow time.Duration `long:"pipelines.error-recovery.max-retries-window" usage:"amount of time running without any errors after which a pipeline is considered healthy"` - - // Schema registry configuration - SchemaRegistryType string `long:"schema-registry.type" usage:"schema registry type; accepts builtin,confluent"` - SchemaRegistryConfluentConnectionString string `long:"schema-registry.confluent.connection-string" usage:"confluent schema registry connection string"` - - // Preview features - PreviewPipelineArchV2 bool `long:"preview.pipeline-arch-v2" usage:"enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)"` - - // Development profiling - DevCPUProfile string `long:"dev.cpuprofile" usage:"write CPU profile to file"` - DevMemProfile string `long:"dev.memprofile" usage:"write memory profile to file"` - DevBlockProfile string `long:"dev.blockprofile" usage:"write block profile to file"` + conduit.Config } type RootCommand struct { @@ -94,91 +54,132 @@ type RootCommand struct { cfg conduit.Config } -func (c *RootCommand) updateConfigFromFlags() { - c.cfg.DB.Type = c.flags.DBType - c.cfg.DB.Postgres.ConnectionString = c.flags.DBPostgresConnectionString - c.cfg.DB.Postgres.Table = c.flags.DBPostgresTable - c.cfg.DB.SQLite.Table = c.flags.DBSQLiteTable +func (c *RootCommand) updateFlagValuesFromConfig() { + // Map database configuration + c.flags.DB.Type = c.cfg.DB.Type + c.flags.DB.Postgres.ConnectionString = c.cfg.DB.Postgres.ConnectionString + c.flags.DB.Postgres.Table = c.cfg.DB.Postgres.Table + c.flags.DB.SQLite.Table = c.cfg.DB.SQLite.Table // Map API configuration - c.cfg.API.Enabled = c.flags.APIEnabled - c.cfg.API.HTTP.Address = c.flags.APIHTTPAddress - c.cfg.API.GRPC.Address = c.flags.APIGRPCAddress + c.flags.API.Enabled = c.cfg.API.Enabled + c.flags.API.HTTP.Address = c.cfg.API.HTTP.Address + c.flags.API.GRPC.Address = c.cfg.API.GRPC.Address // Map logging configuration - c.cfg.Log.Level = c.flags.LogLevel - c.cfg.Log.Format = c.flags.LogFormat + c.flags.Log.Level = c.cfg.Log.Level + c.flags.Log.Format = c.cfg.Log.Format // Map pipeline configuration - c.cfg.Pipelines.ExitOnDegraded = c.flags.PipelinesExitOnDegraded - c.cfg.Pipelines.ErrorRecovery.MinDelay = c.flags.PipelinesErrorRecoveryMinDelay - c.cfg.Pipelines.ErrorRecovery.MaxDelay = c.flags.PipelinesErrorRecoveryMaxDelay - c.cfg.Pipelines.ErrorRecovery.BackoffFactor = c.flags.PipelinesErrorRecoveryBackoffFactor - c.cfg.Pipelines.ErrorRecovery.MaxRetries = c.flags.PipelinesErrorRecoveryMaxRetries - c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow = c.flags.PipelinesErrorRecoveryMaxRetriesWindow + c.flags.Pipelines.ExitOnDegraded = c.cfg.Pipelines.ExitOnDegraded + c.flags.Pipelines.ErrorRecovery.MinDelay = c.cfg.Pipelines.ErrorRecovery.MinDelay + c.flags.Pipelines.ErrorRecovery.MaxDelay = c.cfg.Pipelines.ErrorRecovery.MaxDelay + c.flags.Pipelines.ErrorRecovery.BackoffFactor = c.cfg.Pipelines.ErrorRecovery.BackoffFactor + c.flags.Pipelines.ErrorRecovery.MaxRetries = c.cfg.Pipelines.ErrorRecovery.MaxRetries + c.flags.Pipelines.ErrorRecovery.MaxRetriesWindow = c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow // Map schema registry configuration - c.cfg.SchemaRegistry.Type = c.flags.SchemaRegistryType - c.cfg.SchemaRegistry.Confluent.ConnectionString = c.flags.SchemaRegistryConfluentConnectionString + c.flags.SchemaRegistry.Type = c.cfg.SchemaRegistry.Type + c.flags.SchemaRegistry.Confluent.ConnectionString = c.cfg.SchemaRegistry.Confluent.ConnectionString // Map preview features - c.cfg.Preview.PipelineArchV2 = c.flags.PreviewPipelineArchV2 + c.flags.Preview.PipelineArchV2 = c.cfg.Preview.PipelineArchV2 // Map development profiling - c.cfg.Dev.CPUProfile = c.flags.DevCPUProfile - c.cfg.Dev.MemProfile = c.flags.DevMemProfile - c.cfg.Dev.BlockProfile = c.flags.DevBlockProfile + c.flags.Dev.CPUProfile = c.cfg.Dev.CPUProfile + c.flags.Dev.MemProfile = c.cfg.Dev.MemProfile + c.flags.Dev.BlockProfile = c.cfg.Dev.BlockProfile // Update paths - c.cfg.DB.SQLite.Path = c.flags.DBSQLitePath - c.cfg.DB.Badger.Path = c.flags.DBBadgerPath - c.cfg.Pipelines.Path = c.flags.PipelinesPath - c.cfg.Connectors.Path = c.flags.ConnectorsPath - c.cfg.Processors.Path = c.flags.ProcessorsPath + c.flags.DB.SQLite.Path = c.cfg.DB.SQLite.Path + c.flags.DB.Badger.Path = c.cfg.DB.Badger.Path + c.flags.Pipelines.Path = c.cfg.Pipelines.Path + c.flags.Connectors.Path = c.cfg.Connectors.Path + c.flags.Processors.Path = c.cfg.Processors.Path } -func (c *RootCommand) updateFlagValuesFromConfig() { - // Map database configuration - c.flags.DBType = c.cfg.DB.Type - c.flags.DBPostgresConnectionString = c.cfg.DB.Postgres.ConnectionString - c.flags.DBPostgresTable = c.cfg.DB.Postgres.Table - c.flags.DBSQLiteTable = c.cfg.DB.SQLite.Table - - // Map API configuration - c.flags.APIEnabled = c.cfg.API.Enabled - c.flags.APIHTTPAddress = c.cfg.API.HTTP.Address - c.flags.APIGRPCAddress = c.cfg.API.GRPC.Address - - // Map logging configuration - c.flags.LogLevel = c.cfg.Log.Level - c.flags.LogFormat = c.cfg.Log.Format +func (c *RootCommand) LoadConfig() error { + v := viper.New() + + // Set default values + v.SetDefault("config.path", c.flags.ConduitConfigPath) + v.SetDefault("db.type", c.flags.DB.Type) + v.SetDefault("api.enabled", c.flags.API.Enabled) + v.SetDefault("log.level", c.flags.Log.Level) + v.SetDefault("log.format", c.flags.Log.Format) + v.SetDefault("connectors.path", c.flags.Connectors.Path) + v.SetDefault("processors.path", c.flags.Processors.Path) + v.SetDefault("pipelines.path", c.flags.Pipelines.Path) + v.SetDefault("pipelines.exit-on-degraded", c.flags.Pipelines.ExitOnDegraded) + v.SetDefault("pipelines.error-recovery.min-delay", c.flags.Pipelines.ErrorRecovery.MinDelay) + v.SetDefault("pipelines.error-recovery.max-delay", c.flags.Pipelines.ErrorRecovery.MaxDelay) + v.SetDefault("pipelines.error-recovery.backoff-factor", c.flags.Pipelines.ErrorRecovery.BackoffFactor) + v.SetDefault("pipelines.error-recovery.max-retries", c.flags.Pipelines.ErrorRecovery.MaxRetries) + v.SetDefault("pipelines.error-recovery.max-retries-window", c.flags.Pipelines.ErrorRecovery.MaxRetriesWindow) + v.SetDefault("schema-registry.type", c.flags.SchemaRegistry.Type) + v.SetDefault("schema-registry.confluent.connection-string", c.flags.SchemaRegistry.Confluent.ConnectionString) + v.SetDefault("preview.pipeline-arch-v2", c.flags.Preview.PipelineArchV2) + v.SetDefault("dev.cpuprofile", c.flags.Dev.CPUProfile) + v.SetDefault("dev.memprofile", c.flags.Dev.MemProfile) + v.SetDefault("dev.blockprofile", c.flags.Dev.BlockProfile) + + // Read configuration from file + v.SetConfigFile(c.flags.ConduitConfigPath) + if err := v.ReadInConfig(); err != nil { + //return fmt.Errorf("error reading config file: %w", err) + } - // Map pipeline configuration - c.flags.PipelinesExitOnDegraded = c.cfg.Pipelines.ExitOnDegraded - c.flags.PipelinesErrorRecoveryMinDelay = c.cfg.Pipelines.ErrorRecovery.MinDelay - c.flags.PipelinesErrorRecoveryMaxDelay = c.cfg.Pipelines.ErrorRecovery.MaxDelay - c.flags.PipelinesErrorRecoveryBackoffFactor = c.cfg.Pipelines.ErrorRecovery.BackoffFactor - c.flags.PipelinesErrorRecoveryMaxRetries = c.cfg.Pipelines.ErrorRecovery.MaxRetries - c.flags.PipelinesErrorRecoveryMaxRetriesWindow = c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow + // Set environment variable prefix and automatic mapping + v.SetEnvPrefix(ConduitPrefix) + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.AutomaticEnv() + + // Bind flags to Viper + v.BindEnv("db.type") + v.BindEnv("db.badger.path") + v.BindEnv("db.postgres.connection-string") + v.BindEnv("db.postgres.table") + v.BindEnv("db.sqlite.path") + v.BindEnv("db.sqlite.table") + v.BindEnv("api.enabled") + v.BindEnv("http.address") + v.BindEnv("grpc.address") + v.BindEnv("log.level") + v.BindEnv("log.format") + v.BindEnv("connectors.path") + v.BindEnv("processors.path") + v.BindEnv("pipelines.path") + v.BindEnv("pipelines.exit-on-degraded") + v.BindEnv("pipelines.error-recovery.min-delay") + v.BindEnv("pipelines.error-recovery.max-delay") + v.BindEnv("pipelines.error-recovery.backoff-factor") + v.BindEnv("pipelines.error-recovery.max-retries") + v.BindEnv("pipelines.error-recovery.max-retries-window") + v.BindEnv("schema-registry.type") + v.BindEnv("schema-registry.confluent.connection-string") + v.BindEnv("preview.pipeline-arch-v2") + v.BindEnv("dev.cpuprofile") + v.BindEnv("dev.memprofile") + v.BindEnv("dev.blockprofile") + + // Unmarshal into the configuration struct + if err := v.Unmarshal(&c.cfg); err != nil { + return fmt.Errorf("unable to decode into struct: %w", err) + } - // Map schema registry configuration - c.flags.SchemaRegistryType = c.cfg.SchemaRegistry.Type - c.flags.SchemaRegistryConfluentConnectionString = c.cfg.SchemaRegistry.Confluent.ConnectionString + return nil +} - // Map preview features - c.flags.PreviewPipelineArchV2 = c.cfg.Preview.PipelineArchV2 +func (c *RootCommand) updateConfiguration() error { + // 1. Load conduit configuration file and update general config. + if err := c.LoadConfig(); err != nil { + return err + } - // Map development profiling - c.flags.DevCPUProfile = c.cfg.Dev.CPUProfile - c.flags.DevMemProfile = c.cfg.Dev.MemProfile - c.flags.DevBlockProfile = c.cfg.Dev.BlockProfile + // 4. Update flags from global configuration (this will be needed for conduit init) + c.updateFlagValuesFromConfig() - // Update paths - c.flags.DBSQLitePath = c.cfg.DB.SQLite.Path - c.flags.DBBadgerPath = c.cfg.DB.Badger.Path - c.flags.PipelinesPath = c.cfg.Pipelines.Path - c.flags.ConnectorsPath = c.cfg.Connectors.Path - c.flags.ProcessorsPath = c.cfg.Processors.Path + return nil } func (c *RootCommand) Execute(_ context.Context) error { @@ -187,22 +188,10 @@ func (c *RootCommand) Execute(_ context.Context) error { return nil } - // 1. Load conduit configuration file and update general config. - if err := internal.LoadConfigFromFile(c.flags.ConduitConfigPath, &c.cfg); err != nil { - return err - } - - // 2. Load environment variables and update general config. - if err := internal.LoadConfigFromEnv(&c.cfg); err != nil { + if err := c.updateConfiguration(); err != nil { return err } - // 3. Update the general config from flags. - c.updateConfigFromFlags() - - // 4. Update flags from global configuration (this will be needed for conduit init) - c.updateFlagValuesFromConfig() - e := &conduit.Entrypoint{} e.Serve(c.cfg) return nil diff --git a/go.mod b/go.mod index 35e59368c..84e359965 100644 --- a/go.mod +++ b/go.mod @@ -388,3 +388,5 @@ require ( mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect pluginrpc.com/pluginrpc v0.5.0 // indirect ) + +replace github.com/conduitio/ecdysis => ../ecdysis diff --git a/go.sum b/go.sum index 6033cf2f9..b0f88218c 100644 --- a/go.sum +++ b/go.sum @@ -244,8 +244,6 @@ github.com/conduitio/conduit-processor-sdk v0.4.0 h1:wF1Fj31aneNixNbW5rJ0/5Q3vwW github.com/conduitio/conduit-processor-sdk v0.4.0/go.mod h1:Jj9ZBTee7nO0XeociDxe9gSvLFN1GbPWP1Aj04DPeZQ= github.com/conduitio/conduit-schema-registry v0.2.2 h1:Q0uL8egRAzJlRV7Ed5nEcqZ1yE/UQeZJad3VmhgTSFE= github.com/conduitio/conduit-schema-registry v0.2.2/go.mod h1:EmT4ylkz15LYddL6qU4wDX52n1Yp0aHvEDRIWOYYzFs= -github.com/conduitio/ecdysis v0.0.0-20241104140515-1031f323f080 h1:Pd5uzNGyPy/Va3rCFp0Ni2xzohrpvgsqbdbAyeFpoZs= -github.com/conduitio/ecdysis v0.0.0-20241104140515-1031f323f080/go.mod h1:JWjm2WhbGExspVAOH+8tb1t9iu+RxKrHiMvWgtPsUI8= github.com/conduitio/yaml/v3 v3.3.0 h1:kbbaOSHcuH39gP4+rgbJGl6DSbLZcJgEaBvkEXJlCsI= github.com/conduitio/yaml/v3 v3.3.0/go.mod h1:JNgFMOX1t8W4YJuRZOh6GggVtSMsgP9XgTw+7dIenpc= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= diff --git a/pkg/conduit/config.go b/pkg/conduit/config.go index 1ddeb7aa7..afceb9334 100644 --- a/pkg/conduit/config.go +++ b/pkg/conduit/config.go @@ -46,43 +46,42 @@ type Config struct { // fields. Driver database.DB - Type string + Type string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite"` Badger struct { - Path string + Path string `long:"db.badger.path" usage:"path to badger DB"` } Postgres struct { - ConnectionString string - Table string + ConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format"` + Table string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)"` } SQLite struct { - Path string - Table string + Path string `long:"db.sqlite.path" usage:"path to sqlite3 DB"` + Table string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)"` } } API struct { - Enabled bool - - HTTP struct { - Address string + Enabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API"` + HTTP struct { + Address string `long:"http.address" usage:"address for serving the HTTP API"` } GRPC struct { - Address string + Address string `long:"grpc.address" usage:"address for serving the gRPC API"` } } Log struct { NewLogger func(level, format string) log.CtxLogger - Level string - Format string + Level string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace"` + Format string `long:"log.format" usage:"sets the format of the logging; accepts json, cli"` } Connectors struct { - Path string + Path string `long:"connectors.path" usage:"path to standalone connectors' directory"` } Processors struct { - Path string + Path string `long:"processors.path" usage:"path to standalone processors' directory"` } Pipelines struct { @@ -90,37 +89,37 @@ type Config struct { ExitOnDegraded bool ErrorRecovery struct { // MinDelay is the minimum delay before restart: Default: 1 second - MinDelay time.Duration + MinDelay time.Duration `long:"pipelines.error-recovery.min-delay" usage:"minimum delay before restart"` // MaxDelay is the maximum delay before restart: Default: 10 minutes - MaxDelay time.Duration + MaxDelay time.Duration `long:"pipelines.error-recovery.max-delay" usage:"maximum delay before restart"` // BackoffFactor is the factor by which the delay is multiplied after each restart: Default: 2 - BackoffFactor int + BackoffFactor int `long:"pipelines.error-recovery.backoff-factor" usage:"backoff factor applied to the last delay"` // MaxRetries is the maximum number of restarts before the pipeline is considered unhealthy: Default: -1 (infinite) - MaxRetries int64 + MaxRetries int64 `long:"pipelines.error-recovery.max-retries" usage:"maximum number of retries"` // MaxRetriesWindow is the duration window in which the max retries are counted: Default: 5 minutes - MaxRetriesWindow time.Duration + MaxRetriesWindow time.Duration `long:"pipelines.error-recovery.max-retries-window" usage:"amount of time running without any errors after which a pipeline is considered healthy"` } } ConnectorPlugins map[string]sdk.Connector SchemaRegistry struct { - Type string + Type string `long:"schema-registry.type" usage:"schema registry type; accepts builtin,confluent"` Confluent struct { - ConnectionString string + ConnectionString string `long:"schema-registry.confluent.connection-string" usage:"confluent schema registry connection string"` } } Preview struct { // PipelineArchV2 enables the new pipeline architecture. - PipelineArchV2 bool + PipelineArchV2 bool `long:"preview.pipeline-arch-v2" usage:"enables experimental pipeline architecture v2 (note that the new architecture currently supports only 1 source and 1 destination per pipeline)"` } Dev struct { - CPUProfile string - MemProfile string - BlockProfile string + CPUProfile string `long:"dev.cpuprofile" usage:"write CPU profile to file"` + MemProfile string `long:"dev.memprofile" usage:"write memory profile to file"` + BlockProfile string `long:"dev.blockprofile" usage:"write block profile to file"` } } From 5964d6b9ab67dc013e58f087adcf04359f6f04f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 27 Nov 2024 18:09:55 +0100 Subject: [PATCH 20/22] use config on init --- cmd/conduit/root/init.go | 54 +++++++++++++++++++++++++++------------- cmd/conduit/root/root.go | 28 ++++++--------------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/cmd/conduit/root/init.go b/cmd/conduit/root/init.go index 097bc8c42..365d61f12 100644 --- a/cmd/conduit/root/init.go +++ b/cmd/conduit/root/init.go @@ -22,6 +22,7 @@ import ( "reflect" "github.com/conduitio/conduit/cmd/conduit/internal" + "github.com/conduitio/conduit/pkg/conduit" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/ecdysis" "github.com/conduitio/yaml/v3" @@ -33,6 +34,7 @@ var ( ) type InitCommand struct { + cfg *conduit.Config rootFlags *RootFlags } @@ -70,28 +72,26 @@ func (c *InitCommand) createDirs() error { func (c *InitCommand) createConfigYAML() error { cfgYAML := internal.NewYAMLTree() - v := reflect.Indirect(reflect.ValueOf(c.rootFlags)) + v := reflect.Indirect(reflect.ValueOf(c.cfg)) t := v.Type() - ignoreKeys := map[string]bool{ - "ConduitConfigPath": true, - "Version": true, - "DevCPUProfile": true, - "DevMemProfile": true, - "DevBlockProfile": true, - } - for i := 0; i < v.NumField(); i++ { - if ignoreKeys[t.Field(i).Name] { - continue - } - field := t.Field(i) - value := fmt.Sprintf("%v", v.Field(i).Interface()) - usage := field.Tag.Get("usage") - longName := field.Tag.Get("long") - cfgYAML.Insert(longName, value, usage) + fieldValue := v.Field(i) + + if fieldValue.Kind() == reflect.Struct { + embedStructYAML(fieldValue, field, cfgYAML) + } else { + value := fmt.Sprintf("%v", fieldValue.Interface()) + usage := field.Tag.Get("usage") + longName := field.Tag.Get("long") + + if longName != "" { + cfgYAML.Insert(longName, value, usage) + } + } } + yamlData, err := yaml.Marshal(cfgYAML.Root) if err != nil { return cerrors.Errorf("error marshaling YAML: %w\n", err) @@ -106,6 +106,26 @@ func (c *InitCommand) createConfigYAML() error { return nil } +func embedStructYAML(v reflect.Value, field reflect.StructField, cfgYAML *internal.YAMLTree) { + t := v.Type() + for i := 0; i < v.NumField(); i++ { + subField := t.Field(i) + subFieldValue := v.Field(i) + + if subFieldValue.Kind() == reflect.Struct { + embedStructYAML(subFieldValue, subField, cfgYAML) + } else { + value := fmt.Sprintf("%v", subFieldValue.Interface()) + usage := subField.Tag.Get("usage") + longName := subField.Tag.Get("long") + + if longName != "" { + cfgYAML.Insert(longName, value, usage) + } + } + } +} + func (c *InitCommand) Execute(ctx context.Context) error { err := c.createDirs() if err != nil { diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index ba0a96be5..0b28d3fd3 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -98,7 +98,7 @@ func (c *RootCommand) updateFlagValuesFromConfig() { c.flags.Processors.Path = c.cfg.Processors.Path } -func (c *RootCommand) LoadConfig() error { +func (c *RootCommand) updateConfig() error { v := viper.New() // Set default values @@ -125,16 +125,15 @@ func (c *RootCommand) LoadConfig() error { // Read configuration from file v.SetConfigFile(c.flags.ConduitConfigPath) - if err := v.ReadInConfig(); err != nil { - //return fmt.Errorf("error reading config file: %w", err) - } + + // ignore if file doesn't exist. Maybe we could check if user is trying read from a file that doesn't exist. + _ = v.ReadInConfig() // Set environment variable prefix and automatic mapping v.SetEnvPrefix(ConduitPrefix) v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.AutomaticEnv() - // Bind flags to Viper v.BindEnv("db.type") v.BindEnv("db.badger.path") v.BindEnv("db.postgres.connection-string") @@ -162,33 +161,20 @@ func (c *RootCommand) LoadConfig() error { v.BindEnv("dev.memprofile") v.BindEnv("dev.blockprofile") - // Unmarshal into the configuration struct if err := v.Unmarshal(&c.cfg); err != nil { - return fmt.Errorf("unable to decode into struct: %w", err) + return fmt.Errorf("unable to unmarshal the configuration: %w", err) } return nil } -func (c *RootCommand) updateConfiguration() error { - // 1. Load conduit configuration file and update general config. - if err := c.LoadConfig(); err != nil { - return err - } - - // 4. Update flags from global configuration (this will be needed for conduit init) - c.updateFlagValuesFromConfig() - - return nil -} - func (c *RootCommand) Execute(_ context.Context) error { if c.flags.Version { _, _ = fmt.Fprintf(os.Stdout, "%s\n", conduit.Version(true)) return nil } - if err := c.updateConfiguration(); err != nil { + if err := c.updateConfig(); err != nil { return err } @@ -245,7 +231,7 @@ func (c *RootCommand) Docs() ecdysis.Docs { func (c *RootCommand) SubCommands() []ecdysis.Command { return []ecdysis.Command{ - &InitCommand{rootFlags: &c.flags}, + &InitCommand{cfg: &c.cfg, rootFlags: &c.flags}, &pipelines.PipelinesCommand{}, } } From 62f20cf536e106f8ca658a7470ca09eb23634074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 2 Dec 2024 12:53:09 +0100 Subject: [PATCH 21/22] test in progress --- cmd/conduit/root/root_test.go | 149 +++++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 2 deletions(-) diff --git a/cmd/conduit/root/root_test.go b/cmd/conduit/root/root_test.go index 8be1c2d3f..5a653aec6 100644 --- a/cmd/conduit/root/root_test.go +++ b/cmd/conduit/root/root_test.go @@ -15,14 +15,17 @@ package root import ( + "os" + "path/filepath" "testing" + "github.com/conduitio/conduit/pkg/conduit" "github.com/conduitio/ecdysis" - "github.com/matryer/is" + isT "github.com/matryer/is" ) func TestRootCommandFlags(t *testing.T) { - is := is.New(t) + is := isT.New(t) expectedFlags := []struct { longName string @@ -83,3 +86,145 @@ func TestRootCommandFlags(t *testing.T) { } } } + +func TestRootCommand_updateConfig(t *testing.T) { + is := isT.New(t) + + tmpDir, err := os.MkdirTemp("", "config-test") + is.NoErr(err) + defer os.RemoveAll(tmpDir) + + configFileContent := ` +db: + type: "sqlite" + sqlite: + path: "/custom/path/db" +log: + level: "debug" + format: "json" +api: + enabled: false +` + + configPath := filepath.Join(tmpDir, "config.yaml") + err = os.WriteFile(configPath, []byte(configFileContent), 0644) + is.NoErr(err) + + tests := []struct { + name string + flags *RootFlags + configFile string + envVars map[string]string + assertFunc func(*isT.I, conduit.Config) + }{ + { + name: "default values only", + flags: &RootFlags{ + ConduitConfigPath: "nonexistent.yaml", + }, + assertFunc: func(is *isT.I, cfg conduit.Config) { + defaultCfg := conduit.DefaultConfigWithBasePath(tmpDir) + is.Equal(cfg.DB.Type, defaultCfg.DB.Type) + is.Equal(cfg.Log.Level, defaultCfg.Log.Level) + is.Equal(cfg.API.Enabled, defaultCfg.API.Enabled) + }, + }, + { + name: "config file overrides defaults", + flags: &RootFlags{ + ConduitConfigPath: configPath, + }, + assertFunc: func(is *isT.I, cfg conduit.Config) { + is.Equal(cfg.DB.Type, "sqlite") + is.Equal(cfg.DB.SQLite.Path, "/custom/path/db") + is.Equal(cfg.Log.Level, "debug") + is.Equal(cfg.Log.Format, "json") + is.Equal(cfg.API.Enabled, false) + }, + }, + { + name: "env vars override config file", + flags: &RootFlags{ + ConduitConfigPath: configPath, + }, + envVars: map[string]string{ + "CONDUIT_DB_TYPE": "postgres", + "CONDUIT_LOG_LEVEL": "warn", + "CONDUIT_API_ENABLED": "true", + }, + assertFunc: func(is *isT.I, cfg conduit.Config) { + is.Equal(cfg.DB.Type, "postgres") + is.Equal(cfg.Log.Level, "warn") + is.Equal(cfg.API.Enabled, true) + // Config file values that weren't overridden should remain + is.Equal(cfg.Log.Format, "json") + }, + }, + { + name: "flags override everything", + flags: &RootFlags{ + ConduitConfigPath: configPath, + Config: conduit.Config{ + DB: conduit.Config{}.DB, + Log: conduit.Config{}.Log, + API: conduit.Config{}.API, + }, + }, + envVars: map[string]string{ + "CONDUIT_DB_TYPE": "postgres", + "CONDUIT_LOG_LEVEL": "warn", + "CONDUIT_API_ENABLED": "false", + }, + assertFunc: func(is *isT.I, cfg conduit.Config) { + is.Equal(cfg.DB.Type, "badger") + is.Equal(cfg.Log.Level, "error") + is.Equal(cfg.Log.Format, "text") + is.Equal(cfg.API.Enabled, true) + }, + }, + { + name: "partial overrides", + flags: &RootFlags{ + ConduitConfigPath: configPath, + Config: conduit.Config{ + Log: conduit.Config{}.Log, + }, + }, + envVars: map[string]string{ + "CONDUIT_DB_TYPE": "postgres", + }, + assertFunc: func(is *isT.I, cfg conduit.Config) { + is.Equal(cfg.DB.Type, "postgres") + is.Equal(cfg.Log.Level, "error") + is.Equal(cfg.Log.Format, "json") + is.Equal(cfg.API.Enabled, false) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + is := isT.New(t) + + os.Clearenv() + + for k, v := range tt.envVars { + os.Setenv(k, v) + } + defer func() { + for k := range tt.envVars { + os.Unsetenv(k) + } + }() + + c := &RootCommand{ + flags: *tt.flags, + } + + err := c.updateConfig() + is.NoErr(err) + + tt.assertFunc(is, c.cfg) + }) + } +} From 243fcb9e9b11d77e5068e0a2f2707754917a65da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 2 Dec 2024 16:11:08 +0100 Subject: [PATCH 22/22] fixes --- cmd/conduit/root/root.go | 44 ----------------------- cmd/conduit/root/root_test.go | 31 +++++++++++----- pkg/conduit/config.go | 68 ++++++++++++++++++----------------- 3 files changed, 59 insertions(+), 84 deletions(-) diff --git a/cmd/conduit/root/root.go b/cmd/conduit/root/root.go index 0b28d3fd3..f68c9a58e 100644 --- a/cmd/conduit/root/root.go +++ b/cmd/conduit/root/root.go @@ -54,50 +54,6 @@ type RootCommand struct { cfg conduit.Config } -func (c *RootCommand) updateFlagValuesFromConfig() { - // Map database configuration - c.flags.DB.Type = c.cfg.DB.Type - c.flags.DB.Postgres.ConnectionString = c.cfg.DB.Postgres.ConnectionString - c.flags.DB.Postgres.Table = c.cfg.DB.Postgres.Table - c.flags.DB.SQLite.Table = c.cfg.DB.SQLite.Table - - // Map API configuration - c.flags.API.Enabled = c.cfg.API.Enabled - c.flags.API.HTTP.Address = c.cfg.API.HTTP.Address - c.flags.API.GRPC.Address = c.cfg.API.GRPC.Address - - // Map logging configuration - c.flags.Log.Level = c.cfg.Log.Level - c.flags.Log.Format = c.cfg.Log.Format - - // Map pipeline configuration - c.flags.Pipelines.ExitOnDegraded = c.cfg.Pipelines.ExitOnDegraded - c.flags.Pipelines.ErrorRecovery.MinDelay = c.cfg.Pipelines.ErrorRecovery.MinDelay - c.flags.Pipelines.ErrorRecovery.MaxDelay = c.cfg.Pipelines.ErrorRecovery.MaxDelay - c.flags.Pipelines.ErrorRecovery.BackoffFactor = c.cfg.Pipelines.ErrorRecovery.BackoffFactor - c.flags.Pipelines.ErrorRecovery.MaxRetries = c.cfg.Pipelines.ErrorRecovery.MaxRetries - c.flags.Pipelines.ErrorRecovery.MaxRetriesWindow = c.cfg.Pipelines.ErrorRecovery.MaxRetriesWindow - - // Map schema registry configuration - c.flags.SchemaRegistry.Type = c.cfg.SchemaRegistry.Type - c.flags.SchemaRegistry.Confluent.ConnectionString = c.cfg.SchemaRegistry.Confluent.ConnectionString - - // Map preview features - c.flags.Preview.PipelineArchV2 = c.cfg.Preview.PipelineArchV2 - - // Map development profiling - c.flags.Dev.CPUProfile = c.cfg.Dev.CPUProfile - c.flags.Dev.MemProfile = c.cfg.Dev.MemProfile - c.flags.Dev.BlockProfile = c.cfg.Dev.BlockProfile - - // Update paths - c.flags.DB.SQLite.Path = c.cfg.DB.SQLite.Path - c.flags.DB.Badger.Path = c.cfg.DB.Badger.Path - c.flags.Pipelines.Path = c.cfg.Pipelines.Path - c.flags.Connectors.Path = c.cfg.Connectors.Path - c.flags.Processors.Path = c.cfg.Processors.Path -} - func (c *RootCommand) updateConfig() error { v := viper.New() diff --git a/cmd/conduit/root/root_test.go b/cmd/conduit/root/root_test.go index 5a653aec6..4ad0127e9 100644 --- a/cmd/conduit/root/root_test.go +++ b/cmd/conduit/root/root_test.go @@ -110,6 +110,8 @@ api: err = os.WriteFile(configPath, []byte(configFileContent), 0644) is.NoErr(err) + defaultCfg := conduit.DefaultConfigWithBasePath(tmpDir) + tests := []struct { name string flags *RootFlags @@ -121,9 +123,13 @@ api: name: "default values only", flags: &RootFlags{ ConduitConfigPath: "nonexistent.yaml", + Config: conduit.Config{ + DB: conduit.DefaultConfig().DB, + Log: conduit.DefaultConfig().Log, + API: conduit.DefaultConfig().API, + }, }, assertFunc: func(is *isT.I, cfg conduit.Config) { - defaultCfg := conduit.DefaultConfigWithBasePath(tmpDir) is.Equal(cfg.DB.Type, defaultCfg.DB.Type) is.Equal(cfg.Log.Level, defaultCfg.Log.Level) is.Equal(cfg.API.Enabled, defaultCfg.API.Enabled) @@ -165,9 +171,16 @@ api: flags: &RootFlags{ ConduitConfigPath: configPath, Config: conduit.Config{ - DB: conduit.Config{}.DB, - Log: conduit.Config{}.Log, - API: conduit.Config{}.API, + DB: conduit.ConfigDB{ + Type: "sqlite", + }, + Log: conduit.ConfigLog{ + Level: "error", + Format: "text", + }, + API: conduit.ConfigAPI{ + Enabled: true, + }, }, }, envVars: map[string]string{ @@ -176,7 +189,7 @@ api: "CONDUIT_API_ENABLED": "false", }, assertFunc: func(is *isT.I, cfg conduit.Config) { - is.Equal(cfg.DB.Type, "badger") + is.Equal(cfg.DB.Type, "sqlite") is.Equal(cfg.Log.Level, "error") is.Equal(cfg.Log.Format, "text") is.Equal(cfg.API.Enabled, true) @@ -187,7 +200,7 @@ api: flags: &RootFlags{ ConduitConfigPath: configPath, Config: conduit.Config{ - Log: conduit.Config{}.Log, + Log: conduit.ConfigLog{Level: "warn"}, }, }, envVars: map[string]string{ @@ -195,9 +208,9 @@ api: }, assertFunc: func(is *isT.I, cfg conduit.Config) { is.Equal(cfg.DB.Type, "postgres") - is.Equal(cfg.Log.Level, "error") + is.Equal(cfg.Log.Level, "warn") is.Equal(cfg.Log.Format, "json") - is.Equal(cfg.API.Enabled, false) + is.Equal(cfg.API.Enabled, defaultCfg.API.Enabled) }, }, } @@ -221,6 +234,8 @@ api: flags: *tt.flags, } + c.cfg = conduit.DefaultConfigWithBasePath(tmpDir) + err := c.updateConfig() is.NoErr(err) diff --git a/pkg/conduit/config.go b/pkg/conduit/config.go index afceb9334..b9523d4fd 100644 --- a/pkg/conduit/config.go +++ b/pkg/conduit/config.go @@ -39,42 +39,46 @@ const ( SchemaRegistryTypeBuiltin = "builtin" ) -// Config holds all configurable values for Conduit. -type Config struct { - DB struct { - // When Driver is specified it takes precedence over other DB related - // fields. - Driver database.DB - - Type string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite"` - Badger struct { - Path string `long:"db.badger.path" usage:"path to badger DB"` - } - Postgres struct { - ConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format"` - Table string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)"` - } - SQLite struct { - Path string `long:"db.sqlite.path" usage:"path to sqlite3 DB"` - Table string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)"` - } +type ConfigDB struct { + // When Driver is specified it takes precedence over other DB related + // fields. + Driver database.DB + + Type string `long:"db.type" usage:"database type; accepts badger,postgres,inmemory,sqlite"` + Badger struct { + Path string `long:"db.badger.path" usage:"path to badger DB"` } - - API struct { - Enabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API"` - HTTP struct { - Address string `long:"http.address" usage:"address for serving the HTTP API"` - } - GRPC struct { - Address string `long:"grpc.address" usage:"address for serving the gRPC API"` - } + Postgres struct { + ConnectionString string `long:"db.postgres.connection-string" usage:"postgres connection string, may be a database URL or in PostgreSQL keyword/value format"` + Table string `long:"db.postgres.table" usage:"postgres table in which to store data (will be created if it does not exist)"` + } + SQLite struct { + Path string `long:"db.sqlite.path" usage:"path to sqlite3 DB"` + Table string `long:"db.sqlite.table" usage:"sqlite3 table in which to store data (will be created if it does not exist)"` } +} - Log struct { - NewLogger func(level, format string) log.CtxLogger - Level string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace"` - Format string `long:"log.format" usage:"sets the format of the logging; accepts json, cli"` +type ConfigAPI struct { + Enabled bool `long:"api.enabled" usage:"enable HTTP and gRPC API"` + HTTP struct { + Address string `long:"http.address" usage:"address for serving the HTTP API"` + } + GRPC struct { + Address string `long:"grpc.address" usage:"address for serving the gRPC API"` } +} + +type ConfigLog struct { + NewLogger func(level, format string) log.CtxLogger + Level string `long:"log.level" usage:"sets logging level; accepts debug, info, warn, error, trace"` + Format string `long:"log.format" usage:"sets the format of the logging; accepts json, cli"` +} + +// Config holds all configurable values for Conduit. +type Config struct { + DB ConfigDB + API ConfigAPI + Log ConfigLog Connectors struct { Path string `long:"connectors.path" usage:"path to standalone connectors' directory"`