From 23b4f58ede6f6f1a83c444d11e60a806f3ca0682 Mon Sep 17 00:00:00 2001 From: jack-tee Date: Sat, 11 Feb 2023 16:07:33 +0000 Subject: [PATCH] add diff command --- README.md | 35 +++++++++- cmd/diff.go | 175 +++++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 16 ++++- cmd/templates.go | 33 +++++++++ cmd/util.go | 2 +- 5 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 cmd/diff.go diff --git a/README.md b/README.md index 13ec0bb..e9d7f7e 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,37 @@ my-other-connector /another/path/myconnector.json Valid Validation errors found, skipping the loading of configs and exiting. ``` +## Comparing Connector Config (diff) + +The `diff` command can be used to show differences between connector json files and what is currently deployed to Kafka Connect. + +This can be used before loading connectors to check what changes will be applied. + +``` +> diff my-connectors/*.json + +Unchanged Connectors: 2 + my-existing-connector-1 + my-existing-connector-2 + +New Connectors: 2 + my-new-connector-1 + my-new-connector-2 + +Changed Connectors: 3 + my-example-connector + + query.suffix: LIMIT 20000 + ~ poll.interval.ms: 86400000 -> 900000 + - numeric.mapping: best_fit + + my-custom-query-connector + ~ query: + - SELECT * FROM (SELECT * FROM example WHERE status IN ('Created')) + + SELECT * FROM (SELECT * FROM example WHERE status IN ('Created', 'Removed')) +``` +Coloured output helps make the above command more readable. + + ## Saving and Setting Connector State Imagine the scenario where you have many connectors running and you need to pause a chunk of them for whatever reason and then want to return to the previous state. Conan can save the current state of all connectors using @@ -160,8 +191,8 @@ You'd then pause whichever connectors you need to, e.g ... Connector 2 db1-table1-connector paused. -Connector 2 db1-table2-connector paused. -Connector 2 db1-table3-connector paused. +Connector 3 db1-table2-connector paused. +Connector 4 db1-table3-connector paused. ``` diff --git a/cmd/diff.go b/cmd/diff.go new file mode 100644 index 0000000..4a7abb8 --- /dev/null +++ b/cmd/diff.go @@ -0,0 +1,175 @@ +/* +Copyright © 2023 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "fmt" + "path/filepath" + "strings" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +type DiffResults struct { + NewConnectors []string + ChangedConnectors []DiffResult + UnchangedConnectors []string +} + +type DiffResult struct { + ConnectorName string + NewKeys map[string]string + MatchKeys map[string]string + MismatchKeys map[string]MismatchVals + RemovedKeys map[string]string +} + +type MismatchVals struct { + Deployed string + File string +} + +// diffCmd represents the diff command +var diffCmd = &cobra.Command{ + Use: "diff", + Short: "Compare connector config files with what is currently deployed", + Long: `Compare connector config files with what is currently deployed`, + PreRun: toggleDebug, + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "No args provided. Please provide paths to the configuration files to load e.g > conan diff /conf/conf.json /otherconf/*.json\n") + return + } + + files := make([]ConfigFile, 0) + + for _, path := range args { + matches, err := filepath.Glob(path) + + cobra.CheckErr(err) + + if matches == nil { + log.Warn("no files found for arg ", path) + } else { + log.Debug("for arg ", path, " found files ", matches) + for _, file := range matches { + configFile := ConfigFile{FileName: file} + configFile.Read() + files = append(files, configFile) + } + } + } + + if len(files) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "No configuration files found for provided paths.\n") + return + } + + var diffResults DiffResults + + // loop through connectors and compare their config to what is deployed + for _, file := range files { + log.Debug(file.ConnectorName) + + // get the currently deployed config for the connector + resp := GetConnectorConfig(host, port, file.ConnectorName) + log.Debug(resp) + if _, exists := resp["error_code"]; exists { + if message, exists := resp["message"]; exists { + if strings.Contains(message, "not found") { + log.Debug("the connector " + file.ConnectorName + " doesn't exist") + diffResults.NewConnectors = append(diffResults.NewConnectors, file.ConnectorName) + } else { + log.Warn("there was an error getting the deployed connector config for " + file.ConnectorName + " - " + message) + } + } else { + log.Warn("there was an error getting the deployed connector config for " + file.ConnectorName) + } + continue + } + + newKeys := make(map[string]string) + matchKeys := make(map[string]string) + mismatchKeys := make(map[string]MismatchVals) + removedKeys := make(map[string]string) + + for fileKey, fileVal := range file.Config { + + if deployedVal, exists := resp[fileKey]; exists { + if fileVal == deployedVal { + matchKeys[fileKey] = cleanseVal(fileKey, fileVal) + } else { + mismatchKeys[fileKey] = MismatchVals{Deployed: cleanseVal(fileKey, deployedVal), File: cleanseVal(fileKey, fileVal)} + } + + } else { + newKeys[fileKey] = cleanseVal(fileKey, fileVal) + } + } + + for deployedKey, deployedVal := range resp { + if _, exists := file.Config[deployedKey]; !exists { + removedKeys[deployedKey] = cleanseVal(deployedKey, deployedVal) + } + } + result := DiffResult{ + file.ConnectorName, + newKeys, + matchKeys, + mismatchKeys, + removedKeys, + } + if len(result.NewKeys) == 0 && len(result.MismatchKeys) == 0 && len(result.RemovedKeys) == 0 { + // the connector is unchanged + diffResults.UnchangedConnectors = append(diffResults.UnchangedConnectors, result.ConnectorName) + } else { + diffResults.ChangedConnectors = append(diffResults.ChangedConnectors, result) + } + } + log.Debug(diffResults) + + err := templates.ExecuteTemplate(cmd.OutOrStdout(), "DiffTemplate", diffResults) + if err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Error rendering DiffTemplate template %e.\n", err) + return + } + }, +} + +var keysToHide = []string{"connection.pass", "connection.user", "connection.url", "password"} + +func cleanseVal(key string, val string) string { + for _, substr := range keysToHide { + if strings.Contains(key, substr) { + return "***hidden***" + } + } + return val +} +func init() { + rootCmd.AddCommand(diffCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // diffCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // diffCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/root.go b/cmd/root.go index 75da248..a89b1dc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -69,8 +69,22 @@ func toggleDebug(cmd *cobra.Command, args []string) { log.SetLevel(log.DebugLevel) } log.SetFormatter(&log.TextFormatter{}) + funcs := template.FuncMap{ + "Green": func(t string) string { + return Green + t + Reset + }, + "Red": func(t string) string { + return Red + t + Reset + }, + "Yellow": func(t string) string { + return Yellow + t + Reset + }, + "Gray": func(t string) string { + return Gray + t + Reset + }, + } - templates = template.Must(template.New("").Parse(defaultTemplates)) + templates = template.Must(template.New("").Funcs(funcs).Parse(defaultTemplates)) log.Debug("Loading templates using path ", templatesPath) templates.ParseGlob(templatesPath) diff --git a/cmd/templates.go b/cmd/templates.go index 64d4a88..e876006 100644 --- a/cmd/templates.go +++ b/cmd/templates.go @@ -39,6 +39,39 @@ VALIDATION: {{ len . }} Connectors {{ end }} {{ end }} +{{ define "DiffTemplate" -}} +Unchanged Connectors: {{ len .UnchangedConnectors }} +{{- range $id, $name := .UnchangedConnectors }} + {{ $name }} +{{- end }} + +New Connectors: {{ len .NewConnectors }} +{{- range $id, $name := .NewConnectors }} + {{ Green $name }} +{{- end }} + +Changed Connectors: {{ len .ChangedConnectors }} +{{- range $id, $diff := .ChangedConnectors }} + {{ $diff.ConnectorName }} + {{- range $key, $val := $diff.NewKeys }} + {{ Green (printf "+ %s: %s" $key $val) }} + {{- end }} + {{- range $key, $val := $diff.MismatchKeys }} + {{- if gt (len $val.Deployed ) 40 }} + {{ Yellow "~" }} {{ $key }}: + {{ Red (printf "- %s" $val.Deployed) }} + {{ Green (printf "+ %s" $val.File) }} + {{- else }} + {{ Yellow "~" }} {{ $key }}: {{ Red $val.Deployed }} -> {{ Green $val.File }} + {{- end }} + {{- end }} + {{- range $key, $val := $diff.RemovedKeys }} + {{ Red (printf "- %s: %s" $key $val) }} + {{- end }} +{{ end }} +Unchanged: {{ len .UnchangedConnectors }}, New: {{ len .NewConnectors }}, Changed: {{ len .ChangedConnectors }} +{{ end }} + {{ define "io.confluent.connect.jdbc.JdbcSourceConnector" }} {{- if ne (index .Config "tables") "" -}} diff --git a/cmd/util.go b/cmd/util.go index 47c00cb..62e27bc 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -133,7 +133,7 @@ var Yellow = "\033[33m" var Blue = "\033[34m" var Purple = "\033[35m" var Cyan = "\033[36m" -var Gray = "\033[37m" +var Gray = "\033[38m" var White = "\033[97m" func FormatState(state string) string {