Skip to content

Commit

Permalink
add diff command
Browse files Browse the repository at this point in the history
  • Loading branch information
jacktaylorsbg committed Feb 11, 2023
1 parent e2bf6c0 commit 23b4f58
Show file tree
Hide file tree
Showing 5 changed files with 257 additions and 4 deletions.
35 changes: 33 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,37 @@ my-other-connector /another/path/myconnector.json Valid
Validation errors found, skipping the loading of configs and exiting.
```

## Comparing Connector Config (diff)

The `diff` command can be used to show differences between connector json files and what is currently deployed to Kafka Connect.

This can be used before loading connectors to check what changes will be applied.

```
> diff my-connectors/*.json
Unchanged Connectors: 2
my-existing-connector-1
my-existing-connector-2
New Connectors: 2
my-new-connector-1
my-new-connector-2
Changed Connectors: 3
my-example-connector
+ query.suffix: LIMIT 20000
~ poll.interval.ms: 86400000 -> 900000
- numeric.mapping: best_fit
my-custom-query-connector
~ query:
- SELECT * FROM (SELECT * FROM example WHERE status IN ('Created'))
+ SELECT * FROM (SELECT * FROM example WHERE status IN ('Created', 'Removed'))
```
Coloured output helps make the above command more readable.


## Saving and Setting Connector State

Imagine the scenario where you have many connectors running and you need to pause a chunk of them for whatever reason and then want to return to the previous state. Conan can save the current state of all connectors using
Expand All @@ -160,8 +191,8 @@ You'd then pause whichever connectors you need to, e.g
...
Connector 2 db1-table1-connector paused.
Connector 2 db1-table2-connector paused.
Connector 2 db1-table3-connector paused.
Connector 3 db1-table2-connector paused.
Connector 4 db1-table3-connector paused.
```

Expand Down
175 changes: 175 additions & 0 deletions cmd/diff.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
/*
Copyright © 2023 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd

import (
"fmt"
"path/filepath"
"strings"

log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

type DiffResults struct {
NewConnectors []string
ChangedConnectors []DiffResult
UnchangedConnectors []string
}

type DiffResult struct {
ConnectorName string
NewKeys map[string]string
MatchKeys map[string]string
MismatchKeys map[string]MismatchVals
RemovedKeys map[string]string
}

type MismatchVals struct {
Deployed string
File string
}

// diffCmd represents the diff command
var diffCmd = &cobra.Command{
Use: "diff",
Short: "Compare connector config files with what is currently deployed",
Long: `Compare connector config files with what is currently deployed`,
PreRun: toggleDebug,
Run: func(cmd *cobra.Command, args []string) {
if len(args) == 0 {
fmt.Fprintf(cmd.OutOrStdout(), "No args provided. Please provide paths to the configuration files to load e.g > conan diff /conf/conf.json /otherconf/*.json\n")
return
}

files := make([]ConfigFile, 0)

for _, path := range args {
matches, err := filepath.Glob(path)

cobra.CheckErr(err)

if matches == nil {
log.Warn("no files found for arg ", path)
} else {
log.Debug("for arg ", path, " found files ", matches)
for _, file := range matches {
configFile := ConfigFile{FileName: file}
configFile.Read()
files = append(files, configFile)
}
}
}

if len(files) == 0 {
fmt.Fprintf(cmd.OutOrStdout(), "No configuration files found for provided paths.\n")
return
}

var diffResults DiffResults

// loop through connectors and compare their config to what is deployed
for _, file := range files {
log.Debug(file.ConnectorName)

// get the currently deployed config for the connector
resp := GetConnectorConfig(host, port, file.ConnectorName)
log.Debug(resp)
if _, exists := resp["error_code"]; exists {
if message, exists := resp["message"]; exists {
if strings.Contains(message, "not found") {
log.Debug("the connector " + file.ConnectorName + " doesn't exist")
diffResults.NewConnectors = append(diffResults.NewConnectors, file.ConnectorName)
} else {
log.Warn("there was an error getting the deployed connector config for " + file.ConnectorName + " - " + message)
}
} else {
log.Warn("there was an error getting the deployed connector config for " + file.ConnectorName)
}
continue
}

newKeys := make(map[string]string)
matchKeys := make(map[string]string)
mismatchKeys := make(map[string]MismatchVals)
removedKeys := make(map[string]string)

for fileKey, fileVal := range file.Config {

if deployedVal, exists := resp[fileKey]; exists {
if fileVal == deployedVal {
matchKeys[fileKey] = cleanseVal(fileKey, fileVal)
} else {
mismatchKeys[fileKey] = MismatchVals{Deployed: cleanseVal(fileKey, deployedVal), File: cleanseVal(fileKey, fileVal)}
}

} else {
newKeys[fileKey] = cleanseVal(fileKey, fileVal)
}
}

for deployedKey, deployedVal := range resp {
if _, exists := file.Config[deployedKey]; !exists {
removedKeys[deployedKey] = cleanseVal(deployedKey, deployedVal)
}
}
result := DiffResult{
file.ConnectorName,
newKeys,
matchKeys,
mismatchKeys,
removedKeys,
}
if len(result.NewKeys) == 0 && len(result.MismatchKeys) == 0 && len(result.RemovedKeys) == 0 {
// the connector is unchanged
diffResults.UnchangedConnectors = append(diffResults.UnchangedConnectors, result.ConnectorName)
} else {
diffResults.ChangedConnectors = append(diffResults.ChangedConnectors, result)
}
}
log.Debug(diffResults)

err := templates.ExecuteTemplate(cmd.OutOrStdout(), "DiffTemplate", diffResults)
if err != nil {
fmt.Fprintf(cmd.OutOrStdout(), "Error rendering DiffTemplate template %e.\n", err)
return
}
},
}

var keysToHide = []string{"connection.pass", "connection.user", "connection.url", "password"}

func cleanseVal(key string, val string) string {
for _, substr := range keysToHide {
if strings.Contains(key, substr) {
return "***hidden***"
}
}
return val
}
func init() {
rootCmd.AddCommand(diffCmd)

// Here you will define your flags and configuration settings.

// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// diffCmd.PersistentFlags().String("foo", "", "A help for foo")

// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// diffCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
16 changes: 15 additions & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,22 @@ func toggleDebug(cmd *cobra.Command, args []string) {
log.SetLevel(log.DebugLevel)
}
log.SetFormatter(&log.TextFormatter{})
funcs := template.FuncMap{
"Green": func(t string) string {
return Green + t + Reset
},
"Red": func(t string) string {
return Red + t + Reset
},
"Yellow": func(t string) string {
return Yellow + t + Reset
},
"Gray": func(t string) string {
return Gray + t + Reset
},
}

templates = template.Must(template.New("").Parse(defaultTemplates))
templates = template.Must(template.New("").Funcs(funcs).Parse(defaultTemplates))
log.Debug("Loading templates using path ", templatesPath)
templates.ParseGlob(templatesPath)

Expand Down
33 changes: 33 additions & 0 deletions cmd/templates.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,39 @@ VALIDATION: {{ len . }} Connectors
{{ end }}
{{ end }}
{{ define "DiffTemplate" -}}
Unchanged Connectors: {{ len .UnchangedConnectors }}
{{- range $id, $name := .UnchangedConnectors }}
{{ $name }}
{{- end }}
New Connectors: {{ len .NewConnectors }}
{{- range $id, $name := .NewConnectors }}
{{ Green $name }}
{{- end }}
Changed Connectors: {{ len .ChangedConnectors }}
{{- range $id, $diff := .ChangedConnectors }}
{{ $diff.ConnectorName }}
{{- range $key, $val := $diff.NewKeys }}
{{ Green (printf "+ %s: %s" $key $val) }}
{{- end }}
{{- range $key, $val := $diff.MismatchKeys }}
{{- if gt (len $val.Deployed ) 40 }}
{{ Yellow "~" }} {{ $key }}:
{{ Red (printf "- %s" $val.Deployed) }}
{{ Green (printf "+ %s" $val.File) }}
{{- else }}
{{ Yellow "~" }} {{ $key }}: {{ Red $val.Deployed }} -> {{ Green $val.File }}
{{- end }}
{{- end }}
{{- range $key, $val := $diff.RemovedKeys }}
{{ Red (printf "- %s: %s" $key $val) }}
{{- end }}
{{ end }}
Unchanged: {{ len .UnchangedConnectors }}, New: {{ len .NewConnectors }}, Changed: {{ len .ChangedConnectors }}
{{ end }}
{{ define "io.confluent.connect.jdbc.JdbcSourceConnector" }}
{{- if ne (index .Config "tables") "" -}}
Expand Down
2 changes: 1 addition & 1 deletion cmd/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ var Yellow = "\033[33m"
var Blue = "\033[34m"
var Purple = "\033[35m"
var Cyan = "\033[36m"
var Gray = "\033[37m"
var Gray = "\033[38m"
var White = "\033[97m"

func FormatState(state string) string {
Expand Down

0 comments on commit 23b4f58

Please sign in to comment.