Added cosign plugin in the dynamic buildkite template (#20)

* In addition to command line options, a dynamic buildkite template will provide config.yaml for overrides * updte main.go * updte main.go * Added cosign plugin in the dynamic buildkite template * update README.md * update pipeline test.go * update output to default stderr * deleted github util.go file * update * updated the code as per the review notes * adding co-sign * updated bug fix * version fix and update * update README and cosign file * Update varname correctly * added docker metatadata plugin * update plugin step * update test file * update conf.yaml keyless is false * add ssm plugin config * update pugin and test file * updated keyless for cosign * Added docker build plugin into the dynamic template * updated conf file * updated conf file * updated conf file * changed the order of the plugins * changed the order of the plugins and conf file
equinixmetal-buildkite · Jan 25, 2024 · 258055a · 258055a
1 parent 95a423d
commit 258055a
Show file tree

Hide file tree

Showing 23 changed files with 736 additions and 291 deletions.
diff --git a/README.md b/README.md
@@ -19,44 +19,45 @@ make deb
 ```
 
 # Usage
-Here's how you can generate the trivy plugin template
+Here's how you can generate the buildkite template
 ```
-$ ./dynamic-buildkite-template trivy --version=v1.18.2 --skip-files="cosign.key"
+$ ./dynamic-buildkite-template
 steps:
   - command: ls
     plugins:
-      - equinixmetal-buildkite/trivy#v1.18.2:
+      - equinixmetal-buildkite/cosign#v0.1.0:
+          image: ghcr.io/my-project/my-image:latest
+          keyless-config:
+            fulcio-url: https://fulcio.sigstore.dev
+            rekor-url: https://rekor.sigstore.dev
+          cosign-version: v0.1.0
+      - equinixmetal-buildkite/trivy#v1.18.3:
           timeout : 5m0s
           severity: HIGH,CRITICAL
           ignore-unfixed: true
           security-checks: vuln,config
-          skip-files: 'cosign.key'
 ```
 ## Configuration and Overrides
-* Configurations are stored in `conf.yaml` and it has default values.
-* Configurations from the file `conf.yaml` can be overridden by command line flags as this example:
-  Using default configs
-  ```
-  $ go run main.go trivy
-  steps:
-    - command: ls
-      plugins:
-        - equinixmetal-buildkite/trivy#v1.18.2:
-            timeout : 5m0s
-            severity: HIGH,CRITICAL
-            ignore-unfixed: true
-            security-checks: vuln,config
-  ```
+* Configurations are stored in `resources/config/conf.yaml` and it has default values.
+* Configurations from the file `resources/config/conf.yaml` can be overridden by command line flags by using the yaml configuration path as below:
+```
+$ ./dynamic-buildkite-template --overrides plugins.trivy.skip-files="x.txt,y.txt" --overrides plugins.cosign.keyless=false
+steps:
+  - command: ls
+    plugins:
+      - equinixmetal-buildkite/cosign#v0.1.0:
+          image: ghcr.io/my-project/my-image:latest
+          keyless : false
+          keyed-config:
+            key: sample-key
+          cosign-version: v0.1.0
+      - equinixmetal-buildkite/trivy#v1.18.3:
+          timeout : 5m0s
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          security-checks: vuln,config
+          skip-files: 'x.txt,y.txt'
+```
+If you notice you can provide multiple `--overrides` flags and this would in turn collate to a `map[string]string` being passed to the program. The keys in override are in the yaml path format. So for a given config override you can check the path hierarchy in the `conf.yaml` and mention the override accordingly.
 
-  Using command line flags to override timeout
-  ```
-  $ go run main.go trivy --timeout=7m15s
-  steps:
-    - command: ls
-      plugins:
-        - equinixmetal-buildkite/trivy#v1.18.2:
-            timeout : 7m15s
-            severity: HIGH,CRITICAL
-            ignore-unfixed: true
-            security-checks: vuln,config
-  ```
+For long term config changes, it's suggested to update the `conf.yaml` file itself.
diff --git a/cmd/cosign.go b/cmd/cosign.go
@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"dynamic-buildkite-template/generator"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/viper"
+)
+
+var (
+	cosignPluginConfig generator.CosignPluginConfig
+)
+
+// LoadCosignConfigs loads cosign plugin configuration from conf.yaml using "plugins.cosign" key
+func LoadCosignConfigs() {
+	// load from config
+	s := viper.Sub("plugins.cosign")
+	if s == nil {
+		log.Warn("Cosign Plugin configuration not found in the config file. .")
+		return
+	}
+
+	log.Info("Cosign plugin found in the config file")
+
+	err := s.Unmarshal(&cosignPluginConfig) // unmarshal to the cosignPluginConfig object
+	if err != nil {
+		log.Error("Error unmarshalling cosign plugin from config file", err)
+		return
+	}
+
+	// fetch latest cosign plugin version, if not defined in the config
+	if strings.TrimSpace(cosignPluginConfig.CosignVersion) == "" {
+		cosignPluginConfig.CosignVersion = GetLatestPluginTag("cosign-buildkite-plugin")
+	}
+	g.CosignConfig = cosignPluginConfig
+	// mark cosign plugin as enabled
+	g.CosignPluginEnabled = true
+}
diff --git a/cmd/docker-build.go b/cmd/docker-build.go
@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"dynamic-buildkite-template/generator"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/viper"
+)
+
+var (
+	dockerBuildPluginConfig generator.DockerBuildConfig
+)
+
+// LoadDockerBuildConfigs loads docker build plugin configuration from conf.yaml using "plugins.docker-build" key
+func LoadDockerBuildConfigs() {
+	// load from config
+	s := viper.Sub("plugins.docker-build")
+	if s == nil {
+		log.Warn("Docker Build Plugin configuration not found in the config file. .")
+		return
+	}
+
+	log.Info("Docker Build plugin found in the config file")
+
+	err := s.Unmarshal(&dockerBuildPluginConfig) // unmarshal to the dockerBuildPluginConfig object
+	if err != nil {
+		log.Error("Error unmarshalling docker plugin from config file", err)
+		return
+	}
+
+	// fetch latest docker build plugin version, if not defined in the config
+	if strings.TrimSpace(dockerBuildPluginConfig.Version) == "" {
+		dockerBuildPluginConfig.Version = GetLatestPluginTag("docker-build-buildkite-plugin")
+	}
+	g.DockerBuildConfig = dockerBuildPluginConfig
+	// mark docker build plugin as enabled
+	g.DockerBuildPluginEnabled = true
+}
diff --git a/cmd/docker-metadata.go b/cmd/docker-metadata.go
@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"dynamic-buildkite-template/generator"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/viper"
+)
+
+var (
+	dockermetadaPluginConfig generator.DockerMetadataPluginConfig
+)
+
+// LoadDockerMetaDataConfigs loads docker metadata plugin configuration from conf.yaml using "plugins.dockermetadata" key
+func LoadDockerMetaDataConfigs() {
+	// load from config
+	s := viper.Sub("plugins.docker-metadata")
+	if s == nil {
+		log.Warn("docker-metadata Plugin configuration not found in the config file. .")
+		return
+	}
+
+	log.Info("docker-metadata plugin found in the config file")
+
+	err := s.Unmarshal(&dockermetadaPluginConfig) // unmarshal to the dockermetadataPluginConfig object
+	if err != nil {
+		log.Error("Error unmarshalling docker-metadata plugin from config file", err)
+		return
+	}
+
+	// fetch latest docker-metadata plugin version, if not defined in the config
+	if strings.TrimSpace(dockermetadaPluginConfig.Version) == "" {
+		dockermetadaPluginConfig.Version = GetLatestPluginTag("docker-metadata-buildkite-plugin")
+	}
+	g.DockerMetadataConfig = dockermetadaPluginConfig
+	// mark docker-metadata plugin as enabled
+	g.DockerMetadataPluginEnabled = true
+}
diff --git a/cmd/generate.go b/cmd/generate.go
@@ -3,10 +3,13 @@ package cmd
 import (
 	"dynamic-buildkite-template/config"
 	"dynamic-buildkite-template/generator"
+	"dynamic-buildkite-template/util"
 	"fmt"
+	"os"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 var (
@@ -15,6 +18,14 @@ var (
 	defaultConfigFilePath = "conf.yaml"
 )
 
+func init() {
+	cobra.OnInitialize(initConfig)
+	// set the flag for passing overrides
+	generateCmd.Flags().StringToString("overrides", nil, `pass the overrides in the maps syntax as --overrides plugins.trivy.skip-files="x.txt,y.txt" --overrides plugins.cosign.keyless=false`)
+
+	generateCmd.PersistentFlags().StringVar(&ConfigFilePath, "config", "", fmt.Sprintf("config file path (default %q)", defaultConfigFilePath))
+}
+
 var generateCmd = &cobra.Command{
 	Use:   "generate",
 	Short: "Generates plugin step for the provided plugins with configurations",
@@ -23,14 +34,26 @@ Usage of dynamic-buildkite-template
 This Program generates step for the provided plugins with configurations
 	`,
 	Run: func(cmd *cobra.Command, args []string) {
+		// check for overrides
+		ParseOverrides(g, cmd)
+		// load trivy plugin config
+		LoadTrivyConfigs()
+		// load cosign plugin config
+		LoadCosignConfigs()
+		// load docker-metadata plugin config
+		LoadDockerMetaDataConfigs()
+		// load docker build plugin config
+		LoadDockerBuildConfigs()
+		// load SSM plugin
+		LoadSSMDataConfigs()
+		// generate the build template
+		err := generator.GenerateBuildSteps(g, os.Stdout, util.TemplateFilePath)
+		if err != nil {
+			log.Fatalf("Failed to generate build steps. %s", err.Error())
+		}
 	},
 }
 
-func init() {
-	cobra.OnInitialize(initConfig)
-	generateCmd.PersistentFlags().StringVar(&ConfigFilePath, "config", "", fmt.Sprintf("config file path (default %q)", defaultConfigFilePath))
-}
-
 func initConfig() {
 	if ConfigFilePath != "" {
 		log.Debug("config path: ", ConfigFilePath)
@@ -42,10 +65,28 @@ func initConfig() {
 
 	log.Debug("config path:", defaultConfigFilePath)
 	if err := config.LoadConfig(defaultConfigFilePath); err != nil {
-		log.Debug("error while loading the configuration file. Loading the defaults")
+		log.Fatalf("error while loading the configuration file: %s. Configuration file must be present.", defaultConfigFilePath)
 	}
 }
 
 func Execute() error {
 	return generateCmd.Execute()
 }
+
+// ParseOverrides checks for command line flags for the overrides and updates the viper global object
+func ParseOverrides(g generator.Generator, cmd *cobra.Command) {
+	m, err := cmd.Flags().GetStringToString("overrides") // check for --overrides flag for map[string]string
+	if err != nil {
+		log.Warn("No overrides defined. Continuing with defaults defined in the config file.")
+		return
+	}
+	vNew := viper.New() // new viper object for storing overrides
+	for k, v := range m {
+		vNew.Set(k, v)
+	}
+
+	err = viper.MergeConfigMap(vNew.AllSettings()) // merge to global viper object
+	if err != nil {
+		log.Fatalf("Failed merging the configuration with command line overrides. %s", err.Error())
+	}
+}
diff --git a/cmd/ssm.go b/cmd/ssm.go
@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"dynamic-buildkite-template/generator"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/viper"
+)
+
+var (
+	ssmPluginConfig generator.SSMPluginConfig
+)
+
+// LoadDockerMetaDataConfigs loads ssm-buildkite plugin configuration from conf.yaml using "plugins.dockermetadata" key
+func LoadSSMDataConfigs() {
+	// load from config
+	s := viper.Sub("plugins.ssm-buildkite-plugin")
+	if s == nil {
+		log.Warn("ssm-buildkite Plugin configuration not found in the config file. .")
+		return
+	}
+
+	log.Info("ssm-buildkite plugin found in the config file")
+
+	err := s.Unmarshal(&ssmPluginConfig) // unmarshal to the dockermetadataPluginConfig object
+	if err != nil {
+		log.Error("Error unmarshalling ssm-buildkite plugin from config file", err)
+		return
+	}
+
+	// fetch latest ssm-buildkite plugin version, if not defined in the config
+	if strings.TrimSpace(ssmPluginConfig.Version) == "" {
+		ssmPluginConfig.Version = GetLatestPluginTag("ssm-buildkite-plugin")
+	}
+	g.SSMConfig = ssmPluginConfig
+	// mark ssm-buildkite plugin as enabled
+	g.SSMPluginEnabled = true
+}