Add SSH capabilities to CLI (#539)

- Add ssh-key and ssh-user flags to CLI to specify a key/user to SSH to containers in the cluster. - Use k8s secrets to store the secret then mount into the containers - Specify env vars so different cloud providers can find the key at the intended location. Fixes: #536 Signed-off-by: John Schnake <[email protected]>
vmware-tanzu · Oct 4, 2018 · e87e9bb · e87e9bb
1 parent c14a483
commit e87e9bb
Show file tree

Hide file tree

Showing 8 changed files with 308 additions and 0 deletions.
diff --git a/cmd/sonobuoy/app/args.go b/cmd/sonobuoy/app/args.go
@@ -191,3 +191,22 @@ func AddImagePullPolicyFlag(policy *ImagePullPolicy, flags *pflag.FlagSet) {
 		fmt.Sprintf("The ImagePullPolicy Sonobuoy should use for the aggregators and workers. Valid options are %s.", strings.Join(ValidPullPolicies(), ", ")),
 	)
 }
+
+// AddSSHKeyPathFlag initialises an SSH key path flag. The SSH key is uploaded
+// as a secret and used in the containers to enable running of E2E tests which
+// require SSH keys to be present.
+func AddSSHKeyPathFlag(path *string, flags *pflag.FlagSet) {
+	flags.StringVar(
+		path, "ssh-key", "",
+		fmt.Sprintf("Path to the private key enabling SSH to cluster nodes."),
+	)
+}
+
+// AddSSHUserFlag initialises an SSH user flag. Used by the container when
+// enabling E2E tests which require SSH.
+func AddSSHUserFlag(user *string, flags *pflag.FlagSet) {
+	flags.StringVar(
+		user, "ssh-user", "",
+		fmt.Sprintf("SSH user for ssh-key."),
+	)
+}
diff --git a/cmd/sonobuoy/app/gen.go b/cmd/sonobuoy/app/gen.go
@@ -40,6 +40,8 @@ type genFlags struct {
 	namespace                   string
 	sonobuoyImage               string
 	kubeConformanceImage        string
+	sshKeyPath                  string
+	sshUser                     string
 	kubeConformanceImageVersion ConformanceImageVersion
 	imagePullPolicy             ImagePullPolicy
 }
@@ -59,6 +61,8 @@ func GenFlagSet(cfg *genFlags, rbac RBACMode, version ConformanceImageVersion) *
 	AddSonobuoyImage(&cfg.sonobuoyImage, genset)
 	AddKubeConformanceImage(&cfg.kubeConformanceImage, genset)
 	AddKubeConformanceImageVersion(&cfg.kubeConformanceImageVersion, genset, version)
+	AddSSHKeyPathFlag(&cfg.sshKeyPath, genset)
+	AddSSHUserFlag(&cfg.sshUser, genset)
 
 	return genset
 }
@@ -118,6 +122,8 @@ func (g *genFlags) Config() (*client.GenConfig, error) {
 		EnableRBAC:           rbacEnabled,
 		ImagePullPolicy:      g.imagePullPolicy.String(),
 		KubeConformanceImage: image,
+		SSHKeyPath:           g.sshKeyPath,
+		SSHUser:              g.sshUser,
 	}, nil
 }
 

diff --git a/pkg/client/gen.go b/pkg/client/gen.go
@@ -18,7 +18,9 @@ package client
 
 import (
 	"bytes"
+	"encoding/base64"
 	"encoding/json"
+	"io/ioutil"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -39,6 +41,8 @@ type templateValues struct {
 	EnableRBAC           bool
 	ImagePullPolicy      string
 	KubeConformanceImage string
+	SSHKey               string
+	SSHUser              string
 }
 
 // GenerateManifest fills in a template with a Sonobuoy config
@@ -48,6 +52,15 @@ func (*SonobuoyClient) GenerateManifest(cfg *GenConfig) ([]byte, error) {
 		return nil, errors.Wrap(err, "couldn't marshall selector")
 	}
 
+	sshKeyData := []byte{}
+	if len(cfg.SSHKeyPath) > 0 {
+		var err error
+		sshKeyData, err = ioutil.ReadFile(cfg.SSHKeyPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to read SSH key file: %v", cfg.SSHKeyPath)
+		}
+	}
+
 	// Template values that are regexps (`E2EFocus` and `E2ESkip`) are
 	// embedded in YAML files using single quotes to remove the need to
 	// escape characters e.g. `\` as they would be if using double quotes.
@@ -67,6 +80,8 @@ func (*SonobuoyClient) GenerateManifest(cfg *GenConfig) ([]byte, error) {
 		EnableRBAC:           cfg.EnableRBAC,
 		ImagePullPolicy:      cfg.ImagePullPolicy,
 		KubeConformanceImage: cfg.KubeConformanceImage,
+		SSHKey:               base64.StdEncoding.EncodeToString(sshKeyData),
+		SSHUser:              cfg.SSHUser,
 	}
 
 	var buf bytes.Buffer

diff --git a/pkg/client/gen_test.go b/pkg/client/gen_test.go
@@ -3,6 +3,9 @@ package client_test
 import (
 	"bytes"
 	"encoding/json"
+	"flag"
+	"io/ioutil"
+	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -14,6 +17,8 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 )
 
+var update = flag.Bool("update", false, "update .golden files")
+
 func TestGenerateManifest(t *testing.T) {
 	tcs := []struct {
 		name     string
@@ -123,3 +128,47 @@ func TestGenerateManifest(t *testing.T) {
 		})
 	}
 }
+
+func TestGenerateManifestSSH(t *testing.T) {
+	tcs := []struct {
+		name       string
+		inputcm    *client.GenConfig
+		goldenFile string
+	}{
+		{
+			name: "Enabling SSH",
+			inputcm: &client.GenConfig{
+				E2EConfig:  &client.E2EConfig{},
+				Config:     &config.Config{},
+				SSHKeyPath: filepath.Join("testdata", "test_ssh.key"),
+				SSHUser:    "ssh-user",
+			},
+			goldenFile: filepath.Join("testdata", "ssh.golden"),
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			sbc, err := client.NewSonobuoyClient(nil, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			manifest, err := sbc.GenerateManifest(tc.inputcm)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if *update {
+				ioutil.WriteFile(tc.goldenFile, manifest, 0666)
+			} else {
+				fileData, err := ioutil.ReadFile(tc.goldenFile)
+				if err != nil {
+					t.Fatalf("Failed to read golden file %v: %v", tc.goldenFile, err)
+				}
+				if !bytes.Equal(fileData, manifest) {
+					t.Errorf("Expected manifest to equal goldenfile: %v but instead got: %v", tc.goldenFile, string(manifest))
+				}
+			}
+		})
+	}
+}
diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go
@@ -46,6 +46,8 @@ type GenConfig struct {
 	EnableRBAC           bool
 	ImagePullPolicy      string
 	KubeConformanceImage string
+	SSHKeyPath           string
+	SSHUser              string
 }
 
 // E2EConfig is the configuration of the E2E tests.

diff --git a/pkg/client/testdata/ssh.golden b/pkg/client/testdata/ssh.golden
@@ -0,0 +1,168 @@
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: 
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    component: sonobuoy
+  name: sonobuoy-serviceaccount
+  namespace: 
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ssh-key
+  namespace: 
+type: Opaque
+data:
+  id_rsa: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDV2dJQkFBS0JnR2gzQ3k2S01SUVhrbVlRRFpYMjZLNUYvalAwUFZPR1RsNHpqTGVnV3dxZlNrenFWNjl0ClMyWi8vSFFmYUJHVlpBVUZadUw5Rjh0RmliTGFod0FNWjIvOFUwK2JCT05jd2dPRnl2ZXd4WGpwQ04xR1JKL3QKMW12Q3NucHY2OUdHZjFDSmd2UU9XbWRJRVliUnF1WWRrZURDaE9YSkQvWEt6SUsweGIvZ3phWHpBZ01CQUFFQwpnWUExc3lRclcxeFpRTGpBQUhhTFpXNDh6N3daeHU2VGRoSG8rTVRZUy9VZzQ3dFZDSTNrbnVGSW9uZ0sxVzR3Cnc4T3psUkJMbE9EYlhEYXBtdzRVcDB1d0xDM2xTQUJtcUVNUnl2TjM3d0kyUU9LelBuWnRVU1VsNlBJdVRMU00KcFc2eFJPc0JxRm9Ta1NyR21ZZHk1eFJOL3RvQlRQZS93NDlCZDl1RUozQ0VHUUpCQUtla0ZyRWVnYUpKK2pzcQpuQ0MwbkNWNGZBRzY4SmtDVWJaZWc1RHZSeTJXWFB2SnNraWlHVUVOOThQbUxzMWsySHdHa3B2aUJWL0pJUVFGCjBOK2UvOFVDUVFDZmhwUnRvUzVVOVA0VlBBSFlwazd0NmNIbzJCdC9aT1lVRkFSL0xFQlpZdGtSRllhMjhTejgKU0JoaEQwKzJqZmllMHNOUVhxS1NoUlJxaVN6N1UzSlhBa0JDVS9BT2dDMTJVcUxKQ2lEZjU5WG9GWFdyRVpYawpWRUNVVVd6b3lXTjJMQW42TFdGMnozd0NlMDVPbWlKbWJjSWh3RS85a1QzakdpY1ArbVVjTlVLcEFrQXZPaDV6CkhrYWpMQ3lVaU1GYkxGRXRxZ2lZTUhKamtyRnl2RUpnaSs4NUIwU1FCRGx2cTVMUDFXQXlTK0FqZ1lCYk5MWlIKOHRKYVlqK2JFcVZZZW1uZEFrQk5HUTVWbFI0ZktVa2Q2cEZ5NW5oYUlWY0EybnF2M3llWWxCSzNWUy90SUd6SApoVDczTkpFYlovU1l6MUhiQmVnbFQrUDE5alVwRGtHclZCS1RJTS9vCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
+---
+apiVersion: v1
+data:
+  config.json: |
+    {"Description":"","UUID":"","Version":"","ResultsDir":"","Resources":null,"Filters":{"Namespaces":"","LabelSelector":""},"Limits":{"PodLogs":{"LimitSize":"","LimitTime":""}},"Server":{"bindaddress":"","bindport":0,"advertiseaddress":"","timeoutseconds":0},"Plugins":null,"PluginSearchPath":null,"Namespace":"","LoadedPlugins":null,"WorkerImage":"","ImagePullPolicy":""}
+kind: ConfigMap
+metadata:
+  labels:
+    component: sonobuoy
+  name: sonobuoy-config-cm
+  namespace: 
+---
+apiVersion: v1
+data:
+  e2e.yaml: |
+    sonobuoy-config:
+      driver: Job
+      plugin-name: e2e
+      result-type: e2e
+    spec:
+      env:
+      - name: E2E_FOCUS
+        value: ''
+      - name: E2E_SKIP
+        value: ''
+      - name: E2E_PARALLEL
+        value: ''
+      - name: LOCAL_SSH_KEY
+        value: 'id_rsa'
+      - name: AWS_SSH_KEY
+        value: '/root/.ssh/id_rsa'
+      - name: KUBE_SSH_KEY
+        value: 'id_rsa'
+      - name: KUBE_SSH_USER
+        value: ssh-user
+      command: ["/run_e2e.sh"]
+      image: 
+      imagePullPolicy: 
+      name: e2e
+      volumeMounts:
+      - mountPath: /tmp/results
+        name: results
+        readOnly: false
+      - mountPath: /root/.ssh
+        name: sshkey-vol
+      tolerations:
+        - operator: "Exists"
+    extra-volumes:
+    - name: sshkey-vol
+      secret:
+        secretName: ssh-key
+        defaultMode: 256
+  systemd-logs.yaml: |
+    sonobuoy-config:
+      driver: DaemonSet
+      plugin-name: systemd-logs
+      result-type: systemd_logs
+    spec:
+      command: ["/bin/sh", "-c", "/get_systemd_logs.sh && sleep 3600"]
+      env:
+      - name: NODE_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: spec.nodeName
+      - name: RESULTS_DIR
+        value: /tmp/results
+      - name: CHROOT_DIR
+        value: /node
+      image: gcr.io/heptio-images/sonobuoy-plugin-systemd-logs:latest
+      imagePullPolicy: 
+      name: sonobuoy-systemd-logs-config
+      securityContext:
+        privileged: true
+      volumeMounts:
+      - mountPath: /tmp/results
+        name: results
+        readOnly: false
+      - mountPath: /node
+        name: root
+        readOnly: false
+kind: ConfigMap
+metadata:
+  labels:
+    component: sonobuoy
+  name: sonobuoy-plugins-cm
+  namespace: 
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    component: sonobuoy
+    run: sonobuoy-master
+    tier: analysis
+  name: sonobuoy
+  namespace: 
+spec:
+  containers:
+  - command:
+    - /bin/bash
+    - -c
+    - /sonobuoy master --no-exit=true -v 3 --logtostderr
+    env:
+    - name: SONOBUOY_ADVERTISE_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    image: 
+    imagePullPolicy: 
+    name: kube-sonobuoy
+    volumeMounts:
+    - mountPath: /etc/sonobuoy
+      name: sonobuoy-config-volume
+    - mountPath: /plugins.d
+      name: sonobuoy-plugins-volume
+    - mountPath: /tmp/sonobuoy
+      name: output-volume
+  restartPolicy: Never
+  serviceAccountName: sonobuoy-serviceaccount
+  volumes:
+  - configMap:
+      name: sonobuoy-config-cm
+    name: sonobuoy-config-volume
+  - configMap:
+      name: sonobuoy-plugins-cm
+    name: sonobuoy-plugins-volume
+  - emptyDir: {}
+    name: output-volume
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    component: sonobuoy
+    run: sonobuoy-master
+  name: sonobuoy-master
+  namespace: 
+spec:
+  ports:
+  - port: 8080
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    run: sonobuoy-master
+  type: ClusterIP
diff --git a/pkg/client/testdata/test_ssh.key b/pkg/client/testdata/test_ssh.key
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICWgIBAAKBgGh3Cy6KMRQXkmYQDZX26K5F/jP0PVOGTl4zjLegWwqfSkzqV69t
+S2Z//HQfaBGVZAUFZuL9F8tFibLahwAMZ2/8U0+bBONcwgOFyvewxXjpCN1GRJ/t
+1mvCsnpv69GGf1CJgvQOWmdIEYbRquYdkeDChOXJD/XKzIK0xb/gzaXzAgMBAAEC
+gYA1syQrW1xZQLjAAHaLZW48z7wZxu6TdhHo+MTYS/Ug47tVCI3knuFIongK1W4w
+w8OzlRBLlODbXDapmw4Up0uwLC3lSABmqEMRyvN37wI2QOKzPnZtUSUl6PIuTLSM
+pW6xROsBqFoSkSrGmYdy5xRN/toBTPe/w49Bd9uEJ3CEGQJBAKekFrEegaJJ+jsq
+nCC0nCV4fAG68JkCUbZeg5DvRy2WXPvJskiiGUEN98PmLs1k2HwGkpviBV/JIQQF
+0N+e/8UCQQCfhpRtoS5U9P4VPAHYpk7t6cHo2Bt/ZOYUFAR/LEBZYtkRFYa28Sz8
+SBhhD0+2jfie0sNQXqKShRRqiSz7U3JXAkBCU/AOgC12UqLJCiDf59XoFXWrEZXk
+VECUUWzoyWN2LAn6LWF2z3wCe05OmiJmbcIhwE/9kT3jGicP+mUcNUKpAkAvOh5z
+HkajLCyUiMFbLFEtqgiYMHJjkrFyvEJgi+85B0SQBDlvq5LP1WAyS+AjgYBbNLZR
+8tJaYj+bEqVYemndAkBNGQ5VlR4fKUkd6pFy5nhaIVcA2nqv3yeYlBK3VS/tIGzH
+hT73NJEbZ/SYz1HbBeglT+P19jUpDkGrVBKTIM/o
+-----END RSA PRIVATE KEY-----