From 981ccb338b9b8dbb451b43676c53b36579fdeada Mon Sep 17 00:00:00 2001 From: Jon <38672162+gainsley@users.noreply.github.com> Date: Mon, 4 Feb 2019 15:27:26 -0800 Subject: [PATCH] helm prometheus on dind (#33) --- mexos/dns.go | 7 +++++ mexos/helm.go | 70 +++++++++++++++++++++++++++++++++------------ mexos/kubectl.go | 38 +++--------------------- mexos/kubernetes.go | 3 ++ mexos/manifest.go | 40 ++++++++++++++++++-------- mexos/sshlocal.go | 63 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 157 insertions(+), 64 deletions(-) create mode 100644 mexos/sshlocal.go diff --git a/mexos/dns.go b/mexos/dns.go index 87dd268fc..a89be4f53 100644 --- a/mexos/dns.go +++ b/mexos/dns.go @@ -150,6 +150,13 @@ func KubePatchServiceLocal(servicename string, ipaddr string) error { return nil } +// TODO: This function and createAppDNS share a lot of duplicate code, +// but are subtly different. It'd be good to consolidate and remove +// duplicate code and highlight what the different use cases are, +// since it's not clear when to use one or the other. +// This should be easier to consolidate now that kubeParam can issue +// commands locally for DIND or other cases. +// Same for KubeDeleteDNSRecords and deleteAppDNS. func KubeAddDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error { log.DebugLog(log.DebugLevelMexos, "adding dns records for kubenernets app", "name", mf.Metadata.Name) rootLBIPaddr, err := GetServerIPAddr(mf, mf.Values.Network.External, rootLB.Name) diff --git a/mexos/helm.go b/mexos/helm.go index 6b0b6a4d2..f138b4ae8 100644 --- a/mexos/helm.go +++ b/mexos/helm.go @@ -2,6 +2,7 @@ package mexos import ( "fmt" + "regexp" "github.com/mobiledgex/edge-cloud/log" ) @@ -22,14 +23,26 @@ func DeleteHelmAppManifest(mf *Manifest) error { if err != nil { return err } - // remove DNS entries - if err = KubeDeleteDNSRecords(rootLB, mf, kp); err != nil { - log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) - } - // remove Security rules - if err = DeleteProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { - log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) + if IsLocalDIND(mf) { + // remove DNS entries + kconf, err := GetKconf(mf, false) + if err != nil { + return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err) + } + if err = deleteAppDNS(mf, kconf); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) + } + } else { + // remove DNS entries + if err = KubeDeleteDNSRecords(rootLB, mf, kp); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) + } + // remove Security rules + if err = DeleteProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) + } } + cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, mf.Metadata.Name) out, err := kp.client.Output(cmd) if err != nil { @@ -85,22 +98,43 @@ func CreateHelmAppManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "helm tiller initialized") } - cmd = fmt.Sprintf("%s helm install %s --name %s", kp.kubeconfig, mf.Spec.Image, mf.Metadata.Name) + helmOpts := "" + // XXX This gets helm's prometheus able to query kubelet metrics. + // This can be removed once Lev passes in an option in the yaml to + // set the helm command line options. + prom, err := regexp.MatchString("prometheus", mf.Metadata.Name) + if err == nil && prom { + log.DebugLog(log.DebugLevelMexos, "setting helm prometheus option") + helmOpts = "--set kubelet.serviceMonitor.https=true" + } + cmd = fmt.Sprintf("%s helm install %s --name %s %s", kp.kubeconfig, mf.Spec.Image, mf.Metadata.Name, helmOpts) out, err = kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deploying helm chart, %s, %s, %v", cmd, out, err) } log.DebugLog(log.DebugLevelMexos, "applied helm chart") - // Add security rules - if err = AddProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) - return err - } - log.DebugLog(log.DebugLevelMexos, "add spec ports", "ports", mf.Spec.Ports) - // Add DNS Zone - if err = KubeAddDNSRecords(rootLB, mf, kp); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) - return err + if IsLocalDIND(mf) { + kconf, err := GetKconf(mf, false) + if err != nil { + return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err) + } + // Add DNS Zone + if err = createAppDNS(mf, kconf); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) + return err + } + } else { + // Add security rules + if err = AddProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "add spec ports", "ports", mf.Spec.Ports) + // Add DNS Zone + if err = KubeAddDNSRecords(rootLB, mf, kp); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) + return err + } } return nil } diff --git a/mexos/kubectl.go b/mexos/kubectl.go index 3e00dd40f..6ce408897 100644 --- a/mexos/kubectl.go +++ b/mexos/kubectl.go @@ -9,8 +9,6 @@ import ( "time" sh "github.com/codeskyblue/go-sh" - "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" - "github.com/mobiledgex/edge-cloud/cloudcommon" "github.com/mobiledgex/edge-cloud/log" ) @@ -91,9 +89,9 @@ func runKubectlCreateApp(mf *Manifest, kubeManifest string) error { if err != nil { return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err) } - out, err := sh.Command("kubectl", "create", "-f", kfile, "--kubeconfig="+kconf).Output() + out, err := sh.Command("kubectl", "create", "-f", kfile, "--kubeconfig="+kconf).CombinedOutput() if err != nil { - return fmt.Errorf("error creating app, %s, %v, %v", out, err, mf) + return fmt.Errorf("error creating app, %s, %v, %v, %s", out, err, mf, kubeManifest) } err = createAppDNS(mf, kconf) if err != nil { @@ -164,12 +162,6 @@ func getSvcNames(name string, kconf string) ([]string, error) { } func runKubectlDeleteApp(mf *Manifest, kubeManifest string) error { - if err := CheckCredentialsCF(mf); err != nil { - return err - } - if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { - return fmt.Errorf("cannot init cloudflare api, %v", err) - } kconf, err := GetKconf(mf, false) if err != nil { return fmt.Errorf("error deleting app due to kconf missing, %v, %v", mf, err) @@ -180,35 +172,13 @@ func runKubectlDeleteApp(mf *Manifest, kubeManifest string) error { return err } defer os.Remove(kfile) - serviceNames, err := getSvcNames(mf.Metadata.Name, kconf) - if err != nil { - return err - } - if len(serviceNames) < 1 { - return fmt.Errorf("no service names starting with %s", mf.Metadata.Name) - } out, err := sh.Command("kubectl", "delete", "-f", kfile, "--kubeconfig="+kconf).CombinedOutput() if err != nil { return fmt.Errorf("error deleting app, %s, %v, %v", out, mf, err) } - if mf.Metadata.DNSZone == "" { - return fmt.Errorf("missing dns zone, metadata %v", mf.Metadata) - } - fqdnBase := uri2fqdn(mf.Spec.URI) - dr, err := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + err = deleteAppDNS(mf, kconf) if err != nil { - return fmt.Errorf("cannot get dns records for %s, %v", mf.Metadata.DNSZone, err) - } - for _, sn := range serviceNames { - fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) - for _, d := range dr { - if d.Type == "A" && d.Name == fqdn { - if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, d.ID); err != nil { - return fmt.Errorf("cannot delete DNS record, %v", d) - } - log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) - } - } + return fmt.Errorf("error deleting dns entry for app, %v, %v", err, mf) } return nil } diff --git a/mexos/kubernetes.go b/mexos/kubernetes.go index ef74e1da3..6c261efab 100644 --- a/mexos/kubernetes.go +++ b/mexos/kubernetes.go @@ -81,6 +81,9 @@ func ValidateKubernetesParameters(mf *Manifest, rootLB *MEXRootLB, clustName str if mf.Values.Network.External == "" { return nil, fmt.Errorf("validate kubernetes parameters, missing external network in platform config") } + if IsLocalDIND(mf) { + return &kubeParam{client: &sshLocal{}}, nil + } client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) if err != nil { return nil, err diff --git a/mexos/manifest.go b/mexos/manifest.go index e7833f26c..a2e183dbe 100644 --- a/mexos/manifest.go +++ b/mexos/manifest.go @@ -159,14 +159,23 @@ func MEXAppCreateAppManifest(mf *Manifest) error { if IsLocalDIND(mf) { masteraddr := dind.GetMasterAddr() - log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind") - if err = AddNginxProxy(mf, "localhost", mf.Metadata.Name, masteraddr, mf.Spec.Ports, dind.GetDockerNetworkName(mf.Values.Cluster.Name)); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports) - return err + if len(mf.Spec.Ports) > 0 { + log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind", "ports", mf.Spec.Ports) + if err = AddNginxProxy(mf, "localhost", mf.Metadata.Name, masteraddr, mf.Spec.Ports, dind.GetDockerNetworkName(mf.Values.Cluster.Name)); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports) + return err + } } log.DebugLog(log.DebugLevelMexos, "call runKubectlCreateApp for dind") - err := runKubectlCreateApp(mf, kubeManifest) + var err error + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + err = runKubectlCreateApp(mf, kubeManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + err = CreateHelmAppManifest(mf) + } else { + err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) + } if err != nil { log.DebugLog(log.DebugLevelMexos, "error creating dind app", "mf", mf) return err @@ -213,18 +222,25 @@ func MEXAppDeleteAppManifest(mf *Manifest) error { } if IsLocalDIND(mf) { log.DebugLog(log.DebugLevelMexos, "run kubectl delete app for dind") - err := runKubectlDeleteApp(mf, kubeManifest) + var err error + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + err = runKubectlDeleteApp(mf, kubeManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + err = DeleteHelmAppManifest(mf) + } else { + err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) + } if err != nil { return err } - log.DebugLog(log.DebugLevelMexos, "call DeleteNginxProxy for dind") - - if err = DeleteNginxProxy(mf, "localhost", mf.Metadata.Name); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", mf.Metadata.Name) - return err + if len(mf.Spec.Ports) > 0 { + log.DebugLog(log.DebugLevelMexos, "call DeleteNginxProxy for dind") + if err = DeleteNginxProxy(mf, "localhost", mf.Metadata.Name); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", mf.Metadata.Name) + return err + } } - return nil } diff --git a/mexos/sshlocal.go b/mexos/sshlocal.go new file mode 100644 index 000000000..ea687388e --- /dev/null +++ b/mexos/sshlocal.go @@ -0,0 +1,63 @@ +package mexos + +import ( + "fmt" + "io" + "os/exec" + "strings" +) + +// Implements nanobox-io's ssh.Client interface, but runs commands locally. +// This is used for kubernetes DIND or other local testing. +type sshLocal struct { + cmd *exec.Cmd +} + +// Output returns the output of the command run on the remote host. +func (s *sshLocal) Output(command string) (string, error) { + cmd := exec.Command("sh", "-c", command) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// Shell requests a shell from the remote. If an arg is passed, it tries to +// exec them on the server. +func (s *sshLocal) Shell(args ...string) error { + cmd := exec.Command("sh", "-c", strings.Join(args, " ")) + return cmd.Run() +} + +// Start starts the specified command without waiting for it to finish. You +// have to call the Wait function for that. +// +// The first two io.ReadCloser are the standard output and the standard +// error of the executing command respectively. The returned error follows +// the same logic as in the exec.Cmd.Start function. +func (s *sshLocal) Start(command string) (io.ReadCloser, io.ReadCloser, error) { + cmd := exec.Command("sh", "-c", command) + sout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + errout, err := cmd.StderrPipe() + if err != nil { + return nil, nil, err + } + err = cmd.Start() + if err != nil { + return nil, nil, err + } + s.cmd = cmd + return sout, errout, nil +} + +// Wait waits for the command started by the Start function to exit. The +// returned error follows the same logic as in the exec.Cmd.Wait function. +func (s *sshLocal) Wait() error { + if s.cmd == nil { + return fmt.Errorf("no command started") + } + err := s.cmd.Wait() + s.cmd = nil + return err +}