Skip to content

Commit

Permalink
helm prometheus on dind (#33)
Browse files Browse the repository at this point in the history
  • Loading branch information
gainsley authored Feb 4, 2019
1 parent 7beacb0 commit 981ccb3
Show file tree
Hide file tree
Showing 6 changed files with 157 additions and 64 deletions.
7 changes: 7 additions & 0 deletions mexos/dns.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,13 @@ func KubePatchServiceLocal(servicename string, ipaddr string) error {
return nil
}

// TODO: This function and createAppDNS share a lot of duplicate code,
// but are subtly different. It'd be good to consolidate and remove
// duplicate code and highlight what the different use cases are,
// since it's not clear when to use one or the other.
// This should be easier to consolidate now that kubeParam can issue
// commands locally for DIND or other cases.
// Same for KubeDeleteDNSRecords and deleteAppDNS.
func KubeAddDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error {
log.DebugLog(log.DebugLevelMexos, "adding dns records for kubenernets app", "name", mf.Metadata.Name)
rootLBIPaddr, err := GetServerIPAddr(mf, mf.Values.Network.External, rootLB.Name)
Expand Down
70 changes: 52 additions & 18 deletions mexos/helm.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package mexos

import (
"fmt"
"regexp"

"github.com/mobiledgex/edge-cloud/log"
)
Expand All @@ -22,14 +23,26 @@ func DeleteHelmAppManifest(mf *Manifest) error {
if err != nil {
return err
}
// remove DNS entries
if err = KubeDeleteDNSRecords(rootLB, mf, kp); err != nil {
log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err)
}
// remove Security rules
if err = DeleteProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil {
log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err)
if IsLocalDIND(mf) {
// remove DNS entries
kconf, err := GetKconf(mf, false)
if err != nil {
return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err)
}
if err = deleteAppDNS(mf, kconf); err != nil {
log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err)
}
} else {
// remove DNS entries
if err = KubeDeleteDNSRecords(rootLB, mf, kp); err != nil {
log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err)
}
// remove Security rules
if err = DeleteProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil {
log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err)
}
}

cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, mf.Metadata.Name)
out, err := kp.client.Output(cmd)
if err != nil {
Expand Down Expand Up @@ -85,22 +98,43 @@ func CreateHelmAppManifest(mf *Manifest) error {
log.DebugLog(log.DebugLevelMexos, "helm tiller initialized")
}

cmd = fmt.Sprintf("%s helm install %s --name %s", kp.kubeconfig, mf.Spec.Image, mf.Metadata.Name)
helmOpts := ""
// XXX This gets helm's prometheus able to query kubelet metrics.
// This can be removed once Lev passes in an option in the yaml to
// set the helm command line options.
prom, err := regexp.MatchString("prometheus", mf.Metadata.Name)
if err == nil && prom {
log.DebugLog(log.DebugLevelMexos, "setting helm prometheus option")
helmOpts = "--set kubelet.serviceMonitor.https=true"
}
cmd = fmt.Sprintf("%s helm install %s --name %s %s", kp.kubeconfig, mf.Spec.Image, mf.Metadata.Name, helmOpts)
out, err = kp.client.Output(cmd)
if err != nil {
return fmt.Errorf("error deploying helm chart, %s, %s, %v", cmd, out, err)
}
log.DebugLog(log.DebugLevelMexos, "applied helm chart")
// Add security rules
if err = AddProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err)
return err
}
log.DebugLog(log.DebugLevelMexos, "add spec ports", "ports", mf.Spec.Ports)
// Add DNS Zone
if err = KubeAddDNSRecords(rootLB, mf, kp); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err)
return err
if IsLocalDIND(mf) {
kconf, err := GetKconf(mf, false)
if err != nil {
return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err)
}
// Add DNS Zone
if err = createAppDNS(mf, kconf); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err)
return err
}
} else {
// Add security rules
if err = AddProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err)
return err
}
log.DebugLog(log.DebugLevelMexos, "add spec ports", "ports", mf.Spec.Ports)
// Add DNS Zone
if err = KubeAddDNSRecords(rootLB, mf, kp); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err)
return err
}
}
return nil
}
38 changes: 4 additions & 34 deletions mexos/kubectl.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@ import (
"time"

sh "github.com/codeskyblue/go-sh"
"github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare"
"github.com/mobiledgex/edge-cloud/cloudcommon"
"github.com/mobiledgex/edge-cloud/log"
)

Expand Down Expand Up @@ -91,9 +89,9 @@ func runKubectlCreateApp(mf *Manifest, kubeManifest string) error {
if err != nil {
return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err)
}
out, err := sh.Command("kubectl", "create", "-f", kfile, "--kubeconfig="+kconf).Output()
out, err := sh.Command("kubectl", "create", "-f", kfile, "--kubeconfig="+kconf).CombinedOutput()
if err != nil {
return fmt.Errorf("error creating app, %s, %v, %v", out, err, mf)
return fmt.Errorf("error creating app, %s, %v, %v, %s", out, err, mf, kubeManifest)
}
err = createAppDNS(mf, kconf)
if err != nil {
Expand Down Expand Up @@ -164,12 +162,6 @@ func getSvcNames(name string, kconf string) ([]string, error) {
}

func runKubectlDeleteApp(mf *Manifest, kubeManifest string) error {
if err := CheckCredentialsCF(mf); err != nil {
return err
}
if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil {
return fmt.Errorf("cannot init cloudflare api, %v", err)
}
kconf, err := GetKconf(mf, false)
if err != nil {
return fmt.Errorf("error deleting app due to kconf missing, %v, %v", mf, err)
Expand All @@ -180,35 +172,13 @@ func runKubectlDeleteApp(mf *Manifest, kubeManifest string) error {
return err
}
defer os.Remove(kfile)
serviceNames, err := getSvcNames(mf.Metadata.Name, kconf)
if err != nil {
return err
}
if len(serviceNames) < 1 {
return fmt.Errorf("no service names starting with %s", mf.Metadata.Name)
}
out, err := sh.Command("kubectl", "delete", "-f", kfile, "--kubeconfig="+kconf).CombinedOutput()
if err != nil {
return fmt.Errorf("error deleting app, %s, %v, %v", out, mf, err)
}
if mf.Metadata.DNSZone == "" {
return fmt.Errorf("missing dns zone, metadata %v", mf.Metadata)
}
fqdnBase := uri2fqdn(mf.Spec.URI)
dr, err := cloudflare.GetDNSRecords(mf.Metadata.DNSZone)
err = deleteAppDNS(mf, kconf)
if err != nil {
return fmt.Errorf("cannot get dns records for %s, %v", mf.Metadata.DNSZone, err)
}
for _, sn := range serviceNames {
fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase)
for _, d := range dr {
if d.Type == "A" && d.Name == fqdn {
if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, d.ID); err != nil {
return fmt.Errorf("cannot delete DNS record, %v", d)
}
log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn)
}
}
return fmt.Errorf("error deleting dns entry for app, %v, %v", err, mf)
}
return nil
}
Expand Down
3 changes: 3 additions & 0 deletions mexos/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ func ValidateKubernetesParameters(mf *Manifest, rootLB *MEXRootLB, clustName str
if mf.Values.Network.External == "" {
return nil, fmt.Errorf("validate kubernetes parameters, missing external network in platform config")
}
if IsLocalDIND(mf) {
return &kubeParam{client: &sshLocal{}}, nil
}
client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser)
if err != nil {
return nil, err
Expand Down
40 changes: 28 additions & 12 deletions mexos/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,14 +159,23 @@ func MEXAppCreateAppManifest(mf *Manifest) error {

if IsLocalDIND(mf) {
masteraddr := dind.GetMasterAddr()
log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind")

if err = AddNginxProxy(mf, "localhost", mf.Metadata.Name, masteraddr, mf.Spec.Ports, dind.GetDockerNetworkName(mf.Values.Cluster.Name)); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports)
return err
if len(mf.Spec.Ports) > 0 {
log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind", "ports", mf.Spec.Ports)
if err = AddNginxProxy(mf, "localhost", mf.Metadata.Name, masteraddr, mf.Spec.Ports, dind.GetDockerNetworkName(mf.Values.Cluster.Name)); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports)
return err
}
}
log.DebugLog(log.DebugLevelMexos, "call runKubectlCreateApp for dind")
err := runKubectlCreateApp(mf, kubeManifest)
var err error
if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes {
err = runKubectlCreateApp(mf, kubeManifest)
} else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm {
err = CreateHelmAppManifest(mf)
} else {
err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType)
}
if err != nil {
log.DebugLog(log.DebugLevelMexos, "error creating dind app", "mf", mf)
return err
Expand Down Expand Up @@ -213,18 +222,25 @@ func MEXAppDeleteAppManifest(mf *Manifest) error {
}
if IsLocalDIND(mf) {
log.DebugLog(log.DebugLevelMexos, "run kubectl delete app for dind")
err := runKubectlDeleteApp(mf, kubeManifest)
var err error
if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes {
err = runKubectlDeleteApp(mf, kubeManifest)
} else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm {
err = DeleteHelmAppManifest(mf)
} else {
err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType)
}
if err != nil {
return err
}

log.DebugLog(log.DebugLevelMexos, "call DeleteNginxProxy for dind")

if err = DeleteNginxProxy(mf, "localhost", mf.Metadata.Name); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", mf.Metadata.Name)
return err
if len(mf.Spec.Ports) > 0 {
log.DebugLog(log.DebugLevelMexos, "call DeleteNginxProxy for dind")
if err = DeleteNginxProxy(mf, "localhost", mf.Metadata.Name); err != nil {
log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", mf.Metadata.Name)
return err
}
}

return nil

}
Expand Down
63 changes: 63 additions & 0 deletions mexos/sshlocal.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package mexos

import (
"fmt"
"io"
"os/exec"
"strings"
)

// Implements nanobox-io's ssh.Client interface, but runs commands locally.
// This is used for kubernetes DIND or other local testing.
type sshLocal struct {
cmd *exec.Cmd
}

// Output returns the output of the command run on the remote host.
func (s *sshLocal) Output(command string) (string, error) {
cmd := exec.Command("sh", "-c", command)
out, err := cmd.CombinedOutput()
return string(out), err
}

// Shell requests a shell from the remote. If an arg is passed, it tries to
// exec them on the server.
func (s *sshLocal) Shell(args ...string) error {
cmd := exec.Command("sh", "-c", strings.Join(args, " "))
return cmd.Run()
}

// Start starts the specified command without waiting for it to finish. You
// have to call the Wait function for that.
//
// The first two io.ReadCloser are the standard output and the standard
// error of the executing command respectively. The returned error follows
// the same logic as in the exec.Cmd.Start function.
func (s *sshLocal) Start(command string) (io.ReadCloser, io.ReadCloser, error) {
cmd := exec.Command("sh", "-c", command)
sout, err := cmd.StdoutPipe()
if err != nil {
return nil, nil, err
}
errout, err := cmd.StderrPipe()
if err != nil {
return nil, nil, err
}
err = cmd.Start()
if err != nil {
return nil, nil, err
}
s.cmd = cmd
return sout, errout, nil
}

// Wait waits for the command started by the Start function to exit. The
// returned error follows the same logic as in the exec.Cmd.Wait function.
func (s *sshLocal) Wait() error {
if s.cmd == nil {
return fmt.Errorf("no command started")
}
err := s.cmd.Wait()
s.cmd = nil
return err
}

0 comments on commit 981ccb3

Please sign in to comment.