From 6dacb373c0f4160bd495c9e5138b98864b1310bb Mon Sep 17 00:00:00 2001 From: Jim Date: Sat, 9 Feb 2019 12:50:25 -0600 Subject: [PATCH] comments per review so far. Have not restested yet --- mexos/{inst.go => appinst.go} | 51 ++++----- mexos/azure.go | 10 +- mexos/cloudletinfra.go | 19 ++++ mexos/gcloud.go | 6 +- mexos/helm.go | 36 +++---- mexos/kubeadmdind.go | 6 +- mexos/kubeconfig.go | 47 +++++---- mexos/kubectl.go | 24 ++--- mexos/kubernetes.go | 82 +++++++++------ mexos/misc.go | 5 - mexos/oscli.go | 13 +-- mexos/rootlb.go | 1 - mexos/securityrule.go | 13 +-- mexos/templates.go | 189 ---------------------------------- mexos/vault.go | 1 - 15 files changed, 169 insertions(+), 334 deletions(-) rename mexos/{inst.go => appinst.go} (74%) delete mode 100644 mexos/templates.go diff --git a/mexos/inst.go b/mexos/appinst.go similarity index 74% rename from mexos/inst.go rename to mexos/appinst.go index 67900e348..d5d1bd981 100644 --- a/mexos/inst.go +++ b/mexos/appinst.go @@ -14,11 +14,12 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, log.DebugLog(log.DebugLevelMexos, "mex create app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) appDeploymentType := app.Deployment - clusterName := clusterInst.Key.ClusterKey.Name - appName := NormalizeName(app.Key.Name) - operatorName := NormalizeName(appInst.Key.CloudletKey.OperatorKey.Name) - - //TODO values.application.template + var kubeNames KubeNames + err := GetKubeNames(clusterInst, app, appInst, &kubeNames) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "GetKubeNames failed") + return err + } if CloudletIsLocalDIND() { masteraddr := dind.GetMasterAddr() @@ -32,17 +33,17 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, if len(portDetail) > 0 { log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind", "ports", portDetail) - if err := AddNginxProxy("localhost", appName, masteraddr, portDetail, dind.GetDockerNetworkName(clusterName)); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", appName, "ports", portDetail) + if err := AddNginxProxy("localhost", kubeNames.appName, masteraddr, portDetail, dind.GetDockerNetworkName(kubeNames.clusterName)); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "appName", kubeNames.appName, "ports", portDetail) return err } } log.DebugLog(log.DebugLevelMexos, "call runKubectlCreateApp for dind") if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - err = runKubectlCreateApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + err = runKubectlCreateApp(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - err = CreateHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + err = CreateHelmAppInst(rootLB, &kubeNames, appInst, clusterInst, app.DeploymentManifest) } else { err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) } @@ -53,12 +54,12 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return nil } - switch operatorName { + switch kubeNames.operatorName { case cloudcommon.OperatorGCP: fallthrough case cloudcommon.OperatorAzure: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return runKubectlCreateApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + return runKubectlCreateApp(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { return fmt.Errorf("not yet supported") } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { @@ -69,10 +70,10 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return fmt.Errorf("unknown deployment type %s", appDeploymentType) default: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return CreateKubernetesAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + return CreateKubernetesAppInst(rootLB, &kubeNames, clusterInst, appInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - return CreateHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + return CreateHelmAppInst(rootLB, &kubeNames, appInst, clusterInst, app.DeploymentManifest) } //TODO -- support these later //} else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { @@ -87,17 +88,21 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, app *edgeproto.App, appInst *edgeproto.AppInst) error { log.DebugLog(log.DebugLevelMexos, "mex delete app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) appDeploymentType := app.Deployment - operatorName := NormalizeName(appInst.Key.CloudletKey.OperatorKey.Name) - appName := NormalizeName(app.Key.Name) + var kubeNames KubeNames + err := GetKubeNames(clusterInst, app, appInst, &kubeNames) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "GetKubeNames failed") + return err + } if CloudletIsLocalDIND() { log.DebugLog(log.DebugLevelMexos, "run kubectl delete app for dind") var err error if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - err = runKubectlDeleteApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + err = runKubectlDeleteApp(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - err = DeleteHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + err = DeleteHelmAppInst(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else { err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) } @@ -112,20 +117,20 @@ func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return err } if len(portDetail) > 0 { - if err = DeleteNginxProxy("localhost", appName); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", appName) + if err = DeleteNginxProxy("localhost", kubeNames.appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", kubeNames.appName) return err } } return nil } - switch operatorName { + switch kubeNames.operatorName { case cloudcommon.OperatorGCP: fallthrough case cloudcommon.OperatorAzure: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return runKubectlDeleteApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + return runKubectlDeleteApp(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { return fmt.Errorf("not yet supported") } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { @@ -136,9 +141,9 @@ func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return fmt.Errorf("unknown image type %s", appDeploymentType) default: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return DeleteKubernetesAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + return DeleteKubernetesAppInst(rootLB, &kubeNames, clusterInst) } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - return DeleteHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + return DeleteHelmAppInst(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } //TODO //} else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { diff --git a/mexos/azure.go b/mexos/azure.go index f167df70c..57ec54ad1 100644 --- a/mexos/azure.go +++ b/mexos/azure.go @@ -9,6 +9,10 @@ import ( "github.com/mobiledgex/edge-cloud/log" ) +func GetResourceGroupForCluster(clusterInst *edgeproto.ClusterInst) string { + return clusterInst.Key.CloudletKey.Name + "_" + clusterInst.Key.ClusterKey.Name +} + func azureCreateAKS(clusterInst *edgeproto.ClusterInst) error { var err error resourceGroup := GetResourceGroupForCluster(clusterInst) @@ -26,10 +30,8 @@ func azureCreateAKS(clusterInst *edgeproto.ClusterInst) error { if err = azure.GetAKSCredentials(resourceGroup, clusterName); err != nil { return err } - kconf, err := GetKconf(clusterInst, false) // XXX - if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v, %v", clusterInst, kconf, err) - } + kconf := GetKconfName(clusterInst) // XXX + log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX //XXX watch out for multiple cluster contexts if err = copyFile(defaultKubeconfig(), kconf); err != nil { diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index 5f1e91096..c6db82ecd 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -17,6 +17,25 @@ import ( var CloudletInfra *edgeproto.CloudletInfraProperties var CloudletInfraCommon *edgeproto.CloudletInfraCommon +//this is possible actions and optional parameters +var actionChoices = map[string]string{ + "CLOUDLET_KIND": "procname", + "stop": "procname", + "status": "procname", + "ctrlapi": "procname", + "ctrlcli": "procname", + "ctrlinfo": "procname", + "dmeapi": "procname", + "deploy": "", + "cleanup": "", + "fetchlogs": "", + "createcluster": "", + "deletecluster": "", + "gencerts": "", + "cleancerts": "", + "sleep": "seconds", +} + func InitializeCloudletInfra(fakecloudlet bool) error { log.DebugLog(log.DebugLevelMexos, "InitializeCloudletInfra called") diff --git a/mexos/gcloud.go b/mexos/gcloud.go index 7e092e56c..31299d9bb 100644 --- a/mexos/gcloud.go +++ b/mexos/gcloud.go @@ -30,10 +30,8 @@ func gcloudCreateGKE(clusterInst *edgeproto.ClusterInst) error { if err = gcloud.GetGKECredentials(clusterName); err != nil { return err } - kconf, err := GetKconf(clusterInst, false) //XXX - if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v", clusterInst, err) - } + kconf := GetKconfName(clusterInst) //XXX + log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX if err = copyFile(defaultKubeconfig(), kconf); err != nil { return fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) diff --git a/mexos/helm.go b/mexos/helm.go index 2adf73cf5..6c4fb1e88 100644 --- a/mexos/helm.go +++ b/mexos/helm.go @@ -8,37 +8,34 @@ import ( "github.com/mobiledgex/edge-cloud/log" ) -func DeleteHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { +func DeleteHelmAppInst(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst, kubeManifest string) error { log.DebugLog(log.DebugLevelMexos, "delete kubernetes helm app") - appName := NormalizeName(app.Key.Name) - clusterName := clusterInst.Key.ClusterKey.Name - var err error if rootLB == nil { return fmt.Errorf("cannot delete helm app, rootLB is null") } - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } if CloudletIsLocalDIND() { // remove DNS entries - if err = deleteAppDNS(kp, appInst.Uri, appName); err != nil { + if err = deleteAppDNS(kp, kubeNames.appURI, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) } } else { // remove DNS entries - if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + if err = KubeDeleteDNSRecords(rootLB, kp, kubeNames.appURI, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) } // remove Security rules - if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) } } - cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, appName) + cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, kubeNames.appName) out, err := kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deleting helm chart, %s, %s, %v", cmd, out, err) @@ -47,18 +44,15 @@ func DeleteHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku return nil } -func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { - log.DebugLog(log.DebugLevelMexos, "create kubernetes helm app", "clusterInst", clusterInst, "appInst", appInst) +func CreateHelmAppInst(rootLB *MEXRootLB, kubeNames *KubeNames, appInst *edgeproto.AppInst, clusterInst *edgeproto.ClusterInst, kubeManifest string) error { + log.DebugLog(log.DebugLevelMexos, "create kubernetes helm app", "clusterInst", clusterInst, "kubeNames", kubeNames) - clusterName := clusterInst.Key.ClusterKey.Name - appName := NormalizeName(app.Key.Name) - appImage := NormalizeName(app.ImagePath) var err error if rootLB == nil { return fmt.Errorf("cannot create helm app, rootLB is null") } - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } @@ -96,12 +90,12 @@ func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku // XXX This gets helm's prometheus able to query kubelet metrics. // This can be removed once Lev passes in an option in the yaml to // set the helm command line options. - prom, err := regexp.MatchString("prometheus", appName) + prom, err := regexp.MatchString("prometheus", kubeNames.appName) if err == nil && prom { log.DebugLog(log.DebugLevelMexos, "setting helm prometheus option") helmOpts = "--set kubelet.serviceMonitor.https=true" } - cmd = fmt.Sprintf("%s helm install %s --name %s %s", kp.kubeconfig, appImage, appName, helmOpts) + cmd = fmt.Sprintf("%s helm install %s --name %s %s", kp.kubeconfig, kubeNames.appImage, kubeNames.appName, helmOpts) out, err = kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deploying helm chart, %s, %s, %v", cmd, out, err) @@ -109,19 +103,19 @@ func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku log.DebugLog(log.DebugLevelMexos, "applied helm chart") if CloudletIsLocalDIND() { // Add DNS Zone - if err = createAppDNS(kp, appInst.Uri, appName); err != nil { + if err = createAppDNS(kp, kubeNames.appURI, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) return err } } else { // Add security rules - if err = AddProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + if err = AddProxySecurityRules(rootLB, kp.ipaddr, kubeNames.appName, appInst); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) return err } - log.DebugLog(log.DebugLevelMexos, "done AddProxySecurityRules", "app", app) + log.DebugLog(log.DebugLevelMexos, "done AddProxySecurityRules", "kubeNames.appName", kubeNames.appName) // Add DNS Zone - if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) return err } diff --git a/mexos/kubeadmdind.go b/mexos/kubeadmdind.go index 71ea85f62..2c117021f 100644 --- a/mexos/kubeadmdind.go +++ b/mexos/kubeadmdind.go @@ -22,10 +22,8 @@ func localCreateDIND(clusterInst *edgeproto.ClusterInst) error { //race condition exists where the config file is not ready until just after the cluster create is done time.Sleep(3 * time.Second) - kconf, err := GetKconf(clusterInst, false) // XXX - if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v", kconf, err) - } + kconf := GetKconfName(clusterInst) // XXX + log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX //XXX watch out for multiple cluster contexts if err = copyFile(defaultKubeconfig(), kconf); err != nil { diff --git a/mexos/kubeconfig.go b/mexos/kubeconfig.go index 56148900a..5b69a7d1c 100644 --- a/mexos/kubeconfig.go +++ b/mexos/kubeconfig.go @@ -23,38 +23,37 @@ func GetKconfName(clusterInst *edgeproto.ClusterInst) string { GetCloudletDNSZone()) } -func GetKconf(clusterInst *edgeproto.ClusterInst, createIfMissing bool) (string, error) { +func GetKconf(clusterInst *edgeproto.ClusterInst) (string, error) { name := GetLocalKconfName(clusterInst) operatorName := clusterInst.Key.CloudletKey.OperatorKey.Name clusterName := clusterInst.Key.ClusterKey.Name log.DebugLog(log.DebugLevelMexos, "get kubeconfig name", "name", name) - if createIfMissing { // XXX - if _, err := os.Stat(name); os.IsNotExist(err) { - // if kubeconfig does not exist, optionally create it. It is possible it was - // created on a different container or we had a restart of the container - log.DebugLog(log.DebugLevelMexos, "creating missing kconf file", "name", name) - switch operatorName { - case cloudcommon.OperatorGCP: - if err = gcloud.GetGKECredentials(clusterName); err != nil { - return "", fmt.Errorf("unable to get GKE credentials %v", err) - } - if err = copyFile(defaultKubeconfig(), name); err != nil { - return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) - } - case cloudcommon.OperatorAzure: - rg := GetResourceGroupForCluster(clusterInst) - if err = azure.GetAKSCredentials(rg, clusterName); err != nil { - return "", fmt.Errorf("unable to get AKS credentials %v", err) - } - if err = copyFile(defaultKubeconfig(), name); err != nil { - return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) - } - default: - log.DebugLog(log.DebugLevelMexos, "warning, not creating missing kubeconfig for operator", "operator", operatorName) + if _, err := os.Stat(name); os.IsNotExist(err) { + // if kubeconfig does not exist, optionally create it. It is possible it was + // created on a different container or we had a restart of the container + log.DebugLog(log.DebugLevelMexos, "creating missing kconf file", "name", name) + switch operatorName { + case cloudcommon.OperatorGCP: + if err = gcloud.GetGKECredentials(clusterName); err != nil { + return "", fmt.Errorf("unable to get GKE credentials %v", err) } + if err = copyFile(defaultKubeconfig(), name); err != nil { + return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) + } + case cloudcommon.OperatorAzure: + rg := GetResourceGroupForCluster(clusterInst) + if err = azure.GetAKSCredentials(rg, clusterName); err != nil { + return "", fmt.Errorf("unable to get AKS credentials %v", err) + } + if err = copyFile(defaultKubeconfig(), name); err != nil { + return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) + } + default: + log.DebugLog(log.DebugLevelMexos, "warning, not creating missing kubeconfig for operator", "operator", operatorName) } } + return name, nil } diff --git a/mexos/kubectl.go b/mexos/kubectl.go index f0e2abfe5..52781f702 100644 --- a/mexos/kubectl.go +++ b/mexos/kubectl.go @@ -46,19 +46,17 @@ func CreateDockerRegistrySecret(clusterInst *edgeproto.ClusterInst, rootLBName s return nil } -func runKubectlCreateApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, rootLB *MEXRootLB, kubeManifest string) error { +func runKubectlCreateApp(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst, kubeManifest string) error { log.DebugLog(log.DebugLevelMexos, "run kubectl create app", "kubeManifest", kubeManifest) - appName := NormalizeName(appInst.Key.AppKey.Name) - clusterName := clusterInst.Key.ClusterKey.Name - kfile := appName + ".yaml" + kfile := kubeNames.appName + ".yaml" if err := writeKubeManifest(kubeManifest, kfile); err != nil { return err } defer os.Remove(kfile) - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } @@ -72,11 +70,11 @@ func runKubectlCreateApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto. cmd = fmt.Sprintf("%s kubectl delete -f %s", kp.kubeconfig, kfile) out, undoerr := kp.client.Output(cmd) if undoerr != nil { - log.DebugLog(log.DebugLevelMexos, "undo kubectl create app failed", "name", appName, "out", out, "err", undoerr) + log.DebugLog(log.DebugLevelMexos, "undo kubectl create app failed", "kubeNames", kubeNames, "out", out, "err", undoerr) } } }() - err = createAppDNS(kp, appInst.Uri, appName) + err = createAppDNS(kp, kubeNames.appURI, kubeNames.appName) if err != nil { return fmt.Errorf("error creating dns entry for app, %v", err) } @@ -136,15 +134,13 @@ func getServices(kp *kubeParam) ([]v1.Service, error) { return svcs.Items, nil } -func runKubectlDeleteApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, rootLB *MEXRootLB, kubeManifest string) error { - appName := NormalizeName(appInst.Key.AppKey.Name) - clusterName := clusterInst.Key.ClusterKey.Name +func runKubectlDeleteApp(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst, kubeManifest string) error { - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } - kfile := appName + ".yaml" + kfile := kubeNames.appName + ".yaml" err = writeKubeManifest(kubeManifest, kfile) if err != nil { return err @@ -155,9 +151,9 @@ func runKubectlDeleteApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto. if err != nil { return fmt.Errorf("error deleting app, %s, %v", out, err) } - err = deleteAppDNS(kp, appInst.Uri, appName) + err = deleteAppDNS(kp, kubeNames.appURI, kubeNames.appName) if err != nil { - return fmt.Errorf("error deleting dns entry for app, %v, %v", appInst, err) + return fmt.Errorf("error deleting dns entry for app, %v, %v", kubeNames.appName, err) } return nil } diff --git a/mexos/kubernetes.go b/mexos/kubernetes.go index 7d74aa298..8f49d3215 100644 --- a/mexos/kubernetes.go +++ b/mexos/kubernetes.go @@ -9,21 +9,47 @@ import ( ssh "github.com/nanobox-io/golang-ssh" ) +type KubeNames struct { + appName string + appURI string + appImage string + clusterName string + operatorName string + kconfName string +} + +// GetKubeNames udpates kubeNames with normalized strings for the included clusterinst, app, and appisnt +func GetKubeNames(clusterInst *edgeproto.ClusterInst, app *edgeproto.App, appInst *edgeproto.AppInst, kubeNames *KubeNames) (err error) { + if clusterInst == nil { + return fmt.Errorf("nil cluster inst") + } + if app == nil { + return fmt.Errorf("nil app") + } + if appInst == nil { + return fmt.Errorf("nil app inst") + } + kubeNames.clusterName = clusterInst.Key.ClusterKey.Name + kubeNames.appName = NormalizeName(app.Key.Name) + kubeNames.appURI = appInst.Uri + kubeNames.appImage = NormalizeName(app.ImagePath) + kubeNames.operatorName = NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name) + kubeNames.kconfName = GetKconfName(clusterInst) + return nil + +} + //CreateKubernetesAppInst instantiates a new kubernetes deployment -func CreateKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { +func CreateKubernetesAppInst(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, kubeManifest string) error { log.DebugLog(log.DebugLevelMexos, "create kubernetes app") - clusterName := clusterInst.Key.ClusterKey.Name - appName := NormalizeName(app.Key.Name) - if rootLB == nil { return fmt.Errorf("cannot create kubernetes app manifest, rootLB is null") } - if appInst.Uri == "" { return fmt.Errorf("empty app URI") } - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } @@ -36,26 +62,26 @@ func CreateKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterIn // return err //} //TODO do not create yaml file but use remote yaml file over https - cmd = fmt.Sprintf("cat <<'EOF'> %s.yaml \n%s\nEOF", appName, kubeManifest) + cmd = fmt.Sprintf("cat <<'EOF'> %s.yaml \n%s\nEOF", kubeNames.appName, kubeManifest) out, err := kp.client.Output(cmd) if err != nil { return fmt.Errorf("error writing KubeManifest, %s, %s, %v", cmd, out, err) } log.DebugLog(log.DebugLevelMexos, "wrote kubernetes manifest file") - cmd = fmt.Sprintf("%s kubectl create -f %s.yaml", kp.kubeconfig, appName) + cmd = fmt.Sprintf("%s kubectl create -f %s.yaml", kp.kubeconfig, kubeNames.appName) out, err = kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deploying kubernetes app, %s, %v", out, err) } log.DebugLog(log.DebugLevelMexos, "applied kubernetes manifest") // Add security rules - if err = AddProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + if err = AddProxySecurityRules(rootLB, kp.ipaddr, kubeNames.appName, appInst); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) return err } log.DebugLog(log.DebugLevelMexos, "ok, added ports", "ports", appInst.MappedPorts) // Add DNS Zone - if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) return err } @@ -69,16 +95,15 @@ type kubeParam struct { } //ValidateKubernetesParameters checks the kubernetes parameters and kubeconfig settings -func ValidateKubernetesParameters(clusterInst *edgeproto.ClusterInst, rootLB *MEXRootLB, clustName string) (*kubeParam, error) { - log.DebugLog(log.DebugLevelMexos, "validate kubernetes parameters rootLB", "cluster", clustName) - clusterName := clusterInst.Key.ClusterKey.Name +func ValidateKubernetesParameters(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst) (*kubeParam, error) { + log.DebugLog(log.DebugLevelMexos, "validate kubernetes parameters", "kubeNames", kubeNames) if CloudletIsDirectKubectlAccess() { // No ssh jump host (rootlb) but kconf configures how to // talk to remote kubernetes cluster. This includes DIND, AKS, GCP - kconf, err := GetKconf(clusterInst, true) + kconf, err := GetKconf(clusterInst) if err != nil { - return nil, fmt.Errorf("kconf missing, %v, %v", clustName, err) + return nil, fmt.Errorf("kconf missing, %v, %v", kubeNames.clusterName, err) } kp := kubeParam{ kubeconfig: fmt.Sprintf("KUBECONFIG=%s", kconf), @@ -96,47 +121,44 @@ func ValidateKubernetesParameters(clusterInst *edgeproto.ClusterInst, rootLB *ME if err != nil { return nil, err } - master, err := FindClusterMaster(clusterName) + master, err := FindClusterMaster(kubeNames.clusterName) if err != nil { - return nil, fmt.Errorf("can't find cluster with key %s, %v", clustName, err) + return nil, fmt.Errorf("can't find cluster with key %s, %v", kubeNames.clusterName, err) } ipaddr, err := FindNodeIP(master) if err != nil { return nil, err } //kubeconfig := fmt.Sprintf("KUBECONFIG=%s.kubeconfig", name[strings.LastIndex(name, "-")+1:]) - kubeconfig := fmt.Sprintf("KUBECONFIG=%s", GetKconfName(clusterInst)) + kubeconfig := fmt.Sprintf("KUBECONFIG=%s", kubeNames.kconfName) return &kubeParam{kubeconfig, client, ipaddr}, nil } -func DeleteKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { +func DeleteKubernetesAppInst(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst) error { log.DebugLog(log.DebugLevelMexos, "delete kubernetes app") - clusterName := clusterInst.Key.ClusterKey.Name - appName := NormalizeName(app.Key.Name) - - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } // Clean up security rules and nginx proxy - if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot clean up security rules", "name", appName, "rootlb", rootLB.Name, "error", err) + if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, kubeNames.appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot clean up security rules", "name", kubeNames.appName, "rootlb", rootLB.Name, "error", err) } if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { return fmt.Errorf("cannot init cloudflare api, %v", err) } // Clean up DNS entries - if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot clean up DNS entries", "name", appName, "rootlb", rootLB.Name, "error", err) + if err = KubeDeleteDNSRecords(rootLB, kp, kubeNames.appURI, kubeNames.appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot clean up DNS entries", "name", kubeNames.appName, "rootlb", rootLB.Name, "error", err) return err } - cmd := fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, appName) + cmd := fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, kubeNames.appName) out, err := kp.client.Output(cmd) if err != nil { - return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", appName, cmd, out, err) + return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", kubeNames.appName, cmd, out, err) } - log.DebugLog(log.DebugLevelMexos, "deleted deployment", "name", appName) + log.DebugLog(log.DebugLevelMexos, "deleted deployment", "name", kubeNames.appName) return nil } diff --git a/mexos/misc.go b/mexos/misc.go index 3559d39da..bac1ba15e 100644 --- a/mexos/misc.go +++ b/mexos/misc.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "os" - "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" "github.com/mobiledgex/edge-cloud/util" ) @@ -108,7 +107,3 @@ func GetHTPassword(rootLBName string) error { log.DebugLog(log.DebugLevelMexos, "downloaded htpasswd") return nil } - -func GetResourceGroupForCluster(clusterInst *edgeproto.ClusterInst) string { - return clusterInst.Key.CloudletKey.Name + "_" + clusterInst.Key.ClusterKey.Name -} diff --git a/mexos/oscli.go b/mexos/oscli.go index 9de3fbc4f..93e7ed6c1 100644 --- a/mexos/oscli.go +++ b/mexos/oscli.go @@ -46,8 +46,9 @@ func ListImages() ([]OSImage, error) { //ListNetworks lists networks known to the platform. Some created by the operator, some by users. func ListNetworks() ([]OSNetwork, error) { - out, err := sh.Command("openstack", "network", "list", "-f", "json").Output() + out, err := sh.Command("openstack", "network", "list", "-f", "json").CombinedOutput() if err != nil { + log.DebugLog(log.DebugLevelMexos, "network list failed", "out", out) err = fmt.Errorf("cannot get network list, %v", err) return nil, err } @@ -168,7 +169,7 @@ func CreateNetwork(name string) error { //DeleteNetwork destroys a named network // Sometimes it will fail. Openstack will refuse if there are resources attached. -func DeleteNetwork( name string) error { +func DeleteNetwork(name string) error { log.DebugLog(log.DebugLevelMexos, "deleting network", "network", name) out, err := sh.Command("openstack", "network", "delete", name).CombinedOutput() if err != nil { @@ -242,7 +243,7 @@ func CreateRouter(routerName string) error { } //DeleteRouter removes the named router. The router needs to not be in use at the time of deletion. -func DeleteRouter( routerName string) error { +func DeleteRouter(routerName string) error { log.DebugLog(log.DebugLevelMexos, "deleting router", "name", routerName) out, err := sh.Command("openstack", "router", "delete", routerName).Output() if err != nil { @@ -358,7 +359,7 @@ func CreateServerImage(serverName, imageName string) error { } //CreateImage puts images into glance -func CreateImage( imageName, qcowFile string) error { +func CreateImage(imageName, qcowFile string) error { log.DebugLog(log.DebugLevelMexos, "creating image in glance", "image", imageName, "qcow", qcowFile) out, err := sh.Command("openstack", "image", "create", imageName, @@ -376,7 +377,7 @@ func CreateImage( imageName, qcowFile string) error { // It will then save that into a local file. The image transfer happens from glance into your own laptop // or whatever. // This can take a while, transferring all the data. -func SaveImage( saveName, imageName string) error { +func SaveImage(saveName, imageName string) error { log.DebugLog(log.DebugLevelMexos, "saving image", "save name", saveName, "image name", imageName) out, err := sh.Command("openstack", "image", "save", "--file", saveName, imageName).Output() if err != nil { @@ -389,7 +390,7 @@ func SaveImage( saveName, imageName string) error { //DeleteImage deletes the named image from glance. Sometimes backing store is still busy and // will refuse to honor the request. Like most things in Openstack, wait for a while and try // again. -func DeleteImage( imageName string) error { +func DeleteImage(imageName string) error { log.DebugLog(log.DebugLevelMexos, "deleting image", "name", imageName) out, err := sh.Command("openstack", "image", "delete", imageName).Output() if err != nil { diff --git a/mexos/rootlb.go b/mexos/rootlb.go index 3411a2359..2df7d9a5c 100644 --- a/mexos/rootlb.go +++ b/mexos/rootlb.go @@ -11,7 +11,6 @@ import ( //MEXRootLB has rootLB data type MEXRootLB struct { Name string - // PlatConf *Manifest } var MEXRootLBMap = make(map[string]*MEXRootLB) diff --git a/mexos/securityrule.go b/mexos/securityrule.go index 443a21e72..acf24f7b2 100644 --- a/mexos/securityrule.go +++ b/mexos/securityrule.go @@ -12,9 +12,7 @@ import ( // TODO service to periodically clean up the leftover rules -func AddProxySecurityRules(rootLB *MEXRootLB, masteraddr string, appInst *edgeproto.AppInst) error { - - name := NormalizeName(appInst.Key.AppKey.Name) +func AddProxySecurityRules(rootLB *MEXRootLB, masteraddr string, appName string, appInst *edgeproto.AppInst) error { ports, err := GetPortDetail(appInst) if err != nil { @@ -43,17 +41,16 @@ func AddProxySecurityRules(rootLB *MEXRootLB, masteraddr string, appInst *edgepr } } if len(ports) > 0 { - if err := AddNginxProxy(rootLB.Name, name, masteraddr, ports, ""); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", name) + if err := AddNginxProxy(rootLB.Name, appName, masteraddr, ports, ""); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "appName", appName) return err } } - log.DebugLog(log.DebugLevelMexos, "added nginx proxy", "name", name, "ports", appInst.MappedPorts) + log.DebugLog(log.DebugLevelMexos, "added nginx proxy", "appName", appName, "ports", appInst.MappedPorts) return nil } -func DeleteProxySecurityRules(rootLB *MEXRootLB, ipaddr string, appInst *edgeproto.AppInst) error { - appName := NormalizeName(appInst.Key.AppKey.Name) +func DeleteProxySecurityRules(rootLB *MEXRootLB, ipaddr string, appName string) error { log.DebugLog(log.DebugLevelMexos, "delete proxy rules", "name", appName) err := DeleteNginxProxy(rootLB.Name, appName) diff --git a/mexos/templates.go b/mexos/templates.go deleted file mode 100644 index 904e37b01..000000000 --- a/mexos/templates.go +++ /dev/null @@ -1,189 +0,0 @@ -package mexos - -/* -type templateFill struct { - Name, Kind, Flavor, Tags, Tenant, DNSZone string - ImageFlavor, RootLB, Resource, ResourceKind, ResourceGroup string - StorageSpec, NetworkScheme string - NodeFlavor, Operator, Key, Image, Options string - ImageType, ProxyPath string - ExternalNetwork, Project string - ExternalRouter, Flags, IPAccess, Swarm string - NumMasters, NumNodes int - Config templateConfig - Command []string - SpecPorts []PortDetail -} - -type templateConfig struct { - Base, Overlay, Deployment, Resources, Manifest, Template string -} - -var yamlMEXCluster = `apiVersion: v1 -kind: {{.ResourceKind}} -resource: {{.Resource}} -metadata: - name: {{.Name}} - tags: {{.Tags}} - kind: {{.Kind}} - tenant: {{.Tenant}} - operator: {{.Operator}} - project: {{.Project}} - dnszone: {{.DNSZone}} - swarm: {{.Swarm}} - resourcegroup: {{.ResourceGroup}} -spec: - flags: {{.Flags}} - flavor: {{.Flavor}} - key: {{.Key}} - dockerregistry: registry.mobiledgex.net:5000 - rootlb: {{.RootLB}} - networkscheme: {{.NetworkScheme}} -` - -var yamlMEXFlavor = `apiVersion: v1 -kind: {{.ResourceKind}} -resource: {{.Resource}} -metadata: - name: {{.Name}} - tags: {{.Tags}} - kind: {{.Kind}} -spec: - flags: {{.Flags}} - flavor: {{.Name}} - flavordetail: - name: {{.Name}} - nodes: {{.NumNodes}} - masters: {{.NumMasters}} - networkscheme: {{.NetworkScheme}} - masterflavor: {{.MasterFlavor}} - nodeflavor: {{.NodeFlavor}} - storagescheme: {{.StorageSpec}} - topology: {{.Topology}} -` - -var yamlMEXPlatform = `apiVersion: v1 -kind: {{.ResourceKind}} -resource: {{.Resource}} -metadata: - kind: {{.Kind}} - name: {{.Name}} - tags: {{.Tags}} - tenant: {{.Tenant}} - openrc: ~/.mobiledgex/openrc - dnszone: {{.DNSZone}} - operator: {{.Operator}} -spec: - flags: {{.Flags}} - flavor: {{.Flavor}} - rootlb: {{.RootLB}} - key: {{.Key}} - dockerregistry: registry.mobiledgex.net:5000 - externalnetwork: {{.ExternalNetwork}} - networkscheme: {{.NetworkScheme}} - externalrouter: {{.ExternalRouter}} - options: {{.Options}} -` - -var yamlMEXApp = `apiVersion: v1 -kind: {{.ResourceKind}} -resource: {{.Resource}} -metadata: - kind: {{.Kind}} - name: {{.Name}} - tags: {{.Tags}} - tenant: {{.Tenant}} - operator: {{.Operator}} - dnszone: {{.DNSZone}} -config: - kind: - source: - detail: - resources: {{.Config.Resources}} - deployment: {{.Config.Deployment}} - manifest: {{.Config.Manifest}} - template: {{.Config.Template}} - base: {{.Config.Base}} - overlay: {{.Config.Overlay}} -spec: - flags: {{.Flags}} - key: {{.Key}} - rootlb: {{.RootLB}} - image: {{.Image}} - imagetype: {{.ImageType}} - imageflavor: {{.ImageFlavor}} - proxypath: {{.ProxyPath}} - flavor: {{.Flavor}} - ipaccess: {{.IPAccess}} - networkscheme: {{.NetworkScheme}} - ports: -{{- range .SpecPorts}} - - {{.Name}} - {{.MexProto}} - {{.Proto}} - {{.InternalPort}} - {{.PublicPort}} - {{.PublicPath}} -{{- end}} - command: -{{- range .Command}} - - {{.}} -{{- end}} -` - -func fillPlatformTemplateCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) (*Manifest, error) { - log.DebugLog(log.DebugLevelMexos, "fill template cloudletkeystr", "cloudletkeystr", cloudletKeyStr) - - - log.DebugLog(log.DebugLevelMexos, "using external network", "extNet", GetCloudletExternalNetwork()) - meximage := os.Getenv("MEX_OS_IMAGE") - if meximage == "" { - return nil, fmt.Errorf("Env variable MEX_OS_IMAGE not set") - } - - data := templateFill{ - ResourceKind: "platform", - Resource: NormalizeName(clk.OperatorKey.Name), - Name: clk.Name, - Tags: clk.Name + "-tag", - Key: clk.Name + "-" + NormalizeName(clk.OperatorKey.Name), - Flavor: "x1.medium", - Operator: NormalizeName(clk.OperatorKey.Name), - RootLB: rootLB.Name, - Kind: "mex-platform", - ExternalNetwork: GetCloudletExternalNetwork(), - NetworkScheme: GetCloudletNetworkScheme(), - DNSZone: GetCloudletDNSZone(), - ExternalRouter: GetCloudletExternalRouter(), - Options: "dhcp", - } - mf, err := templateUnmarshal(&data, yamlMEXPlatform) - if err != nil { - return nil, err - } - return mf, nil -} - - -func templateUnmarshal(data *templateFill, yamltext string) (*Manifest, error) { - //log.DebugLog(log.DebugLevelMexos, "template unmarshal", "yamltext", string, "data", data) - tmpl, err := template.New("mex").Parse(yamltext) - if err != nil { - return nil, fmt.Errorf("can't create template for, %v", err) - } - var outbuffer bytes.Buffer - err = tmpl.Execute(&outbuffer, data) - if err != nil { - //log.DebugLog(log.DebugLevelMexos, "template data", "data", data) - return nil, fmt.Errorf("can't execute template, %v", err) - } - mf := &Manifest{} - err = yaml.Unmarshal(outbuffer.Bytes(), mf) - if err != nil { - log.DebugLog(log.DebugLevelMexos, "error yaml unmarshal, templated data") - return nil, fmt.Errorf("can't unmarshal templated data, %v, %s", err, outbuffer.String()) - } - return mf, nil -} - -*/ diff --git a/mexos/vault.go b/mexos/vault.go index 1c6978931..1bbf2ed66 100644 --- a/mexos/vault.go +++ b/mexos/vault.go @@ -148,7 +148,6 @@ func CheckPlatformEnv(platformType string) error { } func GetVaultEnv(openrc string, mexenv string, cloudletInfra *edgeproto.CloudletInfraProperties) error { - if err := InternVaultEnv(openrc, mexenv, cloudletInfra); err != nil { return err }