From c716f775afdab4cb42b242dc3f4acb8d1d243ebb Mon Sep 17 00:00:00 2001 From: Jim Date: Thu, 7 Feb 2019 13:26:48 -0600 Subject: [PATCH 01/10] refactor mexos to remove manifest and dependency on registry. --- k8s-prov/dind/dind.go | 2 +- mexos/agent.go | 91 ++++++----- mexos/azure.go | 20 ++- mexos/cloudflare.go | 8 +- mexos/cloudletinfra.go | 231 ++++++++++++++++++++++++++++ mexos/cluster.go | 207 ++++++++++++++++--------- mexos/dns.go | 116 ++++++-------- mexos/env.go | 10 +- mexos/flavor.go | 10 -- mexos/fqdn.go | 19 +-- mexos/gcloud.go | 28 ++-- mexos/helm.go | 109 ++++--------- mexos/init.go | 7 +- mexos/inst.go | 199 ++++++++++++++---------- mexos/ip.go | 18 +-- mexos/kubeadmdind.go | 19 ++- mexos/kubeconfig.go | 52 ++++--- mexos/kubectl.go | 103 +++++++------ mexos/kubeproxy.go | 10 +- mexos/kubernetes.go | 210 ++++++-------------------- mexos/kvm.go | 93 +++++++----- mexos/lb.go | 22 +-- mexos/letsencrypt.go | 29 ++-- mexos/manifest.go | 336 ++--------------------------------------- mexos/misc.go | 26 ++-- mexos/network.go | 12 +- mexos/nginx.go | 18 +-- mexos/oscli.go | 56 +++---- mexos/platform.go | 9 +- mexos/ports.go | 14 +- mexos/rootlb.go | 68 ++++----- mexos/securityrule.go | 47 +++--- mexos/ssh.go | 12 +- mexos/stats.go | 4 +- mexos/swarm.go | 37 ++--- mexos/templates.go | 170 +++------------------ mexos/types.go | 56 ++++--- mexos/valid.go | 8 - mexos/vault.go | 42 +++--- 39 files changed, 1122 insertions(+), 1406 deletions(-) create mode 100644 mexos/cloudletinfra.go diff --git a/k8s-prov/dind/dind.go b/k8s-prov/dind/dind.go index 8960f2e5d..349f40ed9 100644 --- a/k8s-prov/dind/dind.go +++ b/k8s-prov/dind/dind.go @@ -36,7 +36,7 @@ func GetDockerNetworkName(clusterName string) string { } //CreateDINDCluster creates kubernetes cluster on local mac -func CreateDINDCluster(group, name string) error { +func CreateDINDCluster(name string) error { os.Setenv("DIND_LABEL", name) os.Setenv("CLUSTER_ID", getClusterID()) log.DebugLog(log.DebugLevelMexos, "CreateDINDCluster via dind-cluster-v1.13.sh", "name", name, "clusterid", getClusterID()) diff --git a/mexos/agent.go b/mexos/agent.go index 323342043..025dc02c4 100644 --- a/mexos/agent.go +++ b/mexos/agent.go @@ -1,11 +1,12 @@ package mexos import ( + "encoding/json" "fmt" "strings" valid "github.com/asaskevich/govalidator" - "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/integration/process" "github.com/mobiledgex/edge-cloud/log" ) @@ -16,25 +17,22 @@ func runLocalMexAgent() error { return localMexos.Start("/tmp/mexosagent.log") } -//RunMEXAgentManifest runs the MEX agent on the RootLB. It first registers FQDN to cloudflare domain registry if not already registered. +//RunMEXAgent runs the MEX agent on the RootLB. It first registers FQDN to cloudflare domain registry if not already registered. // It then obtains certficiates from Letsencrypt, if not done yet. Then it runs the docker instance of MEX agent // on the RootLB. It can be told to manually pull image from docker repository. This allows upgrading with new image. // It uses MEX private docker repository. If an instance is running already, we don't start another one. -func RunMEXAgentManifest(mf *Manifest) error { +func RunMEXAgent(rootLBName string, cloudletKey *edgeproto.CloudletKey) error { log.DebugLog(log.DebugLevelMexos, "run mex agent") - if IsLocalDIND(mf) { + if IsLocalDIND() { return runLocalMexAgent() } - fqdn := mf.Spec.RootLB + fqdn := rootLBName //fqdn is that of the machine/kvm-instance running the agent if !valid.IsDNSName(fqdn) { return fmt.Errorf("fqdn %s is not valid", fqdn) } - if err := setPlatConfManifest(mf); err != nil { - return fmt.Errorf("can't set plat conf, %v", err) - } - sd, err := GetServerDetails(mf, fqdn) + sd, err := GetServerDetails(fqdn) if err == nil { if sd.Name == fqdn { log.DebugLog(log.DebugLevelMexos, "server with same name as rootLB exists", "fqdn", fqdn) @@ -43,7 +41,7 @@ func RunMEXAgentManifest(mf *Manifest) error { return fmt.Errorf("cannot find rootlb %s", fqdn) } //return RunMEXOSAgentContainer(mf, rootLB) - return RunMEXOSAgentService(mf, rootLB) + return RunMEXOSAgentService(rootLB) } } log.DebugLog(log.DebugLevelMexos, "about to create mex agent", "fqdn", fqdn) @@ -54,50 +52,47 @@ func RunMEXAgentManifest(mf *Manifest) error { if rootLB == nil { return fmt.Errorf("cannot run mex agent manifest, rootLB is null") } - if mf.Spec.ExternalNetwork == "" { - return fmt.Errorf("missing external network") - } - if mf.Spec.Agent.Image == "" { + //if mf.Spec.ExternalNetwork == "" { + // return fmt.Errorf("missing external network") + //} + if GetCloudletOSImage() == "" { return fmt.Errorf("missing agent image") } - if mf.Metadata.Name == "" { - return fmt.Errorf("missing name") - } log.DebugLog(log.DebugLevelMexos, "record platform config") - err = EnableRootLB(mf, rootLB) + err = EnableRootLB(rootLB, cloudletKey) if err != nil { log.DebugLog(log.DebugLevelMexos, "can't enable agent", "name", rootLB.Name) return fmt.Errorf("Failed to enable root LB %v", err) } - err = WaitForRootLB(mf, rootLB) + err = WaitForRootLB(rootLB) if err != nil { log.DebugLog(log.DebugLevelMexos, "timeout waiting for agent to run", "name", rootLB.Name) return fmt.Errorf("Error waiting for rootLB %v", err) } - if err := SetupSSHUser(mf, rootLB, sshUser); err != nil { + if err := SetupSSHUser(rootLB, sshUser); err != nil { return err } - if err = ActivateFQDNA(mf, rootLB, rootLB.Name); err != nil { + if err = ActivateFQDNA(rootLB, rootLB.Name); err != nil { return err } log.DebugLog(log.DebugLevelMexos, "FQDN A record activated", "name", rootLB.Name) - err = AcquireCertificates(mf, rootLB, rootLB.Name) //fqdn name may be different than rootLB.Name + err = AcquireCertificates(rootLB, rootLB.Name) //fqdn name may be different than rootLB.Name if err != nil { return fmt.Errorf("can't acquire certificate for %s, %v", rootLB.Name, err) } log.DebugLog(log.DebugLevelMexos, "acquired certificates from letsencrypt", "name", rootLB.Name) - err = GetHTPassword(mf, rootLB) + err = GetHTPassword(rootLB.Name) if err != nil { return fmt.Errorf("can't download htpassword %v", err) } //return RunMEXOSAgentContainer(mf, rootLB) - return RunMEXOSAgentService(mf, rootLB) + return RunMEXOSAgentService(rootLB) } -func RunMEXOSAgentService(mf *Manifest, rootLB *MEXRootLB) error { +func RunMEXOSAgentService(rootLB *MEXRootLB) error { //TODO check if agent is running before restarting again. log.DebugLog(log.DebugLevelMexos, "run mexosagent service") - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return err } @@ -113,7 +108,7 @@ func RunMEXOSAgentService(mf *Manifest, rootLB *MEXRootLB) error { {"/usr/local/bin", "mexosagent"}, {"/lib/systemd/system", "mexosagent.service"}, } { - cmd := fmt.Sprintf("sudo scp -o %s -o %s -i id_rsa_mex mobiledgex@%s:files-repo/mobiledgex/%s %s", sshOpts[0], sshOpts[1], mf.Values.Registry.Name, dest.name, dest.path) + cmd := fmt.Sprintf("sudo scp -o %s -o %s -i id_rsa_mex mobiledgex@%s:files-repo/mobiledgex/%s %s", sshOpts[0], sshOpts[1], GetCloudletRegistryFileServer(), dest.name, dest.path) out, err := client.Output(cmd) if err != nil { log.InfoLog("error: cannot download from registry", "fn", dest.name, "path", dest.path, "error", err, "out", out) @@ -137,15 +132,12 @@ func RunMEXOSAgentService(mf *Manifest, rootLB *MEXRootLB) error { } func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { - if mexEnv(mf, "MEX_DOCKER_REG_PASS") == "" { - return fmt.Errorf("empty docker registry pass env var") - } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return err } //XXX rewrite this with --format {{.Names}} - cmd := fmt.Sprintf("docker ps --filter ancestor=%s --format {{.Names}}", mf.Spec.Agent.Image) + cmd := fmt.Sprintf("docker ps --filter ancestor=%s --format {{.Names}}", GetCloudletAgentContainerImage()) out, err := client.Output(cmd) if err == nil && strings.Contains(out, rootLB.Name) { //agent docker instance exists @@ -153,7 +145,7 @@ func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { log.DebugLog(log.DebugLevelMexos, "agent docker instance already running") return nil } - cmd = fmt.Sprintf("echo %s > .docker-pass", mexEnv(mf, "MEX_DOCKER_REG_PASS")) + cmd = fmt.Sprintf("echo %s > .docker-pass", GetCloudletDockerPass()) out, err = client.Output(cmd) if err != nil { return fmt.Errorf("can't store docker pass, %s, %v", out, err) @@ -162,7 +154,7 @@ func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { dockerinstanceName := fmt.Sprintf("%s-%s", mf.Metadata.Name, rootLB.Name) if mf.Spec.DockerRegistry == "" { log.DebugLog(log.DebugLevelMexos, "warning, empty docker registry spec, using default.") - mf.Spec.DockerRegistry = mf.Values.Registry.Docker + mf.Spec.DockerRegistry = GetCloudletDockerRegistry() } cmd = fmt.Sprintf("cat .docker-pass| docker login -u mobiledgex --password-stdin %s", mf.Spec.DockerRegistry) out, err = client.Output(cmd) @@ -170,13 +162,13 @@ func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { return fmt.Errorf("error docker login at %s, %s, %s, %v", rootLB.Name, cmd, out, err) } log.DebugLog(log.DebugLevelMexos, "docker login ok") - cmd = fmt.Sprintf("docker pull %s", mf.Spec.Agent.Image) //probably redundant + cmd = fmt.Sprintf("docker pull %s", GetCloudletAgentContainerImage()) //probably redundant out, err = client.Output(cmd) if err != nil { return fmt.Errorf("error pulling docker image at %s, %s, %s, %v", rootLB.Name, cmd, out, err) } log.DebugLog(log.DebugLevelMexos, "pulled agent image ok") - cmd = fmt.Sprintf("docker run -d --rm --name %s --net=host -v `pwd`:/var/www/.cache -v /etc/ssl/certs:/etc/ssl/certs %s -debug", dockerinstanceName, mf.Spec.Agent.Image) + cmd = fmt.Sprintf("docker run -d --rm --name %s --net=host -v `pwd`:/var/www/.cache -v /etc/ssl/certs:/etc/ssl/certs %s -debug", dockerinstanceName, GetCloudletAgentContainerImage()) out, err = client.Output(cmd) if err != nil { return fmt.Errorf("error running dockerized agent on RootLB %s, %s, %s, %v", rootLB.Name, cmd, out, err) @@ -185,6 +177,7 @@ func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { return nil } +/* //UpdateMEXAgentManifest upgrades the mex agent func UpdateMEXAgentManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "update mex agent") @@ -195,12 +188,29 @@ func UpdateMEXAgentManifest(mf *Manifest) error { // Force pulling a potentially newer docker image return RunMEXAgentManifest(mf) } +*/ + +//RunMEXAgentCloudletKey calls MEXPlatformInit with templated manifest +func RunMEXAgentCloudletKey(rootLBName string, cloudletKeyStr string) error { + + clk := edgeproto.CloudletKey{} + err := json.Unmarshal([]byte(cloudletKeyStr), &clk) + if err != nil { + return fmt.Errorf("can't unmarshal json cloudletkey %s, %v", cloudletKeyStr, err) + } + log.DebugLog(log.DebugLevelMexos, "unmarshalled cloudletkeystr", "cloudletkey", clk) + if clk.Name == "" || clk.OperatorKey.Name == "" { + return fmt.Errorf("invalid cloudletkeystr %s", cloudletKeyStr) + } + return RunMEXAgent(rootLBName, &clk) +} +/* //RemoveMEXAgentManifest deletes mex agent docker instance func RemoveMEXAgentManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "deleting mex agent") //XXX we are deleting server kvm!!! - err := DeleteServer(mf, mf.Spec.RootLB) + err := DeleteServer(mf.Spec.RootLB) force := strings.Contains(mf.Spec.Flags, "force") if err != nil { if !force { @@ -209,14 +219,14 @@ func RemoveMEXAgentManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "forced to continue, deleting mex agent error", "error", err, "rootLB", mf.Spec.RootLB) } log.DebugLog(log.DebugLevelMexos, "removed rootlb", "name", mf.Spec.RootLB) - sip, err := GetServerIPAddr(mf, mf.Values.Network.External, mf.Spec.RootLB) - if err := DeleteSecurityRule(mf, sip); err != nil { + sip, err := GetServerIPAddr(GetCloudletExternalNetwork(), mf.Spec.RootLB) + if err := DeleteSecurityRule(sip); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rule", "error", err, "server ip", sip) } if mf.Metadata.DNSZone == "" { return fmt.Errorf("missing dns zone in manifest, metadata %v", mf.Metadata) } - if cerr := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); cerr != nil { + if cerr := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); cerr != nil { return fmt.Errorf("cannot init cloudflare api, %v", cerr) } recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) @@ -236,3 +246,4 @@ func RemoveMEXAgentManifest(mf *Manifest) error { //TODO remove mex-k8s internal nets and router return nil } +*/ diff --git a/mexos/azure.go b/mexos/azure.go index 68c49920d..f167df70c 100644 --- a/mexos/azure.go +++ b/mexos/azure.go @@ -5,32 +5,36 @@ import ( "time" "github.com/mobiledgex/edge-cloud-infra/k8s-prov/azure" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) -func azureCreateAKS(mf *Manifest) error { +func azureCreateAKS(clusterInst *edgeproto.ClusterInst) error { var err error - if err = azure.CreateResourceGroup(mf.Metadata.ResourceGroup, mf.Metadata.Location); err != nil { + resourceGroup := GetResourceGroupForCluster(clusterInst) + clusterName := clusterInst.Key.ClusterKey.Name + location := GetCloudletAzureLocation() + if err = azure.CreateResourceGroup(resourceGroup, location); err != nil { return err } - if err = azure.CreateAKSCluster(mf.Metadata.ResourceGroup, mf.Metadata.Name); err != nil { + if err = azure.CreateAKSCluster(resourceGroup, clusterName); err != nil { return err } //race condition exists where the config file is not ready until just after the cluster create is done time.Sleep(3 * time.Second) saveKubeconfig() - if err = azure.GetAKSCredentials(mf.Metadata.ResourceGroup, mf.Metadata.Name); err != nil { + if err = azure.GetAKSCredentials(resourceGroup, clusterName); err != nil { return err } - kconf, err := GetKconf(mf, false) // XXX + kconf, err := GetKconf(clusterInst, false) // XXX if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v, %v", mf, kconf, err) + return fmt.Errorf("cannot get kconf, %v, %v, %v", clusterInst, kconf, err) } log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX //XXX watch out for multiple cluster contexts if err = copyFile(defaultKubeconfig(), kconf); err != nil { return fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) } - log.DebugLog(log.DebugLevelMexos, "created aks", "name", mf.Spec.Key) - return CreateDockerRegistrySecret(mf) + log.DebugLog(log.DebugLevelMexos, "created aks", "name", clusterName) + return CreateDockerRegistrySecret(clusterInst, "") } diff --git a/mexos/cloudflare.go b/mexos/cloudflare.go index b00be60dc..a7486fbb6 100644 --- a/mexos/cloudflare.go +++ b/mexos/cloudflare.go @@ -1,11 +1,6 @@ package mexos -import ( - "fmt" - - "github.com/mobiledgex/edge-cloud/log" -) - +/* //CheckCredentialsCF checks for Cloudflare func CheckCredentialsCF(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "check for cloudflare credentials") @@ -16,3 +11,4 @@ func CheckCredentialsCF(mf *Manifest) error { } return nil } +*/ diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go new file mode 100644 index 000000000..f344a584d --- /dev/null +++ b/mexos/cloudletinfra.go @@ -0,0 +1,231 @@ +package mexos + +// This file stores a global cloudlet infra properties object. The long term solution is for the controller to send this via the +// notification channel when the cloudlet is provisioned. The controller will do the vault access and pass this data down; this +// is a stepping stone to start using edgepro data strucures to hold info abou the cloudlet rather than custom types and so the vault +// is still directly accessed here as are env variable to populate some variables + +import ( + "fmt" + "os" + + "github.com/mobiledgex/edge-cloud/cloudcommon" + "github.com/mobiledgex/edge-cloud/edgeproto" + "github.com/mobiledgex/edge-cloud/log" +) + +var CloudletInfra *edgeproto.CloudletInfraProperties +var CloudletInfraCommon *edgeproto.CloudletInfraCommon + +//this is possible actions and optional parameters +var actionChoices = map[string]string{ + "CLOUDLET_KIND": "procname", + "stop": "procname", + "status": "procname", + "ctrlapi": "procname", + "ctrlcli": "procname", + "ctrlinfo": "procname", + "dmeapi": "procname", + "deploy": "", + "cleanup": "", + "fetchlogs": "", + "createcluster": "", + "deletecluster": "", + "gencerts": "", + "cleancerts": "", + "sleep": "seconds", +} + +func InitializeCloudletInfra(fakecloudlet bool) error { + log.DebugLog(log.DebugLevelMexos, "InitializeCloudletInfra called") + + CloudletInfra = new(edgeproto.CloudletInfraProperties) + CloudletInfra.OpenstackProperties = new(edgeproto.OpenStackProperties) + CloudletInfra.OpenstackProperties.OpenRcVars = make(map[string]string) + CloudletInfra.AzureProperties = new(edgeproto.AzureProperties) + CloudletInfra.GcpProperties = new(edgeproto.GcpProperties) + CloudletInfraCommon = new(edgeproto.CloudletInfraCommon) + + var openRcURL string + var mexEnvURL string + + if fakecloudlet { + CloudletInfra.CloudletKind = cloudcommon.CloudletKindFake + } else { + + CloudletInfra.CloudletKind = os.Getenv("CLOUDLET_KIND") + if CloudletInfra.CloudletKind == "" { + return fmt.Errorf("Env variable CLOUDLET_KIND not set") + } + mexEnvURL = os.Getenv("MEXENV_URL") + if mexEnvURL == "" { + return fmt.Errorf("Env variable MEXENV_URL not set") + } + + openRcURL = os.Getenv("OPENRC_URL") + if openRcURL == "" { + return fmt.Errorf("Env variable OPENRC_URL not set") + } + + err := InternVaultEnv(openRcURL, mexEnvURL, CloudletInfra) + if err != nil { + return fmt.Errorf("failed to InternVaultEnv: %v", err) + } + + CloudletInfraCommon.CFKey = os.Getenv("MEX_CF_KEY") + if CloudletInfraCommon.CFKey == "" { + return fmt.Errorf("Env variable MEX_CF_KEY not set") + } + CloudletInfraCommon.CFUser = os.Getenv("MEX_CF_USER") + if CloudletInfraCommon.CFKey == "" { + return fmt.Errorf("Env variable MEX_CF_USER not set") + } + CloudletInfraCommon.DockerRegPass = os.Getenv("MEX_DOCKER_REG_PASS") + if CloudletInfraCommon.DockerRegPass == "" { + return fmt.Errorf("Env variable MEX_DOCKER_REG_PASS not set") + } + } + + switch CloudletInfra.CloudletKind { + case cloudcommon.CloudletKindOpenStack: + + CloudletInfra.OpenstackProperties.OSExternalNetworkName = os.Getenv("MEX_EXT_NETWORK") + if CloudletInfra.OpenstackProperties.OSExternalNetworkName == "" { + CloudletInfra.OpenstackProperties.OSExternalNetworkName = "external-network-shared" + } + + CloudletInfra.OpenstackProperties.OSImageName = os.Getenv("MEX_OS_IMAGE") + if CloudletInfra.OpenstackProperties.OSImageName == "" { + CloudletInfra.OpenstackProperties.OSImageName = "mobiledgex" + } + + // defaulting some value + CloudletInfra.OpenstackProperties.OSExternalRouterName = "mex-k8s-router-1" + CloudletInfra.OpenstackProperties.OSMexNetwork = "mex-k8s-net-1" + CloudletInfra.OpenstackProperties.OSNetworkScheme = "priv-subnet,mex-k8s-net-1,10.101.X.0/24" + + case cloudcommon.CloudletKindAzure: + CloudletInfra.AzureProperties.Location = os.Getenv("MEX_AZURE_LOCATION") + if CloudletInfra.AzureProperties.Location == "" { + return fmt.Errorf("Env variable MEX_AZURE_LOCATION not set") + } + + CloudletInfra.OpenstackProperties.OSImageName = os.Getenv("MEX_OS_IMAGE") + if CloudletInfra.OpenstackProperties.OSImageName == "" { + CloudletInfra.OpenstackProperties.OSImageName = "mobiledgex" + } + + case cloudcommon.CloudletKindGCP: + CloudletInfra.GcpProperties.Project = os.Getenv("MEX_GCP_LOCATION") + if CloudletInfra.GcpProperties.Project == "" { + //default + CloudletInfra.OpenstackProperties.OSImageName = "still-entity-201400" + } + CloudletInfra.GcpProperties.Zone = os.Getenv("MEX_GCP_ZONE") + if CloudletInfra.GcpProperties.Zone == "" { + return fmt.Errorf("Env variable MEX_GCP_ZONE not set") + } + } + // not supported yet but soon + CloudletInfra.MexosContainerImageName = "not-supported" + + CloudletInfraCommon.DNSZone = "mobiledgex.net" + CloudletInfraCommon.DockerRegistry = "registry.mobiledgex.net:5000" + CloudletInfraCommon.RegistryFileServer = "registry.mobiledgex.net" + + log.DebugLog(log.DebugLevelMexos, "InitializeCloudletInfra done", "CloudletInfra", CloudletInfra) + return nil +} + +func IsLocalDIND() bool { + return CloudletInfra.CloudletKind == cloudcommon.CloudletKindDIND +} + +func GetCloudletKind() string { + return CloudletInfra.CloudletKind +} + +func GetCloudletAzureLocation() string { + return CloudletInfra.AzureProperties.Location +} + +func GetCloudletGCPProject() string { + // default for now + return CloudletInfra.GcpProperties.Project +} + +func GetCloudletGCPZone() string { + // default for now + return CloudletInfra.GcpProperties.Zone +} + +//GetCloudletExternalRouter returns default MEX external router name +func GetCloudletExternalRouter() string { + //TODO validate existence and status + return CloudletInfra.OpenstackProperties.OSExternalRouterName +} + +func GetCloudletExternalNetwork() string { + return CloudletInfra.OpenstackProperties.OSExternalNetworkName +} + +// Utility functions that used to be within manifest. +//GetCloudletNetwork returns default MEX network, internal and prepped +func GetCloudletNetwork() string { + //TODO validate existence and status + return CloudletInfra.OpenstackProperties.OSMexNetwork +} + +func GetCloudletDNSZone() string { + return CloudletInfraCommon.DNSZone +} + +func GetCloudletNetworkScheme() string { + return CloudletInfra.OpenstackProperties.OSNetworkScheme +} + +func GetCloudletOSImage() string { + return CloudletInfra.OpenstackProperties.OSImageName +} + +func GetCloudletAgentContainerImage() string { + return CloudletInfra.MexosContainerImageName +} + +// todo: CRM supports only 1 registry +func GetCloudletDockerRegistry() string { + return CloudletInfraCommon.DockerRegistry +} + +func GetCloudletRegistryFileServer() string { + return CloudletInfraCommon.RegistryFileServer +} + +func GetCloudletCFKey() string { + return CloudletInfraCommon.CFKey +} + +func GetCloudletCFUser() string { + return CloudletInfraCommon.CFUser +} + +func GetCloudletDockerPass() string { + return CloudletInfraCommon.DockerRegPass +} + +// These not in the proto file yet because they may not change for a while +func GetCloudletTenant() string { + return "null" +} +func GetCloudletUserData() string { + return MEXDir() + "/userdata.txt" +} +func GetCloudletSecurityRule() string { + return "default" +} +func GetCloudletMexosAgentPort() string { + return "18889" +} +func GetCloudletRootLBFlavor() string { + return "x1.medium" +} diff --git a/mexos/cluster.go b/mexos/cluster.go index 133540cd8..979f0f8cd 100644 --- a/mexos/cluster.go +++ b/mexos/cluster.go @@ -6,6 +6,11 @@ import ( "strings" "time" + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/azure" + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/dind" + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/gcloud" + "github.com/mobiledgex/edge-cloud/cloudcommon" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) @@ -70,68 +75,69 @@ type ClusterMasterFlavor struct { } //mexCreateClusterKubernetes creates a cluster of nodes. It can take a while, so call from a goroutine. -func mexCreateClusterKubernetes(mf *Manifest) error { - //func mexCreateClusterKubernetes(mf *Manifest) (*string, error) { - //log.DebugLog(log.DebugLevelMexos, "create kubernetes cluster", "cluster metadata", mf.Metadata, "spec", mf.Spec) - rootLB, err := getRootLB(mf.Spec.RootLB) - if err != nil { - return err - } - if rootLB == nil { - return fmt.Errorf("can't create kubernetes cluster, rootLB is null") - } - if mf.Spec.Flavor == "" { +func mexCreateClusterKubernetes(clusterInst *edgeproto.ClusterInst, rootLBName string) error { + + log.DebugLog(log.DebugLevelMexos, "create kubernetes cluster", "cluster", clusterInst) + + nameSuffix := GetK8sNodeNameSuffix(clusterInst) + flavorName := clusterInst.Flavor.Name + tenantName := GetCloudletTenant() + clusterName := clusterInst.Key.ClusterKey.Name + + if flavorName == "" { return fmt.Errorf("empty cluster flavor") } - cf, err := GetClusterFlavor(mf.Spec.Flavor) + cf, err := GetClusterFlavor(flavorName) if err != nil { log.DebugLog(log.DebugLevelMexos, "invalid platform flavor, can't create cluster") return err } //TODO more than one networks - if mf.Spec.NetworkScheme == "" { + netScheme := GetCloudletNetworkScheme() + if netScheme == "" { return fmt.Errorf("empty network spec") } - if !strings.HasPrefix(mf.Spec.NetworkScheme, "priv-subnet") { - return fmt.Errorf("unsupported netSpec kind %s", mf.Spec.NetworkScheme) + if !strings.HasPrefix(netScheme, "priv-subnet") { + return fmt.Errorf("unsupported netSpec kind %s", netScheme) // XXX for now } + + //tags are derived from the cluster name currently + tags := clusterInst.Key.ClusterKey.Name + "-tag" //TODO allow more net types //TODO validate CIDR, etc. - if mf.Metadata.Tags == "" { - return fmt.Errorf("empty tag") - } - if mf.Metadata.Tenant == "" { - return fmt.Errorf("empty tenant") - } - err = ValidateTenant(mf.Metadata.Tenant) + + // this does nothing currently + err = ValidateTenant(tenantName) + if err != nil { return fmt.Errorf("can't validate tenant, %v", err) } - err = ValidateTags(mf.Metadata.Tags) + err = ValidateTags(tags) if err != nil { return fmt.Errorf("invalid tag, %v", err) } - mf.Metadata.Tags += "," + cf.PlatformFlavor + tags += "," + cf.PlatformFlavor //TODO add whole manifest yaml->json into stringified property of the kvm instance for later //XXX should check for quota, permissions, access control, etc. here //guid := xid.New().String() //kvmname := fmt.Sprintf("%s-1-%s-%s", "mex-k8s-master", mf.Metadata.Name, guid) - kvmname := fmt.Sprintf("%s-1-%s", "mex-k8s-master", mf.Metadata.Name) - sd, err := GetServerDetails(mf, kvmname) + kvmname := fmt.Sprintf("%s-1-%s", "mex-k8s-master", nameSuffix) + sd, err := GetServerDetails(kvmname) if err == nil { if sd.Name == kvmname { log.DebugLog(log.DebugLevelMexos, "k8s master exists", "kvmname", kvmname) return nil } } - log.DebugLog(log.DebugLevelMexos, "proceed to create k8s master kvm", "kvmname", kvmname, "netspec", mf.Spec.NetworkScheme, "tags", mf.Metadata.Tags, "flavor", cf.PlatformFlavor) - err = CreateMEXKVM(mf, kvmname, + log.DebugLog(log.DebugLevelMexos, "proceed to create k8s master kvm", "kvmname", kvmname, "netspec", netScheme, "tags", tags, "flavor", flavorName) + err = CreateMEXKVM(kvmname, "k8s-master", - mf.Spec.NetworkScheme, - mf.Metadata.Tags, - mf.Metadata.Tenant, + netScheme, + tags, + tenantName, 1, + clusterInst, cf.PlatformFlavor, ) if err != nil { @@ -140,32 +146,33 @@ func mexCreateClusterKubernetes(mf *Manifest) error { for i := 1; i <= cf.NumNodes; i++ { //construct node name //kvmnodename := fmt.Sprintf("%s-%d-%s-%s", "mex-k8s-node", i, mf.Metadata.Name, guid) - kvmnodename := fmt.Sprintf("%s-%d-%s", "mex-k8s-node", i, mf.Metadata.Name) - err = CreateMEXKVM(mf, kvmnodename, + kvmnodename := fmt.Sprintf("%s-%d-%s", "mex-k8s-node", i, nameSuffix) + err = CreateMEXKVM(kvmnodename, "k8s-node", - mf.Spec.NetworkScheme, - mf.Metadata.Tags, - mf.Metadata.Tenant, + netScheme, + tags, + tenantName, i, + clusterInst, cf.PlatformFlavor, ) if err != nil { return fmt.Errorf("can't create k8s node, %v", err) } } - if mf.Values.Network.External == "" { + if GetCloudletExternalNetwork() == "" { return fmt.Errorf("missing external network in platform config") } - if err = LBAddRoute(mf, rootLB.Name, mf.Values.Network.External, kvmname); err != nil { + if err = LBAddRoute(rootLBName, GetCloudletExternalNetwork(), kvmname); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot add route on rootlb", "error", err) //return err } - if err = SetServerProperty(mf, kvmname, "mex-flavor="+mf.Spec.Flavor); err != nil { + if err = SetServerProperty(kvmname, "mex-flavor="+flavorName); err != nil { return err } ready := false for i := 0; i < 10; i++ { - ready, err = IsClusterReady(mf, rootLB) + ready, err = IsClusterReady(clusterInst, flavorName, rootLBName) if err != nil { return err } @@ -179,16 +186,18 @@ func mexCreateClusterKubernetes(mf *Manifest) error { if !ready { return fmt.Errorf("cluster not ready (yet)") } - if err := SeedDockerSecret(mf, rootLB); err != nil { + if err := SeedDockerSecret(clusterName, rootLBName); err != nil { return err } + /* TODO: Swarm is not integrated with the controller, needs work here. if mf.Metadata.Swarm != "" { - log.DebugLog(log.DebugLevelMexos, "metadata swarm is set, creating docker swarm", "swarm", mf.Metadata.Swarm) - if err := CreateDockerSwarm(mf, rootLB); err != nil { + log.DebugLog(log.DebugLevelMexos, "metadata swarm is set, creating docker swarm", "swarm") + if err := CreateDockerSwarm(clusterName, rootLBName); err != nil { return err } } - if err := CreateDockerRegistrySecret(mf); err != nil { + */ + if err := CreateDockerRegistrySecret(clusterInst, rootLBName); err != nil { return err } //return &guid, nil @@ -196,21 +205,23 @@ func mexCreateClusterKubernetes(mf *Manifest) error { } //mexDeleteClusterKubernetes deletes kubernetes cluster -func mexDeleteClusterKubernetes(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "deleting kubernetes cluster") - rootLB, err := getRootLB(mf.Spec.RootLB) +func mexDeleteClusterKubernetes(clusterInst *edgeproto.ClusterInst, rootLBName string) error { + log.DebugLog(log.DebugLevelMexos, "deleting kubernetes cluster", "clusterInst", clusterInst) + nameSuffix := GetK8sNodeNameSuffix(clusterInst) + + rootLB, err := getRootLB(rootLBName) if err != nil { return err } if rootLB == nil { return fmt.Errorf("can't delete kubernetes cluster, rootLB is null") } - name := mf.Metadata.Name - if name == "" { + + if nameSuffix == "" { log.DebugLog(log.DebugLevelMexos, "error, empty name") return fmt.Errorf("empty name") } - srvs, err := ListServers(mf) + srvs, err := ListServers() if err != nil { return err } @@ -219,15 +230,15 @@ func mexDeleteClusterKubernetes(mf *Manifest) error { // return fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) // } //log.DebugLog(log.DebugLevelMexos, "looking for server", "name", name, "servers", srvs) - force := strings.Contains(mf.Spec.Flags, "force") + force := true // TODO: consider a way to specify this serverDeleted := false for _, s := range srvs { - if strings.Contains(s.Name, name) { + if strings.Contains(s.Name, nameSuffix) { if !strings.HasPrefix(s.Name, "mex-k8s-") { continue } if strings.Contains(s.Name, "mex-k8s-master") { - err = LBRemoveRoute(mf, rootLB.Name, mf.Values.Network.External, s.Name) + err = LBRemoveRoute(rootLB.Name, GetCloudletExternalNetwork(), s.Name) if err != nil { if !force { err = fmt.Errorf("failed to remove route for %s, %v", s.Name, err) @@ -238,7 +249,7 @@ func mexDeleteClusterKubernetes(mf *Manifest) error { } } log.DebugLog(log.DebugLevelMexos, "delete kubernetes server", "name", s.Name) - err = DeleteServer(mf, s.Name) + err = DeleteServer(s.Name) if err != nil { if !force { log.DebugLog(log.DebugLevelMexos, "delete server fail", "error", err) @@ -256,23 +267,23 @@ func mexDeleteClusterKubernetes(mf *Manifest) error { } } if !serverDeleted { - log.DebugLog(log.DebugLevelMexos, "server not found", "name", name) - return fmt.Errorf("no server with name %s", name) + log.DebugLog(log.DebugLevelMexos, "server not found", "name", nameSuffix) + return fmt.Errorf("no server with name %s", nameSuffix) } - sns, err := ListSubnets(mf, "") + sns, err := ListSubnets("") if err != nil { log.DebugLog(log.DebugLevelMexos, "can't list subnets", "error", err) return err } - rn := GetMEXExternalRouter(mf) //XXX for now + rn := GetCloudletExternalRouter() //XXX for now for _, s := range sns { - if strings.Contains(s.Name, name) { - rerr := RemoveRouterSubnet(mf, rn, s.Name) + if strings.Contains(s.Name, nameSuffix) { + rerr := RemoveRouterSubnet(rn, s.Name) if rerr != nil { log.DebugLog(log.DebugLevelMexos, "not fatal, continue, can't remove router from subnet", "error", rerr) //return rerr } - err = DeleteSubnet(mf, s.Name) + err = DeleteSubnet(s.Name) if err != nil { log.DebugLog(log.DebugLevelMexos, "warning, problems deleting subnet", "error", err) } @@ -289,28 +300,27 @@ func mexDeleteClusterKubernetes(mf *Manifest) error { } //IsClusterReady checks to see if cluster is read, i.e. rootLB is running and active -func IsClusterReady(mf *Manifest, rootLB *MEXRootLB) (bool, error) { +func IsClusterReady(clusterInst *edgeproto.ClusterInst, flavorName, rootLBName string) (bool, error) { log.DebugLog(log.DebugLevelMexos, "checking if cluster is ready") - if rootLB == nil { - return false, fmt.Errorf("cannot check if cluster is ready, rootLB is null") - } - cf, err := GetClusterFlavor(mf.Spec.Flavor) + + nameSuffix := GetK8sNodeNameSuffix(clusterInst) + cf, err := GetClusterFlavor(flavorName) if err != nil { log.DebugLog(log.DebugLevelMexos, "invalid cluster flavor, can't check if cluster is ready") return false, err } - name, err := FindClusterWithKey(mf, mf.Spec.Key) + master, err := FindClusterMaster(nameSuffix) if err != nil { - return false, fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) + return false, fmt.Errorf("can't find cluster with name %s, %v", nameSuffix, err) } - ipaddr, err := FindNodeIP(mf, name) + ipaddr, err := FindNodeIP(master) if err != nil { return false, err } - if mf.Values.Network.External == "" { + if GetCloudletExternalNetwork() == "" { return false, fmt.Errorf("is cluster ready, missing external network in platform config") } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLBName, GetCloudletExternalNetwork(), sshUser) if err != nil { return false, fmt.Errorf("can't get ssh client for cluser ready check, %v", err) } @@ -335,7 +345,7 @@ func IsClusterReady(mf *Manifest, rootLB *MEXRootLB) (bool, error) { } log.DebugLog(log.DebugLevelMexos, "cluster nodes", "numnodes", cf.NumNodes, "nummasters", cf.NumMasterNodes) //kcpath := MEXDir() + "/" + name[strings.LastIndex(name, "-")+1:] + ".kubeconfig" - if err := CopyKubeConfig(mf, rootLB, name); err != nil { + if err := CopyKubeConfig(clusterInst, rootLBName, master); err != nil { return false, fmt.Errorf("kubeconfig copy failed, %v", err) } log.DebugLog(log.DebugLevelMexos, "cluster ready.") @@ -343,12 +353,12 @@ func IsClusterReady(mf *Manifest, rootLB *MEXRootLB) (bool, error) { } //FindClusterWithKey finds cluster given a key string -func FindClusterWithKey(mf *Manifest, key string) (string, error) { +func FindClusterMaster(key string) (string, error) { //log.DebugLog(log.DebugLevelMexos, "find cluster with key", "key", key) if key == "" { return "", fmt.Errorf("empty key") } - srvs, err := ListServers(mf) + srvs, err := ListServers() if err != nil { return "", err } @@ -360,3 +370,52 @@ func FindClusterWithKey(mf *Manifest, key string) (string, error) { } return "", fmt.Errorf("key %s not found", key) } + +//MEXClusterCreateInst creates a cluster +func MEXClusterCreateClustInst(clusterInst *edgeproto.ClusterInst, rootLBName string) error { + log.DebugLog(log.DebugLevelMexos, "creating cluster instance", "clusterInst", clusterInst, "rootLBName", rootLBName) + if IsLocalDIND() { + return localCreateDIND(clusterInst) + } + operatorName := NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name) + + switch operatorName { + case cloudcommon.OperatorGCP: + return gcloudCreateGKE(clusterInst) + case cloudcommon.OperatorAzure: + return azureCreateAKS(clusterInst) + default: + //guid, err := mexCreateClusterKubernetes(mf) + err := mexCreateClusterKubernetes(clusterInst, rootLBName) + if err != nil { + return fmt.Errorf("can't create cluster, %v", err) + } + //log.DebugLog(log.DebugLevelMexos, "new guid", "guid", *guid) + log.DebugLog(log.DebugLevelMexos, "created kubernetes cluster") + return nil + } +} + +func MEXClusterRemoveClustInst(clusterInst *edgeproto.ClusterInst, rootLBName string) error { + log.DebugLog(log.DebugLevelMexos, "removing cluster") + + clusterName := clusterInst.Key.ClusterKey.Name + + if IsLocalDIND() { + return dind.DeleteDINDCluster(clusterName) + } + operatorName := NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name) + + switch operatorName { + case cloudcommon.OperatorGCP: + return gcloud.DeleteGKECluster(clusterInst.Key.ClusterKey.Name) + case cloudcommon.OperatorAzure: + resourceGroup := GetResourceGroupForCluster(clusterInst) + return azure.DeleteAKSCluster(resourceGroup) + default: + if err := mexDeleteClusterKubernetes(clusterInst, rootLBName); err != nil { + return fmt.Errorf("can't remove cluster, %v", err) + } + return nil + } +} diff --git a/mexos/dns.go b/mexos/dns.go index a89be4f53..4ed9554fe 100644 --- a/mexos/dns.go +++ b/mexos/dns.go @@ -16,45 +16,37 @@ import ( var dnsRegisterRetryDelay time.Duration = 3 * time.Second -func createAppDNS(mf *Manifest, kconf string) error { +func createAppDNS(kconf string, uri string, name string) error { log.DebugLog(log.DebugLevelMexos, "createAppDNS") - if mf.Metadata.Operator != "gcp" && mf.Metadata.Operator != "azure" && !IsLocalDIND(mf) { - return fmt.Errorf("error, invalid code path") - } - if err := CheckCredentialsCF(mf); err != nil { - return err - } - if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { + if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { return fmt.Errorf("cannot init cloudflare api, %v", err) } - if mf.Spec.URI == "" { - return fmt.Errorf("URI not specified %v", mf) + if uri == "" { + return fmt.Errorf("URI not specified") } - err := validateDomain(mf.Spec.URI) + err := validateDomain(uri) if err != nil { return err } - if mf.Metadata.DNSZone == "" { - return fmt.Errorf("missing DNS zone, metadata %v", mf.Metadata) - } - serviceNames, err := getSvcNames(mf.Metadata.Name, kconf) + + serviceNames, err := getSvcNames(name, kconf) if err != nil { return err } if len(serviceNames) < 1 { - return fmt.Errorf("no service names starting with %s", mf.Metadata.Name) + return fmt.Errorf("no service names starting with %s", name) } - recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + recs, derr := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if derr != nil { - return fmt.Errorf("error getting dns records for %s, %v", mf.Metadata.DNSZone, err) + return fmt.Errorf("error getting dns records for %s, %v", GetCloudletDNSZone(), err) } - fqdnBase := uri2fqdn(mf.Spec.URI) + fqdnBase := uri2fqdn(uri) for _, sn := range serviceNames { // for the DIND case we need to patch the service here externalIP := "" - if IsLocalDIND(mf) { + if IsLocalDIND() { addr := dind.GetMasterAddr() err = KubePatchServiceLocal(sn, addr) if err != nil { @@ -70,13 +62,13 @@ func createAppDNS(mf *Manifest, kconf string) error { fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) for _, rec := range recs { if rec.Type == "A" && rec.Name == fqdn { - if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID); err != nil { + if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), rec.ID); err != nil { return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) } log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) } } - if err := cloudflare.CreateDNSRecord(mf.Metadata.DNSZone, fqdn, "A", externalIP, 1, false); err != nil { + if err := cloudflare.CreateDNSRecord(GetCloudletDNSZone(), fqdn, "A", externalIP, 1, false); err != nil { return fmt.Errorf("can't create DNS record for %s,%s, %v", fqdn, externalIP, err) } //log.DebugLog(log.DebugLevelMexos, "waiting for DNS record to be created on cloudflare...") @@ -89,43 +81,35 @@ func createAppDNS(mf *Manifest, kconf string) error { return nil } -func deleteAppDNS(mf *Manifest, kconf string) error { - if mf.Metadata.Operator != "gcp" && mf.Metadata.Operator != "azure" { - return fmt.Errorf("error, invalid code path") - } - if err := CheckCredentialsCF(mf); err != nil { - return err - } - if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { +func deleteAppDNS(kconf string, uri string, name string) error { + + if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { return fmt.Errorf("cannot init cloudflare api, %v", err) } - if mf.Spec.URI == "" { - return fmt.Errorf("URI not specified %v", mf) + if uri == "" { + return fmt.Errorf("URI not specified") } - err := validateDomain(mf.Spec.URI) + err := validateDomain(uri) if err != nil { return err } - if mf.Metadata.DNSZone == "" { - return fmt.Errorf("missing DNS zone, metadata %v", mf.Metadata) - } - serviceNames, err := getSvcNames(mf.Metadata.Name, kconf) + serviceNames, err := getSvcNames(name, kconf) if err != nil { return err } if len(serviceNames) < 1 { - return fmt.Errorf("no service names starting with %s", mf.Metadata.Name) + return fmt.Errorf("no service names starting with %s", name) } - recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + recs, derr := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if derr != nil { - return fmt.Errorf("cannot get dns records for dns zone %s, error %v", mf.Metadata.DNSZone, err) + return fmt.Errorf("cannot get dns records for dns zone %s, error %v", GetCloudletDNSZone(), err) } - fqdnBase := uri2fqdn(mf.Spec.URI) + fqdnBase := uri2fqdn(uri) for _, sn := range serviceNames { fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) for _, rec := range recs { if rec.Type == "A" && rec.Name == fqdn { - if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID); err != nil { + if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), rec.ID); err != nil { return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) } } @@ -150,16 +134,9 @@ func KubePatchServiceLocal(servicename string, ipaddr string) error { return nil } -// TODO: This function and createAppDNS share a lot of duplicate code, -// but are subtly different. It'd be good to consolidate and remove -// duplicate code and highlight what the different use cases are, -// since it's not clear when to use one or the other. -// This should be easier to consolidate now that kubeParam can issue -// commands locally for DIND or other cases. -// Same for KubeDeleteDNSRecords and deleteAppDNS. -func KubeAddDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error { - log.DebugLog(log.DebugLevelMexos, "adding dns records for kubenernets app", "name", mf.Metadata.Name) - rootLBIPaddr, err := GetServerIPAddr(mf, mf.Values.Network.External, rootLB.Name) +func KubeAddDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name string) error { + log.DebugLog(log.DebugLevelMexos, "adding dns records for kubenernets app", "name", name) + rootLBIPaddr, err := GetServerIPAddr(GetCloudletExternalNetwork(), rootLB.Name) if err != nil { log.DebugLog(log.DebugLevelMexos, "cannot get rootlb IP address", "error", err) return fmt.Errorf("cannot deploy kubernetes app, cannot get rootlb IP") @@ -174,19 +151,19 @@ func KubeAddDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error { if err != nil { return fmt.Errorf("can not unmarshal svc json, %v", err) } - if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { + if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { return fmt.Errorf("cannot init cloudflare api, %v", err) } - recs, err := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + recs, err := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if err != nil { - return fmt.Errorf("error getting dns records for %s, %v", mf.Metadata.DNSZone, err) + return fmt.Errorf("error getting dns records for %s, %v", GetCloudletDNSZone(), err) } log.DebugLog(log.DebugLevelMexos, "number of cloudflare dns recs", "dns recs count", len(recs)) - fqdnBase := uri2fqdn(mf.Spec.URI) + fqdnBase := uri2fqdn(uri) log.DebugLog(log.DebugLevelMexos, "kubernetes services", "services", svcs) processed := 0 for _, item := range svcs.Items { - if !strings.HasPrefix(item.Metadata.Name, mf.Metadata.Name) { + if !strings.HasPrefix(item.Metadata.Name, name) { continue } cmd = fmt.Sprintf(`%s kubectl patch svc %s -p '{"spec":{"externalIPs":["%s"]}}'`, kp.kubeconfig, item.Metadata.Name, kp.ipaddr) @@ -198,25 +175,25 @@ func KubeAddDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error { fqdn := cloudcommon.ServiceFQDN(item.Metadata.Name, fqdnBase) for _, rec := range recs { if rec.Type == "A" && rec.Name == fqdn { - if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID); err != nil { + if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), rec.ID); err != nil { return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) } log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) } } - if err := cloudflare.CreateDNSRecord(mf.Metadata.DNSZone, fqdn, "A", rootLBIPaddr, 1, false); err != nil { + if err := cloudflare.CreateDNSRecord(GetCloudletDNSZone(), fqdn, "A", rootLBIPaddr, 1, false); err != nil { return fmt.Errorf("can't create DNS record for %s,%s, %v", fqdn, rootLBIPaddr, err) } processed++ log.DebugLog(log.DebugLevelMexos, "created DNS record", "name", fqdn, "addr", rootLBIPaddr) } if processed == 0 { - return fmt.Errorf("cannot patch service, %s not found", mf.Metadata.Name) + return fmt.Errorf("cannot patch service, %s not found", name) } return nil } -func KubeDeleteDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error { +func KubeDeleteDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name string) error { //TODO before removing dns records, especially for the purpose of creating // a dns entry that was there before, to overwrite, we need to check // if the user really wants to. For example, if the cluster naming was in error, @@ -231,17 +208,17 @@ func KubeDeleteDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error if err != nil { return fmt.Errorf("can not unmarshal svc json, %v", err) } - if cerr := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); cerr != nil { + if cerr := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); cerr != nil { return fmt.Errorf("cannot init cloudflare api, %v", cerr) } - recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + recs, derr := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if derr != nil { - return fmt.Errorf("error getting dns records for %s, %v", mf.Metadata.DNSZone, derr) + return fmt.Errorf("error getting dns records for %s, %v", GetCloudletDNSZone(), derr) } - fqdnBase := uri2fqdn(mf.Spec.URI) + fqdnBase := uri2fqdn(uri) //FIXME use k8s manifest file to delete the whole services and deployments for _, item := range svcs.Items { - if !strings.HasPrefix(item.Metadata.Name, mf.Metadata.Name) { + if !strings.HasPrefix(item.Metadata.Name, name) { continue } // cmd := fmt.Sprintf("%s kubectl delete service %s", kp.kubeconfig, item.Metadata.Name) @@ -254,18 +231,17 @@ func KubeDeleteDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error fqdn := cloudcommon.ServiceFQDN(item.Metadata.Name, fqdnBase) for _, rec := range recs { if rec.Type == "A" && rec.Name == fqdn { - if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID); err != nil { + if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), rec.ID); err != nil { return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) } log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) } } } - cmd = fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, mf.Metadata.Name) - // cmd = fmt.Sprintf("%s kubectl delete deploy %s", kp.kubeconfig, mf.Metadata.Name+"-deployment") + cmd = fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, name) out, err = kp.client.Output(cmd) if err != nil { - return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", mf.Metadata.Name, cmd, out, err) + return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", name, cmd, out, err) } return nil diff --git a/mexos/env.go b/mexos/env.go index d958e8067..3d1cd86e2 100644 --- a/mexos/env.go +++ b/mexos/env.go @@ -1,11 +1,6 @@ package mexos -import ( - "fmt" - - "github.com/mobiledgex/edge-cloud/log" -) - +/* //MEXCheckEnvVars sets up environment vars and checks for credentials required for running func MEXCheckEnvVars(mf *Manifest) error { // secrets to be passed via Env var still : MEX_CF_KEY, MEX_CF_USER, MEX_DOCKER_REG_PASS @@ -22,6 +17,7 @@ func MEXCheckEnvVars(mf *Manifest) error { return nil } + func mexEnv(mf *Manifest, name string) string { v, ok := mf.Values.VaultEnvMap[name] if !ok { @@ -30,3 +26,5 @@ func mexEnv(mf *Manifest, name string) string { } return v } + +*/ diff --git a/mexos/flavor.go b/mexos/flavor.go index e6e2ffd64..4bcd39ed8 100644 --- a/mexos/flavor.go +++ b/mexos/flavor.go @@ -62,16 +62,6 @@ var AvailableClusterFlavors = []*ClusterFlavor{ }, } -func AddFlavorManifest(mf *Manifest) error { - _, err := GetClusterFlavor(mf.Spec.Flavor) - if err != nil { - return err - } - // Adding flavors in platforms cannot be done dynamically. For example, x1.xlarge cannot be - // implemented in currently DT cloudlets. Controller can learn what flavors available. Not create new ones. - return nil -} - func GetClusterFlavor(flavor string) (*ClusterFlavor, error) { log.DebugLog(log.DebugLevelMexos, "get cluster flavor details", "cluster flavor", flavor) for _, af := range AvailableClusterFlavors { diff --git a/mexos/fqdn.go b/mexos/fqdn.go index 05ed3ed98..f6cec24d4 100644 --- a/mexos/fqdn.go +++ b/mexos/fqdn.go @@ -61,26 +61,23 @@ func uri2fqdn(uri string) string { } //ActivateFQDNA updates and ensures FQDN is registered properly -func ActivateFQDNA(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { +func ActivateFQDNA(rootLB *MEXRootLB, fqdn string) error { if rootLB == nil { return fmt.Errorf("cannot activate certs, rootLB is null") } - if mf.Values.Network.External == "" { + if GetCloudletExternalNetwork() == "" { return fmt.Errorf("activate fqdn A record, missing external network in manifest") } - if err := CheckCredentialsCF(mf); err != nil { - return err - } - if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { + if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { return fmt.Errorf("cannot init cloudflare api, %v", err) } - log.DebugLog(log.DebugLevelMexos, "getting dns record for zone", "DNSZone", mf.Metadata.DNSZone) - dr, err := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + log.DebugLog(log.DebugLevelMexos, "getting dns record for zone", "DNSZone", GetCloudletDNSZone()) + dr, err := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if err != nil { return fmt.Errorf("cannot get dns records for %s, %v", fqdn, err) } - addr, err := GetServerIPAddr(mf, mf.Values.Network.External, fqdn) + addr, err := GetServerIPAddr(GetCloudletExternalNetwork(), fqdn) for _, d := range dr { if d.Type == "A" && d.Name == fqdn { if d.Content == addr { @@ -88,7 +85,7 @@ func ActivateFQDNA(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { return nil } log.DebugLog(log.DebugLevelMexos, "cloudflare A record has different address, it will be overwritten", "existing", d, "addr", addr) - if err = cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, d.ID); err != nil { + if err = cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), d.ID); err != nil { return fmt.Errorf("can't delete DNS record for %s, %v", fqdn, err) } break @@ -98,7 +95,7 @@ func ActivateFQDNA(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { log.DebugLog(log.DebugLevelMexos, "error while talking to cloudflare", "error", err) return err } - if err := cloudflare.CreateDNSRecord(mf.Metadata.DNSZone, fqdn, "A", addr, 1, false); err != nil { + if err := cloudflare.CreateDNSRecord(GetCloudletDNSZone(), fqdn, "A", addr, 1, false); err != nil { return fmt.Errorf("can't create DNS record for %s, %v", fqdn, err) } log.DebugLog(log.DebugLevelMexos, "waiting for cloudflare...") diff --git a/mexos/gcloud.go b/mexos/gcloud.go index 7cf93dce8..7e092e56c 100644 --- a/mexos/gcloud.go +++ b/mexos/gcloud.go @@ -5,39 +5,39 @@ import ( "time" "github.com/mobiledgex/edge-cloud-infra/k8s-prov/gcloud" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) -var GCPDefaultProjectID = "still-entity-201400" // XXX - -func gcloudCreateGKE(mf *Manifest) error { +func gcloudCreateGKE(clusterInst *edgeproto.ClusterInst) error { var err error - if mf.Metadata.Project == "" { - log.DebugLog(log.DebugLevelMexos, "warning, empty gcp project ID, using default", "default", GCPDefaultProjectID) - } - if err = gcloud.SetProject(mf.Metadata.Project); err != nil { + project := GetCloudletGCPProject() + zone := GetCloudletGCPZone() + clusterName := clusterInst.Key.ClusterKey.Name + + if err = gcloud.SetProject(project); err != nil { return err } - if err = gcloud.SetZone(mf.Metadata.Zone); err != nil { + if err = gcloud.SetZone(zone); err != nil { return err } - if err = gcloud.CreateGKECluster(mf.Metadata.Name); err != nil { + if err = gcloud.CreateGKECluster(clusterName); err != nil { return err } //race condition exists where the config file is not ready until just after the cluster create is done time.Sleep(3 * time.Second) saveKubeconfig() - if err = gcloud.GetGKECredentials(mf.Metadata.Name); err != nil { + if err = gcloud.GetGKECredentials(clusterName); err != nil { return err } - kconf, err := GetKconf(mf, false) //XXX + kconf, err := GetKconf(clusterInst, false) //XXX if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v, %v", mf, kconf, err) + return fmt.Errorf("cannot get kconf, %v, %v", clusterInst, err) } log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX if err = copyFile(defaultKubeconfig(), kconf); err != nil { return fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) } - log.DebugLog(log.DebugLevelMexos, "created gke", "name", mf.Spec.Key) - return CreateDockerRegistrySecret(mf) + log.DebugLog(log.DebugLevelMexos, "created gke", "name", clusterName) + return CreateDockerRegistrySecret(clusterInst, "") } diff --git a/mexos/helm.go b/mexos/helm.go index f138b4ae8..5367f87bd 100644 --- a/mexos/helm.go +++ b/mexos/helm.go @@ -2,48 +2,30 @@ package mexos import ( "fmt" - "regexp" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) -func DeleteHelmAppManifest(mf *Manifest) error { +func DeleteHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { log.DebugLog(log.DebugLevelMexos, "delete kubernetes helm app") - rootLB, err := getRootLB(mf.Spec.RootLB) + + clusterName := clusterInst.Key.ClusterKey.Name + appName := NormalizeName(app.Key.Name) + + kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) if err != nil { return err } - if rootLB == nil { - return fmt.Errorf("cannot delete helm app, rootLB is null") - } - if err = ValidateCommon(mf); err != nil { - return err - } - kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) - if err != nil { - return err + // remove DNS entries + if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) } - if IsLocalDIND(mf) { - // remove DNS entries - kconf, err := GetKconf(mf, false) - if err != nil { - return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err) - } - if err = deleteAppDNS(mf, kconf); err != nil { - log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) - } - } else { - // remove DNS entries - if err = KubeDeleteDNSRecords(rootLB, mf, kp); err != nil { - log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) - } - // remove Security rules - if err = DeleteProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { - log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) - } + // remove Security rules + if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) } - - cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, mf.Metadata.Name) + cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, appName) out, err := kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deleting helm chart, %s, %s, %v", cmd, out, err) @@ -52,19 +34,14 @@ func DeleteHelmAppManifest(mf *Manifest) error { return nil } -func CreateHelmAppManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "create kubernetes helm app") - rootLB, err := getRootLB(mf.Spec.RootLB) - if err != nil { - return err - } - if rootLB == nil { - return fmt.Errorf("cannot create helm app, rootLB is null") - } - if err = ValidateCommon(mf); err != nil { - return err - } - kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) +func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { + log.DebugLog(log.DebugLevelMexos, "create kubernetes helm app", "clusterInst", clusterInst, "appInst", appInst) + + clusterName := clusterInst.Key.ClusterKey.Name + appName := NormalizeName(app.Key.Name) + appImage := app.ImagePath + + kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) if err != nil { return err } @@ -98,43 +75,21 @@ func CreateHelmAppManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "helm tiller initialized") } - helmOpts := "" - // XXX This gets helm's prometheus able to query kubelet metrics. - // This can be removed once Lev passes in an option in the yaml to - // set the helm command line options. - prom, err := regexp.MatchString("prometheus", mf.Metadata.Name) - if err == nil && prom { - log.DebugLog(log.DebugLevelMexos, "setting helm prometheus option") - helmOpts = "--set kubelet.serviceMonitor.https=true" - } - cmd = fmt.Sprintf("%s helm install %s --name %s %s", kp.kubeconfig, mf.Spec.Image, mf.Metadata.Name, helmOpts) + cmd = fmt.Sprintf("%s helm install %s --name %s", kp.kubeconfig, appImage, appName) out, err = kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deploying helm chart, %s, %s, %v", cmd, out, err) } log.DebugLog(log.DebugLevelMexos, "applied helm chart") - if IsLocalDIND(mf) { - kconf, err := GetKconf(mf, false) - if err != nil { - return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err) - } - // Add DNS Zone - if err = createAppDNS(mf, kconf); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) - return err - } - } else { - // Add security rules - if err = AddProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) - return err - } - log.DebugLog(log.DebugLevelMexos, "add spec ports", "ports", mf.Spec.Ports) - // Add DNS Zone - if err = KubeAddDNSRecords(rootLB, mf, kp); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) - return err - } + // Add security rules + if err = AddProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) + return err + } + + if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) + return err } return nil } diff --git a/mexos/init.go b/mexos/init.go index 0028e94de..88c9c9a38 100644 --- a/mexos/init.go +++ b/mexos/init.go @@ -1,9 +1,6 @@ package mexos -import ( - "github.com/mobiledgex/edge-cloud/log" -) - +/* //MEXInit initializes MEX API func MEXInit(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "mex init") @@ -11,4 +8,4 @@ func MEXInit(mf *Manifest) error { return err } return nil -} +}*/ diff --git a/mexos/inst.go b/mexos/inst.go index 03c673b73..62500a4a1 100644 --- a/mexos/inst.go +++ b/mexos/inst.go @@ -3,21 +3,13 @@ package mexos import ( "fmt" + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/dind" + "github.com/mobiledgex/edge-cloud/cloudcommon" "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) -//MEXClusterCreateClustInst calls MEXClusterCreate with a manifest created from the template -func MEXClusterCreateClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) error { - //XXX trigger off clusterInst or flavor to pick the right template: mex, aks, gke - mf, err := FillClusterTemplateClustInst(rootLB, clusterInst) - if err != nil { - return err - } - fixValuesInst(mf, rootLB) - return MEXClusterCreateManifest(mf) -} - +/* //MEXClusterRemoveClustInst calls MEXClusterRemove with a manifest created from the template func MEXClusterRemoveClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) error { mf, err := FillClusterTemplateClustInst(rootLB, clusterInst) @@ -27,7 +19,8 @@ func MEXClusterRemoveClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.Cluster fixValuesInst(mf, rootLB) return MEXClusterRemoveManifest(mf) } - +*/ +/* func FillClusterTemplateClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) (*Manifest, error) { log.DebugLog(log.DebugLevelMexos, "fill cluster template manifest cluster inst", "clustinst", clusterInst) if clusterInst.Key.ClusterKey.Name == "" { @@ -49,63 +42,21 @@ func FillClusterTemplateClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.Clus Kind: vp.Cluster.Kind, //"kubernetes", ResourceGroup: clusterInst.Key.CloudletKey.Name + "_" + clusterInst.Key.ClusterKey.Name, Flavor: clusterInst.Flavor.Name, - DNSZone: vp.Network.DNSZone, //"mobiledgex.net", + DNSZone: GetCloudletDNSZone(), RootLB: rootLB.Name, - Region: vp.Cluster.Region, //us-west1 - Zone: vp.Cluster.Zone, //us-west1a - Location: vp.Cluster.Location, // us-west - NetworkScheme: vp.Network.Scheme, //"priv-subnet,mex-k8s-net-1,10.101.X.0/24", + NetworkScheme: GetCloudletNetworkScheme(), Swarm: vp.Cluster.Swarm, } - // // if these env variables are not set, fall back to the - // // existing defaults based on deployment type(operator name) - // data.Region = os.Getenv("CLOUDLET_REGION") - // data.Zone = os.Getenv("CLOUDLET_ZONE") - // data.Location = os.Getenv("CLOUDLET_LOCATION") - - // switch clusterInst.Key.CloudletKey.OperatorKey.Name { - // case "gcp": - // if data.Region == "" { - // data.Region = "us-west1" - // } - // if data.Zone == "" { - // data.Zone = "us-west1-a" - // } - // if data.Location == "" { - // data.Location = "us-west" - // } - // data.Project = "still-entity-201400" // XXX - // case "azure": - // if data.Region == "" { - // data.Region = "centralus" - // } - // if data.Zone == "" { - // data.Zone = "centralus" - // } - // if data.Location == "" { - // data.Location = "centralus" - // } - // default: - // if data.Region == "" { - // data.Region = "eu-central-1" - // } - // if data.Zone == "" { - // data.Zone = "eu-central-1c" - // } - // if data.Location == "" { - // data.Location = "buckhorn" - // } - // } - mf, err := templateUnmarshal(&data, yamlMEXCluster) if err != nil { return nil, err } fixValuesInst(mf, rootLB) return mf, nil -} +} */ +/* this function never actually did anything func MEXAddFlavorClusterInst(rootLB *MEXRootLB, flavor *edgeproto.ClusterFlavor) error { log.DebugLog(log.DebugLevelMexos, "adding cluster inst flavor", "flavor", flavor) @@ -123,11 +74,9 @@ func MEXAddFlavorClusterInst(rootLB *MEXRootLB, flavor *edgeproto.ClusterFlavor) Flags: flavor.Key.Name + "-flags", NumNodes: int(flavor.NumNodes), NumMasters: int(flavor.NumMasters), - NetworkScheme: vp.Network.Scheme, - MasterFlavor: flavor.MasterFlavor.Name, + NetworkScheme: GetCloudletNetworkScheme(), NodeFlavor: flavor.NodeFlavor.Name, StorageSpec: "default", //XXX - Topology: "type-1", //XXX } mf, err := templateUnmarshal(&data, yamlMEXFlavor) if err != nil { @@ -136,31 +85,125 @@ func MEXAddFlavorClusterInst(rootLB *MEXRootLB, flavor *edgeproto.ClusterFlavor) fixValuesInst(mf, rootLB) return MEXAddFlavor(mf) } +*/ -//MEXAppCreateAppInst creates app inst with templated manifest -func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, app *edgeproto.App) error { +//MEXAppDeleteAppInst deletes app with templated manifest +func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, app *edgeproto.App, appInst *edgeproto.AppInst) error { log.DebugLog(log.DebugLevelMexos, "mex create app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) - mf, err := fillAppTemplate(rootLB, appInst, app, clusterInst) - if err != nil { - log.DebugLog(log.DebugLevelMexos, "fillAppTemplate error", "error", err) - return err + + appDeploymentType := app.Deployment + clusterName := clusterInst.Key.ClusterKey.Name + appName := NormalizeName(app.Key.Name) + operatorName := NormalizeName(appInst.Key.CloudletKey.OperatorKey.Name) + + //TODO values.application.template + + if IsLocalDIND() { + masteraddr := dind.GetMasterAddr() + log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind") + + portDetail, err := GetPortDetail(appInst) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "GetPortDetail failed", "appInst", appInst, "err", err) + return err + } + if err := AddNginxProxy("localhost", appName, masteraddr, portDetail, dind.GetDockerNetworkName(clusterName)); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", appName, "ports", appInst.MappedPorts) + return err + } + log.DebugLog(log.DebugLevelMexos, "call runKubectlCreateApp for dind") + err = runKubectlCreateApp(clusterInst, appInst, app.DeploymentManifest) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error creating dind app") + return err + } + return nil + } + + switch operatorName { + case cloudcommon.OperatorGCP: + fallthrough + case cloudcommon.OperatorAzure: + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + return runKubectlCreateApp(clusterInst, appInst, app.DeploymentManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { + return fmt.Errorf("not yet supported") + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + return fmt.Errorf("not yet supported") + } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { + return fmt.Errorf("not yet supported") + } + return fmt.Errorf("unknown deployment type %s", appDeploymentType) + default: + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + return CreateKubernetesAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + return CreateHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + } + //TODO -- support these later + //} else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { + // return CreateQCOW2AppManifest(mf) TODO: support this later + //else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { + // return CreateDockerSwarmAppManifest(mf) + //} + return fmt.Errorf("unknown deployment type %s", appDeploymentType) } - fixValuesInst(mf, rootLB) - return MEXAppCreateAppManifest(mf) } -//MEXAppDeleteAppInst deletes app with templated manifest -func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, app *edgeproto.App) error { +func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, app *edgeproto.App, appInst *edgeproto.AppInst) error { log.DebugLog(log.DebugLevelMexos, "mex delete app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) - mf, err := fillAppTemplate(rootLB, appInst, app, clusterInst) - if err != nil { - log.DebugLog(log.DebugLevelMexos, "fillAppTemplate error", "error", err) - return err + appDeploymentType := app.Deployment + operatorName := NormalizeName(appInst.Key.CloudletKey.OperatorKey.Name) + appName := NormalizeName(app.Key.Name) + + if IsLocalDIND() { + log.DebugLog(log.DebugLevelMexos, "run kubectl delete app for dind") + err := runKubectlDeleteApp(clusterInst, appInst, app.DeploymentManifest) + if err != nil { + return err + } + + log.DebugLog(log.DebugLevelMexos, "call DeleteNginxProxy for dind") + + if err = DeleteNginxProxy("localhost", appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", appName) + return err + } + + return nil + + } + switch operatorName { + case cloudcommon.OperatorGCP: + fallthrough + case cloudcommon.OperatorAzure: + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + return runKubectlDeleteApp(clusterInst, appInst, app.DeploymentManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { + return fmt.Errorf("not yet supported") + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + return fmt.Errorf("not yet supported") + } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { + return fmt.Errorf("not yet supported") + } + return fmt.Errorf("unknown image type %s", appDeploymentType) + default: + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + return DeleteKubernetesAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + return DeleteHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + } + //TODO + //} else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { + // return DeleteQCOW2AppManifest(mf) + //} else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { + // return DeleteDockerSwarmAppManifest(mf) + return fmt.Errorf("unknown deployment type %s", appDeploymentType) } - fixValuesInst(mf, rootLB) - return MEXAppDeleteAppManifest(mf) } +/* func fixValuesInst(mf *Manifest, rootLB *MEXRootLB) error { if mf.Values.Kind == "" { mf.Values = rootLB.PlatConf.Values @@ -169,4 +212,4 @@ func fixValuesInst(mf *Manifest, rootLB *MEXRootLB) error { log.DebugLog(log.DebugLevelMexos, "warning, missing mf values") } return nil -} +}*/ diff --git a/mexos/ip.go b/mexos/ip.go index bf9615b25..e7b017a25 100644 --- a/mexos/ip.go +++ b/mexos/ip.go @@ -10,8 +10,8 @@ import ( var defaultPrivateNetRange = "10.101.X.0/24" //GetInternalIP returns IP of the server -func GetInternalIP(mf *Manifest, name string) (string, error) { - sd, err := GetServerDetails(mf, name) +func GetInternalIP(name string) (string, error) { + sd, err := GetServerDetails(name) if err != nil { return "", err } @@ -23,8 +23,8 @@ func GetInternalIP(mf *Manifest, name string) (string, error) { } //GetInternalCIDR returns CIDR of server -func GetInternalCIDR(mf *Manifest, name string) (string, error) { - addr, err := GetInternalIP(mf, name) +func GetInternalCIDR(name string) (string, error) { + addr, err := GetInternalIP(name) if err != nil { return "", err } @@ -40,11 +40,11 @@ func GetAllowedClientCIDR() string { //XXX allow creating more than one LB //GetServerIPAddr gets the server IP -func GetServerIPAddr(mf *Manifest, networkName, serverName string) (string, error) { +func GetServerIPAddr(networkName, serverName string) (string, error) { //TODO: mexosagent cache //log.DebugLog(log.DebugLevelMexos, "get server ip addr", "networkname", networkName, "servername", serverName) //sd, err := GetServerDetails(rootLB) - sd, err := GetServerDetails(mf, serverName) + sd, err := GetServerDetails(serverName) if err != nil { return "", err } @@ -81,18 +81,18 @@ func GetServerIPAddr(mf *Manifest, networkName, serverName string) (string, erro } //FindNodeIP finds IP for the given node -func FindNodeIP(mf *Manifest, name string) (string, error) { +func FindNodeIP(name string) (string, error) { //log.DebugLog(log.DebugLevelMexos, "find node ip", "name", name) if name == "" { return "", fmt.Errorf("empty name") } - srvs, err := ListServers(mf) + srvs, err := ListServers() if err != nil { return "", err } for _, s := range srvs { if s.Status == "ACTIVE" && s.Name == name { - ipaddr, err := GetInternalIP(mf, s.Name) + ipaddr, err := GetInternalIP(s.Name) if err != nil { return "", fmt.Errorf("can't get IP for %s, %v", s.Name, err) } diff --git a/mexos/kubeadmdind.go b/mexos/kubeadmdind.go index 5ad7b126e..2055eb36a 100644 --- a/mexos/kubeadmdind.go +++ b/mexos/kubeadmdind.go @@ -5,33 +5,36 @@ import ( "time" "github.com/mobiledgex/edge-cloud-infra/k8s-prov/dind" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) -func localCreateDIND(mf *Manifest) error { +func localCreateDIND(clusterInst *edgeproto.ClusterInst) error { var err error - log.DebugLog(log.DebugLevelMexos, "creating local dind cluster", "name", mf.Metadata.Name) - if err = dind.CreateDINDCluster(mf.Metadata.ResourceGroup, mf.Metadata.Name); err != nil { + clusterName := clusterInst.Key.ClusterKey.Name + log.DebugLog(log.DebugLevelMexos, "creating local dind cluster", "clusterName", clusterName) + + if err = dind.CreateDINDCluster(clusterName); err != nil { return err } //race condition exists where the config file is not ready until just after the cluster create is done time.Sleep(3 * time.Second) - kconf, err := GetKconf(mf, false) // XXX + kconf, err := GetKconf(clusterInst, false) // XXX if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v, %v", mf, kconf, err) + return fmt.Errorf("cannot get kconf, %v, %v", kconf, err) } log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX //XXX watch out for multiple cluster contexts if err = copyFile(defaultKubeconfig(), kconf); err != nil { return fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) } - log.DebugLog(log.DebugLevelMexos, "created dind", "name", mf.Spec.Key) + log.DebugLog(log.DebugLevelMexos, "created dind", "name", clusterName) - err = CreateDockerRegistrySecret(mf) + err = CreateDockerRegistrySecret(clusterInst, "") if err != nil { - return fmt.Errorf("cannot create mexreg secret for: %s, err: %v", mf.Spec.Key, err) + return fmt.Errorf("cannot create mexreg secret for: %s, err: %v", clusterName, err) } return nil diff --git a/mexos/kubeconfig.go b/mexos/kubeconfig.go index c735eadf5..562964bcf 100644 --- a/mexos/kubeconfig.go +++ b/mexos/kubeconfig.go @@ -2,29 +2,32 @@ package mexos import ( "fmt" - "io/ioutil" "os" - "github.com/ghodss/yaml" "github.com/mobiledgex/edge-cloud-infra/k8s-prov/azure" "github.com/mobiledgex/edge-cloud-infra/k8s-prov/gcloud" + "github.com/mobiledgex/edge-cloud/cloudcommon" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) -func GetLocalKconfName(mf *Manifest) string { - kconf := fmt.Sprintf("%s/%s", MEXDir(), GetKconfName(mf)) +func GetLocalKconfName(clusterInst *edgeproto.ClusterInst) string { + kconf := fmt.Sprintf("%s/%s", MEXDir(), GetKconfName(clusterInst)) return kconf } -func GetKconfName(mf *Manifest) string { +func GetKconfName(clusterInst *edgeproto.ClusterInst) string { return fmt.Sprintf("%s.%s.%s.kubeconfig", - mf.Values.Cluster.Name, - mf.Values.Operator.Name, - mf.Values.Network.DNSZone) + clusterInst.Key.ClusterKey.Name, + clusterInst.Key.CloudletKey.OperatorKey.Name, + GetCloudletDNSZone()) } -func GetKconf(mf *Manifest, createIfMissing bool) (string, error) { - name := GetLocalKconfName(mf) +func GetKconf(clusterInst *edgeproto.ClusterInst, createIfMissing bool) (string, error) { + name := GetLocalKconfName(clusterInst) + operatorName := clusterInst.Key.CloudletKey.OperatorKey.Name + clusterName := clusterInst.Key.ClusterKey.Name + log.DebugLog(log.DebugLevelMexos, "get kubeconfig name", "name", name) if createIfMissing { // XXX log.DebugLog(log.DebugLevelMexos, "warning, creating missing kubeconfig", "name", name) @@ -32,23 +35,24 @@ func GetKconf(mf *Manifest, createIfMissing bool) (string, error) { // if kubeconfig does not exist, optionally create it. It is possible it was // created on a different container or we had a restart of the container log.DebugLog(log.DebugLevelMexos, "creating missing kconf file", "name", name) - switch mf.Metadata.Operator { - case "gcp": - if err = gcloud.GetGKECredentials(mf.Metadata.Name); err != nil { + switch operatorName { + case cloudcommon.OperatorGCP: + if err = gcloud.GetGKECredentials(clusterName); err != nil { return "", fmt.Errorf("unable to get GKE credentials %v", err) } if err = copyFile(defaultKubeconfig(), name); err != nil { return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) } - case "azure": - if err = azure.GetAKSCredentials(mf.Metadata.ResourceGroup, mf.Metadata.Name); err != nil { + case cloudcommon.OperatorAzure: + rg := GetResourceGroupForCluster(clusterInst) + if err = azure.GetAKSCredentials(rg, clusterName); err != nil { return "", fmt.Errorf("unable to get AKS credentials %v", err) } if err = copyFile(defaultKubeconfig(), name); err != nil { return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) } default: - log.DebugLog(log.DebugLevelMexos, "warning, not creating missing kubeconfig for operator", "operator", mf.Metadata.Operator) + log.DebugLog(log.DebugLevelMexos, "warning, not creating missing kubeconfig for operator", "operator", operatorName) } } } @@ -96,24 +100,22 @@ type clusterKubeconfig struct { } //CopyKubeConfig copies over kubeconfig from the cluster -func CopyKubeConfig(mf *Manifest, rootLB *MEXRootLB, name string) error { +func CopyKubeConfig(clusterInst *edgeproto.ClusterInst, rootLBName, name string) error { log.DebugLog(log.DebugLevelMexos, "copying kubeconfig", "name", name) - if rootLB == nil { - return fmt.Errorf("cannot copy kubeconfig, rootLB is null") - } - ipaddr, err := FindNodeIP(mf, name) + + ipaddr, err := FindNodeIP(name) if err != nil { return err } - if mf.Values.Network.External == "" { + if GetCloudletExternalNetwork() == "" { return fmt.Errorf("copy kube config, missing external network in platform config") } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLBName, GetCloudletExternalNetwork(), sshUser) if err != nil { return fmt.Errorf("can't get ssh client for copying kubeconfig, %v", err) } //kconfname := fmt.Sprintf("%s.kubeconfig", name[strings.LastIndex(name, "-")+1:]) - kconfname := GetKconfName(mf) + kconfname := GetKconfName(clusterInst) log.DebugLog(log.DebugLevelMexos, "attempt to get kubeconfig from k8s master", "name", name, "ipaddr", ipaddr, "dest", kconfname) cmd := fmt.Sprintf("scp -o %s -o %s -i id_rsa_mex %s@%s:.kube/config %s", sshOpts[0], sshOpts[1], sshUser, ipaddr, kconfname) out, err := client.Output(cmd) @@ -134,6 +136,7 @@ func CopyKubeConfig(mf *Manifest, rootLB *MEXRootLB, name string) error { return nil } +/* //ProcessKubeconfig validates kubeconfig and saves it and creates a copy for proxy access func ProcessKubeconfig(mf *Manifest, rootLB *MEXRootLB, name string, port int, dat []byte) error { log.DebugLog(log.DebugLevelMexos, "process kubeconfig file", "name", name) @@ -163,3 +166,4 @@ func ProcessKubeconfig(mf *Manifest, rootLB *MEXRootLB, name string, port int, d log.DebugLog(log.DebugLevelMexos, "kubeconfig file saved", "file", kconfname) return nil } +*/ diff --git a/mexos/kubectl.go b/mexos/kubectl.go index 6ce408897..7be837574 100644 --- a/mexos/kubectl.go +++ b/mexos/kubectl.go @@ -9,58 +9,49 @@ import ( "time" sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" + "github.com/mobiledgex/edge-cloud/cloudcommon" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) -func RunKubectl(mf *Manifest, params string) (*string, error) { +/* +func RunKubectl(clusterInst *edgeproto.ClusterInst, params string, rootLBName string) (*string, error) { log.DebugLog(log.DebugLevelMexos, "run kubectl", "params", params) - rootLB, err := getRootLB(mf.Spec.RootLB) - if err != nil { - return nil, err - } - if rootLB == nil { - return nil, fmt.Errorf("failed to create docker registry secret, rootLB is null") - } + //name, err := FindClusterWithKey(mf, mf.Spec.Key) //if err != nil { // return nil, fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) //} - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLBName, GetCloudletExternalNetwork(), sshUser) if err != nil { return nil, fmt.Errorf("can't get ssh client, %v", err) } - cmd := fmt.Sprintf("kubectl --kubeconfig %s.kubeconfig %s", rootLB.Name, params) + cmd := fmt.Sprintf("kubectl --kubeconfig %s.kubeconfig %s", rootLBName, params) out, err := client.Output(cmd) if err != nil { return nil, fmt.Errorf("kubectl failed, %v, %s", err, out) } return &out, nil } - -func CreateDockerRegistrySecret(mf *Manifest) error { +*/ +func CreateDockerRegistrySecret(clusterInst *edgeproto.ClusterInst, rootLBName string) error { log.DebugLog(log.DebugLevelMexos, "creating docker registry secret in kubernetes") - rootLB, err := getRootLB(mf.Spec.RootLB) - if err != nil { - return err - } - if rootLB == nil { - return fmt.Errorf("failed to create docker registry secret, rootLB is null") - } var out string - //log.DebugLog(log.DebugLevelMexos, "CreateDockerRegistrySecret", "mf", mf) + var err error log.DebugLog(log.DebugLevelMexos, "creating docker registry secret in kubernetes cluster") - if IsLocalDIND(mf) || mf.Metadata.Operator == "gcp" || mf.Metadata.Operator == "azure" { - log.DebugLog(log.DebugLevelMexos, "CreateDockerRegistrySecret locally non OpenStack case") + if rootLBName == "" { + log.DebugLog(log.DebugLevelMexos, "CreateDockerRegistrySecret locally, no rootLB") var o []byte - o, err = sh.Command("kubectl", "create", "secret", "docker-registry", "mexregistrysecret", "--docker-server="+mf.Values.Registry.Docker, "--docker-username=mobiledgex", "--docker-password="+mexEnv(mf, "MEX_DOCKER_REG_PASS"), "--docker-email=mobiledgex@mobiledgex.com").CombinedOutput() + o, err = sh.Command("kubectl", "create", "secret", "docker-registry", "mexregistrysecret", "--docker-server="+GetCloudletDockerRegistry(), "--docker-username=mobiledgex", "--docker-password="+GetCloudletDockerPass(), "--docker-email=mobiledgex@mobiledgex.com").CombinedOutput() out = string(o) } else { - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLBName, GetCloudletExternalNetwork(), sshUser) if err != nil { return fmt.Errorf("can't get ssh client, %v", err) } - cmd := fmt.Sprintf("kubectl create secret docker-registry mexregistrysecret --docker-server=%s --docker-username=mobiledgex --docker-password=%s --docker-email=mobiledgex@mobiledgex.com --kubeconfig=%s", mf.Values.Registry.Docker, mexEnv(mf, "MEX_DOCKER_REG_PASS"), GetKconfName(mf)) + cmd := fmt.Sprintf("kubectl create secret docker-registry mexregistrysecret --docker-server=%s --docker-username=mobiledgex --docker-password=%s --docker-email=mobiledgex@mobiledgex.com --kubeconfig=%s", GetCloudletDockerRegistry(), GetCloudletDockerPass(), GetKconfName(clusterInst)) out, err = client.Output(cmd) } @@ -75,27 +66,27 @@ func CreateDockerRegistrySecret(mf *Manifest) error { return nil } -func runKubectlCreateApp(mf *Manifest, kubeManifest string) error { +func runKubectlCreateApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, kubeManifest string) error { log.DebugLog(log.DebugLevelMexos, "run kubectl create app", "kubeManifest", kubeManifest) - //if err := CreateDockerRegistrySecret(mf); err != nil { - // return err - //} - kfile := mf.Metadata.Name + ".yaml" + + appName := NormalizeName(appInst.Key.AppKey.Name) + + kfile := appName + ".yaml" if err := writeKubeManifest(kubeManifest, kfile); err != nil { return err } defer os.Remove(kfile) - kconf, err := GetKconf(mf, false) + kconf, err := GetKconf(clusterInst, false) if err != nil { - return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err) + return fmt.Errorf("error creating app due to kconf missing, %v, %v", clusterInst, err) } - out, err := sh.Command("kubectl", "create", "-f", kfile, "--kubeconfig="+kconf).CombinedOutput() + out, err := sh.Command("kubectl", "create", "-f", kfile, "--kubeconfig="+kconf).Output() if err != nil { - return fmt.Errorf("error creating app, %s, %v, %v, %s", out, err, mf, kubeManifest) + return fmt.Errorf("error creating app, %s, %v, %v", out, appName, err) } - err = createAppDNS(mf, kconf) + err = createAppDNS(kconf, appInst.Uri, appName) if err != nil { - return fmt.Errorf("error creating dns entry for app, %v, %v", err, mf) + return fmt.Errorf("error creating dns entry for app, %v, %v", appName, err) } return nil } @@ -161,24 +152,50 @@ func getSvcNames(name string, kconf string) ([]string, error) { return serviceNames, nil } -func runKubectlDeleteApp(mf *Manifest, kubeManifest string) error { - kconf, err := GetKconf(mf, false) +func runKubectlDeleteApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, kubeManifest string) error { + if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } + + appName := NormalizeName(appInst.Key.AppKey.Name) + + kconf, err := GetKconf(clusterInst, false) if err != nil { - return fmt.Errorf("error deleting app due to kconf missing, %v, %v", mf, err) + return fmt.Errorf("error deleting app due to kconf missing, %v, %v", clusterInst, err) } - kfile := mf.Metadata.Name + ".yaml" + kfile := appName + ".yaml" err = writeKubeManifest(kubeManifest, kfile) if err != nil { return err } defer os.Remove(kfile) + serviceNames, err := getSvcNames(appName, kconf) + if err != nil { + return err + } + if len(serviceNames) < 1 { + return fmt.Errorf("no service names starting with %s", appName) + } out, err := sh.Command("kubectl", "delete", "-f", kfile, "--kubeconfig="+kconf).CombinedOutput() if err != nil { - return fmt.Errorf("error deleting app, %s, %v, %v", out, mf, err) + return fmt.Errorf("error deleting app, %s, %v, %v", out, appName, err) } - err = deleteAppDNS(mf, kconf) + + fqdnBase := uri2fqdn(appInst.Uri) + dr, err := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if err != nil { - return fmt.Errorf("error deleting dns entry for app, %v, %v", err, mf) + return fmt.Errorf("cannot get dns records for %s, %v", GetCloudletDNSZone(), err) + } + for _, sn := range serviceNames { + fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) + for _, d := range dr { + if d.Type == "A" && d.Name == fqdn { + if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), d.ID); err != nil { + return fmt.Errorf("cannot delete DNS record, %v", d) + } + log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) + } + } } return nil } diff --git a/mexos/kubeproxy.go b/mexos/kubeproxy.go index 044341718..982faf53a 100644 --- a/mexos/kubeproxy.go +++ b/mexos/kubeproxy.go @@ -17,15 +17,15 @@ func StartKubectlProxy(mf *Manifest, rootLB *MEXRootLB, name, kubeconfig string) if rootLB == nil { return 0, fmt.Errorf("cannot kubectl proxy, rootLB is null") } - if mf.Values.Network.External == "" { + if GetCloudletExternalNetwork() == "" { return 0, fmt.Errorf("start kubectl proxy, missing external network in platform config") } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return 0, err } //TODO check /home/ubuntu/.docker-pass file - cmd := fmt.Sprintf("echo %s | docker login -u mobiledgex --password-stdin %s", mexEnv(mf, "MEX_DOCKER_REG_PASS"), mf.Spec.DockerRegistry) + cmd := fmt.Sprintf("echo %s | docker login -u mobiledgex --password-stdin %s", GetCloudletDockerPass(), mf.Spec.DockerRegistry) out, err := client.Output(cmd) if err != nil { return 0, fmt.Errorf("can't docker login, %s, %v", out, err) @@ -62,13 +62,13 @@ func StartKubectlProxy(mf *Manifest, rootLB *MEXRootLB, name, kubeconfig string) return 0, fmt.Errorf("cannot convert port %v, %s", aerr, port) } log.DebugLog(log.DebugLevelMexos, "adding external ingress security rule for kubeproxy", "port", port) - err = AddSecurityRuleCIDR(mf, GetAllowedClientCIDR(), "tcp", GetMEXSecurityRule(mf), portnum+1) //XXX + err = AddSecurityRuleCIDR(GetAllowedClientCIDR(), "tcp", GetCloudletSecurityRule(), portnum+1) //XXX if err != nil { log.DebugLog(log.DebugLevelMexos, "warning, error while adding external ingress security rule for kubeproxy", "error", err, "port", port) } portnum++ //TODO delete security rule when kubectl proxy container deleted - if err := AddNginxKubectlProxy(mf, rootLB.Name, name, portnum); err != nil { + if err := AddNginxKubectlProxy(rootLB.Name, name, portnum); err != nil { return 0, fmt.Errorf("cannot add nginx kubectl proxy, %v", err) } log.DebugLog(log.DebugLevelMexos, "nginx kubectl proxy", "port", portnum) diff --git a/mexos/kubernetes.go b/mexos/kubernetes.go index b3798a728..6c6146308 100644 --- a/mexos/kubernetes.go +++ b/mexos/kubernetes.go @@ -1,66 +1,60 @@ package mexos import ( - "encoding/json" "fmt" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ssh "github.com/nanobox-io/golang-ssh" ) -//CreateKubernetesAppManifest instantiates a new kubernetes deployment -func CreateKubernetesAppManifest(mf *Manifest, kubeManifest string) error { +//CreateKubernetesAppInst instantiates a new kubernetes deployment +func CreateKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { log.DebugLog(log.DebugLevelMexos, "create kubernetes app") - rootLB, err := getRootLB(mf.Spec.RootLB) - if err != nil { - return err - } + + clusterName := clusterInst.Key.ClusterKey.Name + appName := NormalizeName(app.Key.Name) + if rootLB == nil { return fmt.Errorf("cannot create kubernetes app manifest, rootLB is null") } - if err = ValidateCommon(mf); err != nil { - return err - } - if mf.Spec.URI == "" { //XXX TODO register to the DNS registry for public IP app,controller needs to tell us which kind of app + + if appInst.Uri == "" { return fmt.Errorf("empty app URI") } - kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) + kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) if err != nil { return err } log.DebugLog(log.DebugLevelMexos, "will launch app into cluster", "kubeconfig", kp.kubeconfig, "ipaddr", kp.ipaddr) var cmd string - if mexEnv(mf, "MEX_DOCKER_REG_PASS") == "" { + if GetCloudletDockerPass() == "" { return fmt.Errorf("empty docker registry password environment variable") } //if err := CreateDockerRegistrySecret(mf); err != nil { // return err //} //TODO do not create yaml file but use remote yaml file over https - cmd = fmt.Sprintf("cat <<'EOF'> %s.yaml \n%s\nEOF", mf.Metadata.Name, kubeManifest) + cmd = fmt.Sprintf("cat <<'EOF'> %s.yaml \n%s\nEOF", appName, kubeManifest) out, err := kp.client.Output(cmd) if err != nil { return fmt.Errorf("error writing KubeManifest, %s, %s, %v", cmd, out, err) } log.DebugLog(log.DebugLevelMexos, "wrote kubernetes manifest file") - cmd = fmt.Sprintf("%s kubectl create -f %s.yaml", kp.kubeconfig, mf.Metadata.Name) + cmd = fmt.Sprintf("%s kubectl create -f %s.yaml", kp.kubeconfig, appName) out, err = kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deploying kubernetes app, %s, %v", out, err) } log.DebugLog(log.DebugLevelMexos, "applied kubernetes manifest") - // we might not be exposing ports - if len(mf.Spec.Ports) < 1 { - return nil - } // Add security rules - if err = AddProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { + if err = AddProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) return err } - log.DebugLog(log.DebugLevelMexos, "ok, added spec ports", "ports", mf.Spec.Ports) + log.DebugLog(log.DebugLevelMexos, "ok, added ports", "ports", appInst.MappedPorts) // Add DNS Zone - if err = KubeAddDNSRecords(rootLB, mf, kp); err != nil { + if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) return err } @@ -74,175 +68,53 @@ type kubeParam struct { } //ValidateKubernetesParameters checks the kubernetes parameters and kubeconfig settings -func ValidateKubernetesParameters(mf *Manifest, rootLB *MEXRootLB, clustName string) (*kubeParam, error) { +func ValidateKubernetesParameters(clusterInst *edgeproto.ClusterInst, rootLB *MEXRootLB, clustName string) (*kubeParam, error) { log.DebugLog(log.DebugLevelMexos, "validate kubernetes parameters rootLB", "cluster", clustName) + clusterName := clusterInst.Key.ClusterKey.Name + if rootLB == nil { return nil, fmt.Errorf("cannot validate kubernetes parameters, rootLB is null") } - if rootLB.PlatConf == nil { - return nil, fmt.Errorf("validate kubernetes parameters, missing platform config") - } - if mf.Values.Network.External == "" { + if GetCloudletExternalNetwork() == "" { return nil, fmt.Errorf("validate kubernetes parameters, missing external network in platform config") } - if IsLocalDIND(mf) { - return &kubeParam{client: &sshLocal{}}, nil - } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return nil, err } - name, err := FindClusterWithKey(mf, clustName) + master, err := FindClusterMaster(clusterName) if err != nil { return nil, fmt.Errorf("can't find cluster with key %s, %v", clustName, err) } - ipaddr, err := FindNodeIP(mf, name) + ipaddr, err := FindNodeIP(master) if err != nil { return nil, err } //kubeconfig := fmt.Sprintf("KUBECONFIG=%s.kubeconfig", name[strings.LastIndex(name, "-")+1:]) - kubeconfig := fmt.Sprintf("KUBECONFIG=%s", GetKconfName(mf)) + kubeconfig := fmt.Sprintf("KUBECONFIG=%s", GetKconfName(clusterInst)) return &kubeParam{kubeconfig, client, ipaddr}, nil } -//KubernetesApplyManifest does `apply` on the manifest yaml -func KubernetesApplyManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "apply kubernetes manifest") - rootLB, err := getRootLB(mf.Spec.RootLB) - if err != nil { - return err - } - if rootLB == nil { - return fmt.Errorf("cannot apply kubernetes manifest, rootLB is null") - } - if mf.Metadata.Name == "" { - return fmt.Errorf("missing name") - } - kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) - if err != nil { - return err - } - kubeManifest := mf.Config.ConfigDetail.Manifest - cmd := fmt.Sprintf("cat <<'EOF'> %s \n%s\nEOF", mf.Metadata.Name, kubeManifest) - out, err := kp.client.Output(cmd) - if err != nil { - return fmt.Errorf("error writing deployment, %s, %s, %v", cmd, out, err) - } - log.DebugLog(log.DebugLevelMexos, "wrote deployment file") - cmd = fmt.Sprintf("%s kubectl apply -f %s", kp.kubeconfig, mf.Metadata.Name) - out, err = kp.client.Output(cmd) - if err != nil { - return fmt.Errorf("error applying kubernetes manifest, %s, %s, %v", cmd, out, err) - } - return nil -} - -//CreateKubernetesNamespaceManifest creates a new namespace in kubernetes -func CreateKubernetesNamespaceManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "create kubernetes namespace") - err := KubernetesApplyManifest(mf) - if err != nil { - return fmt.Errorf("error applying kubernetes namespace manifest, %v", err) - } - return nil -} - -//TODO DeleteKubernetesNamespace - -//TODO allow configmap creation from files - -//SetKubernetesConfigmapValues sets a key-value in kubernetes configmap -// func SetKubernetesConfigmapValues(rootLBName string, clustername string, configname string, keyvalues ...string) error { -// log.DebugLog(log.DebugLevelMexos, "set configmap values", "rootlbname", rootLBName, "clustername", clustername, "configname", configname) -// rootLB, err := getRootLB(rootLBName) -// if err != nil { -// return err -// } -// if rootLB == nil { -// return fmt.Errorf("cannot set kubeconfig map values, rootLB is null") -// } -// kp, err := ValidateKubernetesParameters(mf, rootLB, clustername) -// if err != nil { -// return err -// } -// //TODO support namespace -// cmd := fmt.Sprintf("%s kubectl create configmap %s ", kp.kubeconfig, configname) -// for _, kv := range keyvalues { -// items := strings.Split(kv, "=") -// if len(items) != 2 { -// return fmt.Errorf("malformed key=value pair, %s", kv) -// } -// cmd = cmd + " --from-literal=" + kv -// } -// out, err := kp.client.Output(cmd) -// if err != nil { -// return fmt.Errorf("error setting key/values to kubernetes configmap, %s, %s, %v", cmd, out, err) -// } -// return nil -// } - -//TODO -//func GetKubernetesConfigmapValues(rootLB, clustername, configname string) (map[string]string, error) { -//} +func DeleteKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { + log.DebugLog(log.DebugLevelMexos, "delete kubernetes app") -//GetKubernetesConfigmapYAML returns yaml reprentation of the key-values -// func GetKubernetesConfigmapYAML(rootLBName string, clustername, configname string) (string, error) { -// log.DebugLog(log.DebugLevelMexos, "get kubernetes configmap", "rootlbname", rootLBName, "clustername", clustername, "configname", configname) -// rootLB, err := getRootLB(rootLBName) -// if err != nil { -// return "", err -// } -// if rootLB == nil { -// return "", fmt.Errorf("cannot get kubeconfigmap yaml, rootLB is null") -// } -// kp, err := ValidateKubernetesParameters(mf, rootLB, clustername) -// if err != nil { -// return "", err -// } -// //TODO support namespace -// cmd := fmt.Sprintf("%s kubectl get configmap %s -o yaml", kp.kubeconfig, configname) -// out, err := kp.client.Output(cmd) -// if err != nil { -// return "", fmt.Errorf("error getting configmap yaml, %s, %s, %v", cmd, out, err) -// } -// return out, nil -// } + clusterName := clusterInst.Key.ClusterKey.Name + appName := NormalizeName(app.Key.Name) -func DeleteKubernetesAppManifest(mf *Manifest, kubeManifest string) error { - log.DebugLog(log.DebugLevelMexos, "delete kubernetes app") - rootLB, err := getRootLB(mf.Spec.RootLB) + kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) if err != nil { return err } - if rootLB == nil { - return fmt.Errorf("cannot remove kubernetes app manifest, rootLB is null") - } - if mf.Spec.URI == "" { //XXX TODO register to the DNS registry for public IP app,controller needs to tell us which kind of app - return fmt.Errorf("empty app URI") - } - //TODO: support other URI: file://, nfs://, ftp://, git://, or embedded as base64 string - //if !strings.Contains(mf.Spec.Flavor, "kubernetes") { - // return fmt.Errorf("unsupported kubernetes flavor %s", mf.Spec.Flavor) - //} - if err = ValidateCommon(mf); err != nil { - return err + // Clean up security rules and nginx proxy + if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot clean up security rules", "name", appName, "rootlb", rootLB.Name, "error", err) } - kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) - if err != nil { + // Clean up DNS entries + if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot clean up DNS entries", "name", appName, "rootlb", rootLB.Name, "error", err) return err } - if len(mf.Spec.Ports) > 0 { - // Clean up security rules and nginx proxy - if err = DeleteProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot clean up security rules", "name", mf.Metadata.Name, "rootlb", rootLB.Name, "error", err) - } - // Clean up DNS entries - if err = KubeDeleteDNSRecords(rootLB, mf, kp); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot clean up DNS entries", "name", mf.Metadata.Name, "rootlb", rootLB.Name, "error", err) - return err - } - } - log.DebugLog(log.DebugLevelMexos, "deleted deployment", "name", mf.Metadata.Name) + log.DebugLog(log.DebugLevelMexos, "deleted deployment", "name", appName) return nil } @@ -282,17 +154,20 @@ type KAPINodes struct { //TODO metadata } +/* func GetKubernetesNodes(mf *Manifest, rootLB *MEXRootLB) ([]KubernetesNode, error) { log.DebugLog(log.DebugLevelMexos, "getting kubernetes nodes") - name, err := FindClusterWithKey(mf, mf.Spec.Key) + clusterName := clusterInst.Key.ClusterKey.Name + + master, err := FindClusterMaster(clusterName) if err != nil { return nil, fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) } - ipaddr, err := FindNodeIP(mf, name) + ipaddr, err := FindNodeIP(master) if err != nil { return nil, err } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return nil, fmt.Errorf("can't get ssh client for getting kubernetes nodes, %v", err) } @@ -334,3 +209,4 @@ func GetKubernetesNodes(mf *Manifest, rootLB *MEXRootLB) ([]KubernetesNode, erro } return kl, nil } +*/ diff --git a/mexos/kvm.go b/mexos/kvm.go index d3df9a350..17ffe24e0 100644 --- a/mexos/kvm.go +++ b/mexos/kvm.go @@ -8,9 +8,16 @@ import ( "time" sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) +func GetK8sNodeNameSuffix(clusterInst *edgeproto.ClusterInst) string { + cloudletName := clusterInst.Key.CloudletKey.Name + clusterName := clusterInst.Key.ClusterKey.Name + return NormalizeName(cloudletName + "-" + clusterName) +} + //CreateQCOW2AppManifest creates qcow2 app func CreateQCOW2AppManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "create qcow2 vm based app") @@ -107,7 +114,7 @@ func CreateQCOW2AppManifest(mf *Manifest) error { //TODO properties //TODO userdata log.DebugLog(log.DebugLevelMexos, "calling create openstack kvm server", "opts", opts) - err = CreateServer(mf, opts) + err = CreateServer(opts) if err != nil { return fmt.Errorf("can't create openstack kvm server instance %v, %v", opts, err) } @@ -119,7 +126,7 @@ func DeleteQCOW2AppManifest(mf *Manifest) error { if mf.Metadata.Name == "" { return fmt.Errorf("missing name, no openstack kvm to delete") } - if err := DeleteServer(mf, mf.Metadata.Name); err != nil { + if err := DeleteServer(mf.Metadata.Name); err != nil { return fmt.Errorf("cannot delete openstack kvm %s, %v", mf.Metadata.Name, err) } return nil @@ -131,11 +138,11 @@ func DeleteQCOW2AppManifest(mf *Manifest) error { // Roles can be any string but special ones are k8s-master and k8s-node. // To avoid running bootstrap setup for creating kubernets cluster, set skipk8s to true. // For more detailed information please read `mobiledgex-init.sh` -func CreateFlavorMEXVM(mf *Manifest, name, image, flavor, netID, userdata, role, edgeproxy, skipk8s, k8smaster, privatenet, privaterouter, tags, tenant string) error { +func CreateFlavorMEXVM(name, image, flavor, netID, userdata, role, edgeproxy, skipk8s, k8smaster, privatenet, privaterouter, tags, tenant string) error { if name == "" { return fmt.Errorf("name required") } - sd, err := GetServerDetails(mf, name) + sd, err := GetServerDetails(name) if err == nil { log.DebugLog(log.DebugLevelMexos, "warning, server already exists", "name", sd.Name, "server detail", sd) return nil @@ -145,13 +152,13 @@ func CreateFlavorMEXVM(mf *Manifest, name, image, flavor, netID, userdata, role, } if image == "" { - image = GetMEXImageName(mf) + image = GetCloudletOSImage() } if flavor == "" { return fmt.Errorf("Missing platform flavor") } if userdata == "" { - userdata = GetMEXUserData(mf) + userdata = GetCloudletUserData() } opts := &OSServerOpt{ Name: name, @@ -178,16 +185,23 @@ func CreateFlavorMEXVM(mf *Manifest, name, image, flavor, netID, userdata, role, props = append(props, "privaterouter="+privaterouter) props = append(props, "tags="+tags) props = append(props, "tenant="+tenant) - if mf.Values.Network.HolePunch != "" { - props = append(props, "holepunch="+mf.Values.Network.HolePunch) + + /* TODO: holepunch has not been used anywhere but need to investigate if we will want this + if GetCloudletExternalNetwork().HolePunch != "" { + props = append(props, "holepunch="+GetCloudletExternalNetwork().HolePunch) } + */ + + /* TODO: update has code for it in the init scripts, but has not been used because the cloudlet-specific files + are not present on the registry and nobody knew this existed. This is for Venky to study. if mf.Values.Registry.Update != "" { props = append(props, "update="+mf.Values.Registry.Update) } + */ opts.Properties = props //log.DebugLog(log.DebugLevelMexos, "create flavor MEX KVM", "flavor", flavor, "server opts", opts) log.DebugLog(log.DebugLevelMexos, "create flavor MEX KVM", "flavor", flavor) - err = CreateServer(mf, opts) + err = CreateServer(opts) if err != nil { log.DebugLog(log.DebugLevelMexos, "error creating flavor MEX KVM", "server opts", opts) return fmt.Errorf("can't create server, opts %v, %v", opts, err) @@ -197,13 +211,18 @@ func CreateFlavorMEXVM(mf *Manifest, name, image, flavor, netID, userdata, role, //CreateMEXKVM is easier way to create a MEX app capable KVM // role can be k8s-master, k8s-node, or something else -func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int, platformFlavor string) error { +func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, clusterInst *edgeproto.ClusterInst, platformFlavor string) error { log.DebugLog(log.DebugLevelMexos, "createMEXKVM", "name", name, "role", role, "netSpec", netSpec, "tags", tags, "tenant", tenant, "id", id) - mexRouter := GetMEXExternalRouter(mf) - netID := GetMEXExternalNetwork(mf) //do we really want to default to ext? + mexRouter := GetCloudletExternalRouter() + netID := GetCloudletExternalNetwork() //do we really want to default to ext? skipk8s := "yes" + nameSuffix := "" + if clusterInst != nil { + nameSuffix = GetK8sNodeNameSuffix(clusterInst) + } + var masterIP, privRouterIP, privNet, edgeProxy string var err error //if role == "mex-agent-node" docker will be installed automatically @@ -238,14 +257,14 @@ func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int if ni.CIDR == "" { return fmt.Errorf("missing CIDR spec in %v", ni) } - if ni.Name != GetMEXNetwork(mf) { //XXX for now - return fmt.Errorf("netspec net name %s not equal to default MEX net %s", ni.Name, GetMEXNetwork(mf)) + if ni.Name != GetCloudletNetwork() { //XXX for now + return fmt.Errorf("netspec net name %s not equal to default MEX net %s", ni.Name, GetCloudletNetwork()) } //XXX openstack bug - subnet does not take tags but description field can be used to tag stuff // Use tag as part of name - sn := ni.Name + "-subnet-" + mf.Values.Cluster.Name + sn := ni.Name + "-subnet-" + nameSuffix log.DebugLog(log.DebugLevelMexos, "using subnet name", "subnet", sn) - snd, snderr := GetSubnetDetail(mf, sn) + snd, snderr := GetSubnetDetail(sn) if snderr != nil { if role == k8snodeRole { return fmt.Errorf("subnet %s does not exist, %v", sn, snderr) @@ -283,7 +302,7 @@ func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int if octno != 2 { return fmt.Errorf("net CIDR, we want octno to be 2 for now") } - sns, snserr := ListSubnets(mf, ni.Name) + sns, snserr := ListSubnets(ni.Name) if snserr != nil { return fmt.Errorf("can't get list of subnets for %s, %v", ni.Name, snserr) } @@ -350,18 +369,18 @@ func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int ni.CIDR = fmt.Sprintf("%d.%d.%d.%d/%s", v4a[0], v4a[1], v4a[2], id, sits[1]) // we can have up to 150 nodes of workers per subnet. // change these values as needed. - sl, err := ListSubnets(mf, ni.Name) + sl, err := ListSubnets(ni.Name) if err != nil { return fmt.Errorf("can't get a list of subnets, %v", err) } for _, sn := range sl { - sd, err := GetSubnetDetail(mf, sn.ID) + sd, err := GetSubnetDetail(sn.ID) if err != nil { return fmt.Errorf("can't get subnet detail, %s, %v", sn.Name, err) } if sd.CIDR == ni.CIDR { log.DebugLog(log.DebugLevelMexos, "subnet exists with the same CIDR, find another range", "CIDR", sd.CIDR) - cidr, err := getNewSubnetRange(mf, id, v4a, sits, sl) + cidr, err := getNewSubnetRange(id, v4a, sits, sl) if err != nil { return fmt.Errorf("failed to get a new subnet range, %v", err) } @@ -383,12 +402,12 @@ func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int if role == k8smasterRole { if snd == nil { log.DebugLog(log.DebugLevelMexos, "k8s master, no existing subnet, creating subnet", "name", sn) - err = CreateSubnet(mf, ni.CIDR, GetMEXNetwork(mf), edgeProxy, sn, false) + err = CreateSubnet(ni.CIDR, GetCloudletNetwork(), edgeProxy, sn, false) if err != nil { return err } //TODO: consider adding tags to subnet - err = AddRouterSubnet(mf, mexRouter, sn) + err = AddRouterSubnet(mexRouter, sn) if err != nil { return fmt.Errorf("cannot add router %s to subnet %s, %v", mexRouter, sn, err) } @@ -403,7 +422,7 @@ func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int //master node num is 1 //so, master node will always have .2 //XXX master always at X.X.X.2 - netID = GetMEXNetwork(mf) + ",v4-fixed-ip=" + ipaddr.String() + netID = GetCloudletNetwork() + ",v4-fixed-ip=" + ipaddr.String() masteripaddr := net.IPv4(v4[0], v4[1], v4[2], byte(2)) masterIP = masteripaddr.String() log.DebugLog(log.DebugLevelMexos, "k8s master ip addr", "netID", netID, "ipaddr", ipaddr, "masterip", masterIP) @@ -411,18 +430,18 @@ func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int // NOT k8s case // for now just agent stuff. log.DebugLog(log.DebugLevelMexos, "create mex kvm, plain kvm, not kubernetes case") - edgeProxy, err = GetExternalGateway(mf, netID) + edgeProxy, err = GetExternalGateway(netID) if err != nil { return fmt.Errorf("can't get external gateway for %s, %v", netID, err) } log.DebugLog(log.DebugLevelMexos, "external gateway", "external gateway, edgeproxy", edgeProxy) - rd, rderr := GetRouterDetail(mf, mexRouter) + rd, rderr := GetRouterDetail(mexRouter) if rderr != nil { return fmt.Errorf("can't get router detail for %s, %v", mexRouter, rderr) } log.DebugLog(log.DebugLevelMexos, "router detail", "detail", rd) - reg, regerr := GetRouterDetailExternalGateway(mf, rd) + reg, regerr := GetRouterDetailExternalGateway(rd) if regerr != nil { //return fmt.Errorf("can't get router detail external gateway, %v", regerr) log.DebugLog(log.DebugLevelMexos, "can't get router detail, not fatal") @@ -455,11 +474,11 @@ func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int if err != nil { return fmt.Errorf("cannot get flavor from tags '%s'", tags) } - err = CreateFlavorMEXVM(mf, name, - GetMEXImageName(mf), + err = CreateFlavorMEXVM(name, + GetCloudletOSImage(), platformFlavor, netID, // either external-net or internal-net,v4-fixed-ip=X.X.X.X - GetMEXUserData(mf), + GetCloudletUserData(), role, // k8s-master,k8s-node,something else edgeProxy, skipk8s, // if yes, skip @@ -477,13 +496,13 @@ func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int return nil } -func getNewSubnetRange(mf *Manifest, id int, v4a []byte, sits []string, sl []OSSubnet) (*string, error) { +func getNewSubnetRange(id int, v4a []byte, sits []string, sl []OSSubnet) (*string, error) { var cidr string for newID := id + 1; newID < MEXSubnetLimit; newID++ { cidr = fmt.Sprintf("%d.%d.%d.%d/%s", v4a[0], v4a[1], v4a[2], newID, sits[1]) found := false for _, snn := range sl { - sdd, err := GetSubnetDetail(mf, snn.ID) + sdd, err := GetSubnetDetail(snn.ID) if err != nil { return nil, fmt.Errorf("cannot get subnet detail %s, %v", snn.Name, err) } @@ -501,20 +520,20 @@ func getNewSubnetRange(mf *Manifest, id int, v4a []byte, sits []string, sl []OSS //DestroyMEXKVM deletes the MEX KVM instance. If server instance is k8s-master, // first remove router from subnet which was created for it. Then remove subnet before // deleting server KVM instance. -func DestroyMEXKVM(mf *Manifest, name, role string) error { +func DestroyMEXKVM(name, role, clusterName string) error { //TODO send shutdown command to running VM. Left undone so we insteadi optionally // send ssh shutdown manually before deleting the KVM instance via API or mexctl. log.DebugLog(log.DebugLevelMexos, "delete mex kvm server", "name", name, "role", role) - err := DeleteServer(mf, name) + err := DeleteServer(name) if err != nil { return fmt.Errorf("can't delete %s, %v", name, err) } if role == k8smasterRole { - sn := "subnet-" + mf.Values.Cluster.Name - rn := GetMEXExternalRouter(mf) + sn := "subnet-" + clusterName + rn := GetCloudletExternalRouter() log.DebugLog(log.DebugLevelMexos, "removing router from subnet", "router", rn, "subnet", sn) - err := RemoveRouterSubnet(mf, rn, sn) + err := RemoveRouterSubnet(rn, sn) if err != nil { return fmt.Errorf("can't remove subnet %s from router %s, %v", sn, rn, err) } @@ -523,7 +542,7 @@ func DestroyMEXKVM(mf *Manifest, name, role string) error { // IP addresses are allocated out of this subnet // All nodes should be deleted first. log.DebugLog(log.DebugLevelMexos, "deleting subnet", "name", sn) - err = DeleteSubnet(mf, sn) + err = DeleteSubnet(sn) if err != nil { return fmt.Errorf("can't delete subnet %s, %v", sn, err) } diff --git a/mexos/lb.go b/mexos/lb.go index 19fb60188..25291161e 100644 --- a/mexos/lb.go +++ b/mexos/lb.go @@ -9,7 +9,7 @@ import ( ) //LBAddRoute adds a route to LB -func LBAddRoute(mf *Manifest, rootLBName, extNet, name string) error { +func LBAddRoute(rootLBName, extNet, name string) error { if rootLBName == "" { return fmt.Errorf("empty rootLB") } @@ -19,7 +19,7 @@ func LBAddRoute(mf *Manifest, rootLBName, extNet, name string) error { if extNet == "" { return fmt.Errorf("empty external network") } - ap, err := LBGetRoute(mf, rootLBName, name) + ap, err := LBGetRoute(rootLBName, name) if err != nil { return err } @@ -27,7 +27,7 @@ func LBAddRoute(mf *Manifest, rootLBName, extNet, name string) error { return fmt.Errorf("expected 2 addresses, got %d", len(ap)) } cmd := fmt.Sprintf("sudo ip route add %s via %s dev ens3", ap[0], ap[1]) - client, err := GetSSHClient(mf, rootLBName, extNet, sshUser) + client, err := GetSSHClient(rootLBName, extNet, sshUser) if err != nil { return err } @@ -43,8 +43,8 @@ func LBAddRoute(mf *Manifest, rootLBName, extNet, name string) error { } //LBRemoveRoute removes route for LB -func LBRemoveRoute(mf *Manifest, rootLB, extNet, name string) error { - ap, err := LBGetRoute(mf, rootLB, name) +func LBRemoveRoute(rootLB, extNet, name string) error { + ap, err := LBGetRoute(rootLB, name) if err != nil { return err } @@ -52,7 +52,7 @@ func LBRemoveRoute(mf *Manifest, rootLB, extNet, name string) error { return fmt.Errorf("expected 2 addresses, got %d", len(ap)) } cmd := fmt.Sprintf("sudo ip route delete %s via %s dev ens3", ap[0], ap[1]) - client, err := GetSSHClient(mf, rootLB, extNet, sshUser) + client, err := GetSSHClient(rootLB, extNet, sshUser) if err != nil { return err } @@ -66,8 +66,8 @@ func LBRemoveRoute(mf *Manifest, rootLB, extNet, name string) error { } //LBGetRoute returns route of LB -func LBGetRoute(mf *Manifest, rootLB, name string) ([]string, error) { - cidr, err := GetInternalCIDR(mf, name) +func LBGetRoute(rootLB, name string) ([]string, error) { + cidr, err := GetInternalCIDR(name) if err != nil { return nil, err } @@ -77,12 +77,12 @@ func LBGetRoute(mf *Manifest, rootLB, name string) ([]string, error) { } v4 := ipn.IP.To4() dn := fmt.Sprintf("%d.%d.%d.0/24", v4[0], v4[1], v4[2]) - rn := GetMEXExternalRouter(mf) - rd, err := GetRouterDetail(mf, rn) + rn := GetCloudletExternalRouter() + rd, err := GetRouterDetail(rn) if err != nil { return nil, fmt.Errorf("can't get router detail for %s, %v", rn, err) } - reg, err := GetRouterDetailExternalGateway(mf, rd) + reg, err := GetRouterDetailExternalGateway(rd) if err != nil { return nil, fmt.Errorf("can't get router detail external gateway, %v", err) } diff --git a/mexos/letsencrypt.go b/mexos/letsencrypt.go index a640b6920..88d279d17 100644 --- a/mexos/letsencrypt.go +++ b/mexos/letsencrypt.go @@ -31,40 +31,37 @@ func checkPEMFile(fn string) error { } //AcquireCertificates obtains certficates from Letsencrypt over ACME. It should be used carefully. The API calls have quota. -func AcquireCertificates(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { +func AcquireCertificates(rootLB *MEXRootLB, fqdn string) error { log.DebugLog(log.DebugLevelMexos, "acquiring certificates for FQDN", "FQDN", fqdn) if rootLB == nil { return fmt.Errorf("cannot acquire certs, rootLB is null") } - if mf.Values.Network.External == "" { - return fmt.Errorf("acquire certificate, missing external network in manifest") - } - if cerr := CheckCredentialsCF(mf); cerr != nil { - return cerr + if GetCloudletExternalNetwork() == "" { + return fmt.Errorf("acquire certificate, missing external network ") } kf := PrivateSSHKey() - srcfile := fmt.Sprintf("mobiledgex@%s:files-repo/certs/%s/fullchain.cer", mf.Values.Registry.Name, fqdn) + srcfile := fmt.Sprintf("mobiledgex@%s:files-repo/certs/%s/fullchain.cer", GetCloudletRegistryFileServer(), fqdn) dkey := fmt.Sprintf("%s/%s.key", fqdn, fqdn) certfile := "cert.pem" keyfile := "key.pem" - log.DebugLog(log.DebugLevelMexos, "trying to get cached cert files") - out, err := sh.Command("scp", "-o", sshOpts[0], "-o", sshOpts[1], "-i", kf, srcfile, certfile).Output() + log.DebugLog(log.DebugLevelMexos, "trying to get cached cert files", "srcfile", srcfile) + out, err := sh.Command("scp", "-o", sshOpts[0], "-o", sshOpts[1], "-i", kf, srcfile, certfile).CombinedOutput() if err != nil { log.DebugLog(log.DebugLevelMexos, "warning, failed to get cached cert file", "src", srcfile, "cert", certfile, "error", err, "out", out) } else if checkPEMFile(certfile) == nil { - srcfile = fmt.Sprintf("mobiledgex@%s:files-repo/certs/%s", mf.Values.Registry.Name, dkey) + srcfile = fmt.Sprintf("mobiledgex@%s:files-repo/certs/%s", GetCloudletRegistryFileServer(), dkey) out, err = sh.Command("scp", "-o", sshOpts[0], "-o", sshOpts[1], "-i", kf, srcfile, keyfile).Output() if err != nil { log.DebugLog(log.DebugLevelMexos, "warning, failed to get cached key file", "src", srcfile, "cert", certfile, "error", err, "out", out) } else if checkPEMFile(keyfile) == nil { //because Letsencrypt complains if we get certs repeated for the same fqdn log.DebugLog(log.DebugLevelMexos, "got cached certs from registry", "FQDN", fqdn) - addr, ierr := GetServerIPAddr(mf, mf.Values.Network.External, fqdn) //XXX should just use fqdn but paranoid + addr, ierr := GetServerIPAddr(GetCloudletExternalNetwork(), fqdn) //XXX should just use fqdn but paranoid if ierr != nil { log.DebugLog(log.DebugLevelMexos, "failed to get server ip addr", "FQDN", fqdn, "error", ierr) return ierr } - client, err := GetSSHClient(mf, fqdn, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(fqdn, GetCloudletExternalNetwork(), sshUser) if err != nil { return fmt.Errorf("can't get ssh client for cert, %v", err) } @@ -83,7 +80,7 @@ func AcquireCertificates(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { } } log.DebugLog(log.DebugLevelMexos, "did not get cached cert and key files, will try to acquire new cert") - client, err := GetSSHClient(mf, fqdn, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(fqdn, GetCloudletExternalNetwork(), sshUser) if err != nil { return fmt.Errorf("can't get ssh client for acme.sh, %v", err) } @@ -93,7 +90,9 @@ func AcquireCertificates(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { if err == nil { return nil } - cmd = fmt.Sprintf("docker run --rm -e CF_Key=%s -e CF_Email=%s -v `pwd`:/acme.sh --net=host neilpang/acme.sh --issue -d %s --dns dns_cf", mexEnv(mf, "MEX_CF_KEY"), mexEnv(mf, "MEX_CF_USER"), fqdn) + cmd = fmt.Sprintf("docker run --rm -e CF_Key=%s -e CF_Email=%s -v `pwd`:/acme.sh --net=host neilpang/acme.sh --issue -d %s --dns dns_cf", GetCloudletCFKey(), GetCloudletCFUser(), fqdn) + log.DebugLog(log.DebugLevelMexos, "running acme.sh to get cert", "cmd", cmd) + res, err := client.Output(cmd) if err != nil { return fmt.Errorf("error running acme.sh docker, %s, %v", res, err) @@ -125,7 +124,7 @@ func AcquireCertificates(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { return fmt.Errorf("cannot copy %s to /root, %v, %s", d.dest, err, out) } } - cmd = fmt.Sprintf("scp -o %s -o %s -i id_rsa_mex -r %s mobiledgex@%s:files-repo/certs", sshOpts[0], sshOpts[1], fqdn, mf.Values.Registry.Name) // XXX + cmd = fmt.Sprintf("scp -o %s -o %s -i id_rsa_mex -r %s mobiledgex@%s:files-repo/certs", sshOpts[0], sshOpts[1], fqdn, GetCloudletRegistryFileServer()) // XXX res, err = client.Output(cmd) if err != nil { return fmt.Errorf("failed to upload certs for %s, %v, %v", fqdn, err, res) diff --git a/mexos/manifest.go b/mexos/manifest.go index 0ea53709f..4f59b5cee 100644 --- a/mexos/manifest.go +++ b/mexos/manifest.go @@ -1,87 +1,6 @@ package mexos -import ( - "bytes" - "fmt" - "strings" - "text/template" - - "github.com/ghodss/yaml" - "github.com/mobiledgex/edge-cloud-infra/k8s-prov/azure" - "github.com/mobiledgex/edge-cloud-infra/k8s-prov/dind" - "github.com/mobiledgex/edge-cloud-infra/k8s-prov/gcloud" - "github.com/mobiledgex/edge-cloud/cloudcommon" - "github.com/mobiledgex/edge-cloud/log" -) - -// IsLocalDIND returns true if the cloudlet is a local DIND deploy -func IsLocalDIND(mf *Manifest) bool { - return mf.Values.Operator.Kind == "localdind" -} - -//MEXClusterCreateManifest creates a cluster -func MEXClusterCreateManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "creating cluster via manifest") - if IsLocalDIND(mf) { - return localCreateDIND(mf) - } - switch mf.Metadata.Operator { - case "gcp": - return gcloudCreateGKE(mf) - case "azure": - return azureCreateAKS(mf) - default: - //guid, err := mexCreateClusterKubernetes(mf) - err := mexCreateClusterKubernetes(mf) - if err != nil { - return fmt.Errorf("can't create cluster, %v", err) - } - //log.DebugLog(log.DebugLevelMexos, "new guid", "guid", *guid) - log.DebugLog(log.DebugLevelMexos, "created kubernetes cluster") - return nil - } -} - -//MEXAddFlavor adds flavor using manifest -func MEXAddFlavor(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "add mex flavor") - //TODO use full manifest and validate against platform data - return AddFlavorManifest(mf) -} - -// TODO DeleteFlavor -- but almost never done - -// TODO lookup guid using name - -//MEXClusterRemoveManifest removes a cluster -func MEXClusterRemoveManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "removing cluster") - if IsLocalDIND(mf) { - return dind.DeleteDINDCluster(mf.Metadata.Name) - } - switch mf.Metadata.Operator { - case "gcp": - return gcloud.DeleteGKECluster(mf.Metadata.Name) - case "azure": - return azure.DeleteAKSCluster(mf.Metadata.ResourceGroup) - default: - if err := mexDeleteClusterKubernetes(mf); err != nil { - return fmt.Errorf("can't remove cluster, %v", err) - } - return nil - } -} - -//MEXPlatformInitCloudletKey calls MEXPlatformInit with templated manifest -func MEXPlatformInitCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) error { - ckmf, err := fillPlatformTemplateCloudletKey(rootLB, cloudletKeyStr) - if err != nil { - return err - } - ckmf.Values = rootLB.PlatConf.Values - return MEXPlatformInitManifest(ckmf) -} - +/* //MEXPlatformCleanCloudletKey calls MEXPlatformClean with templated manifest func MEXPlatformCleanCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) error { mf, err := fillPlatformTemplateCloudletKey(rootLB, cloudletKeyStr) @@ -94,21 +13,14 @@ func MEXPlatformCleanCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) error //MEXPlatformInitManifest initializes platform func MEXPlatformInitManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "init platform") - err := MEXCheckEnvVars(mf) - if err != nil { - return err - } switch mf.Metadata.Operator { - case "gcp": + case cloudcommon.OperatorGCP: return nil //nothing to do - case "azure": + case cloudcommon.OperatorAzure: return nil //nothing to do default: - if err = MEXCheckEnvVars(mf); err != nil { - return err - } //TODO validate all mf content against platform data - if err = RunMEXAgentManifest(mf); err != nil { + if err := RunMEXAgentManifest(mf); err != nil { return err } } @@ -118,262 +30,33 @@ func MEXPlatformInitManifest(mf *Manifest) error { //MEXPlatformCleanManifest cleans up the platform func MEXPlatformCleanManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "clean platform") - err := MEXCheckEnvVars(mf) - if err != nil { - return err - } switch mf.Metadata.Operator { case "gcp": return nil //nothing to do case "azure": return nil default: - if err = MEXCheckEnvVars(mf); err != nil { - return err - } - if err = RemoveMEXAgentManifest(mf); err != nil { + if err := RemoveMEXAgentManifest(mf); err != nil { return err } } return nil } -// Valid manifests are either v1, or apps/v1 -func kubeManifestValidApiVersion(kubeManifest string) bool { - if !strings.HasPrefix(kubeManifest, "apiVersion: v1") && - !strings.HasPrefix(kubeManifest, "apiVersion: apps/v1") { - return false - } - return true -} - //MEXAppCreateAppManifest creates app instances on the cluster platform -func MEXAppCreateAppManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "create app from manifest") - appDeploymentType := mf.Config.ConfigDetail.Deployment - log.DebugLog(log.DebugLevelMexos, "app deployment", "imageType", mf.Spec.ImageType, "deploymentType", appDeploymentType, "config", mf.Config) - var kubeManifest string - var err error - if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - kubeManifest, err = GetKubeManifest(mf) - if err != nil { - return err - } - if !kubeManifestValidApiVersion(kubeManifest) { - log.DebugLog(log.DebugLevelMexos, "bad apiVersion at beginning kubemanifest") - return fmt.Errorf("bad apiversion at beginning of kube manifest") - } - } - //TODO values.application.template - - if IsLocalDIND(mf) { - masteraddr := dind.GetMasterAddr() - - if len(mf.Spec.Ports) > 0 { - log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind", "ports", mf.Spec.Ports) - if err = AddNginxProxy(mf, "localhost", mf.Metadata.Name, masteraddr, mf.Spec.Ports, dind.GetDockerNetworkName(mf.Values.Cluster.Name)); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports) - return err - } - } - log.DebugLog(log.DebugLevelMexos, "call runKubectlCreateApp for dind") - var err error - if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - err = runKubectlCreateApp(mf, kubeManifest) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - err = CreateHelmAppManifest(mf) - } else { - err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) - } - if err != nil { - log.DebugLog(log.DebugLevelMexos, "error creating dind app", "mf", mf) - return err - } - return nil - } - - switch mf.Metadata.Operator { - case "gcp": - fallthrough - case "azure": - if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return runKubectlCreateApp(mf, kubeManifest) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { - return fmt.Errorf("not yet supported") - } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - return fmt.Errorf("not yet supported") - } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { - return fmt.Errorf("not yet supported") - } - return fmt.Errorf("unknown deployment type %s", appDeploymentType) - default: - if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return CreateKubernetesAppManifest(mf, kubeManifest) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { - return CreateQCOW2AppManifest(mf) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - return CreateHelmAppManifest(mf) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { - return CreateDockerSwarmAppManifest(mf) - } - return fmt.Errorf("unknown deployment type %s", appDeploymentType) - } -} - -//MEXAppDeleteManifest kills app -func MEXAppDeleteAppManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "delete app with manifest") - appDeploymentType := mf.Config.ConfigDetail.Deployment - log.DebugLog(log.DebugLevelMexos, "app delete", "imageType", mf.Spec.ImageType, "deploymentType", appDeploymentType, "config", mf.Config) - kubeManifest, err := GetKubeManifest(mf) - if err != nil { - return err - } - if IsLocalDIND(mf) { - log.DebugLog(log.DebugLevelMexos, "run kubectl delete app for dind") - var err error - if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - err = runKubectlDeleteApp(mf, kubeManifest) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - err = DeleteHelmAppManifest(mf) - } else { - err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) - } - if err != nil { - return err - } - - if len(mf.Spec.Ports) > 0 { - log.DebugLog(log.DebugLevelMexos, "call DeleteNginxProxy for dind") - if err = DeleteNginxProxy(mf, "localhost", mf.Metadata.Name); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", mf.Metadata.Name) - return err - } - } - return nil - - } - switch mf.Metadata.Operator { - case "gcp": - fallthrough - case "azure": - if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return runKubectlDeleteApp(mf, kubeManifest) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { - return fmt.Errorf("not yet supported") - } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - return fmt.Errorf("not yet supported") - } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { - return fmt.Errorf("not yet supported") - } - return fmt.Errorf("unknown image type %s", appDeploymentType) - default: - if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return DeleteKubernetesAppManifest(mf, kubeManifest) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { - return DeleteQCOW2AppManifest(mf) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - return DeleteHelmAppManifest(mf) - } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { - return DeleteDockerSwarmAppManifest(mf) - } - return fmt.Errorf("unknown image type %s", mf.Spec.ImageType) - } -} - func GetDefaultRegistryBase(mf *Manifest, base string) string { mf.Base = base if mf.Base == "" { - mf.Base = fmt.Sprintf("scp://%s/files-repo/mobiledgex", mf.Values.Registry.Name) + mf.Base = fmt.Sprintf("scp://%s/files-repo/mobiledgex", GetCloudletRegistryFileServer()) } log.DebugLog(log.DebugLevelMexos, "default registry base", "base", mf.Base) return mf.Base } -func FillManifestValues(mf *Manifest, kind, base string) error { - if mf.Values.Name == "" { - return fmt.Errorf("no name for mf values") - } - base = GetDefaultRegistryBase(mf, base) - var uri string - switch kind { - case "kubectl": - kind = "platform" - fallthrough - case "openstack": - kind = "platform" - fallthrough - case "platform": - fallthrough - case "cluster": - uri = fmt.Sprintf("%s/%s/%s/%s.yaml", mf.Base, kind, mf.Values.Operator.Name, mf.Values.Base) - case "application": - uri = fmt.Sprintf("%s/%s/%s/%s.yaml", mf.Base, kind, mf.Values.Application.Base, mf.Values.Base) - default: - return fmt.Errorf("invalid manifest kind %s", kind) - } - dat, err := GetURIFile(mf, uri) - if err != nil { - return err - } - //log.DebugLog(log.DebugLevelMexos, "got file", "uri", uri, "data", string(dat)) - tmpl, err := template.New(mf.Values.Name).Parse(string(dat)) - if err != nil { - return err - } - var outbuffer bytes.Buffer - //log.DebugLog(log.DebugLevelMexos, "mf values", "values", mf.Values) - err = tmpl.Execute(&outbuffer, &mf.Values) - if err != nil { - return err - } - err = yaml.Unmarshal(outbuffer.Bytes(), mf) - if err != nil { - return err - } - return nil -} - -func GetDefaultSecurityRule(mf *Manifest) string { - return mf.Values.Network.SecurityRule -} - -func GetMEXSecurityRule(mf *Manifest) string { - return mf.Values.Network.SecurityRule -} - -//GetMEXExternalRouter returns default MEX external router name -func GetMEXExternalRouter(mf *Manifest) string { - //TODO validate existence and status - return mf.Values.Network.Router -} - -//GetMEXExternalNetwork returns default MEX external network name -func GetMEXExternalNetwork(mf *Manifest) string { - //TODO validate existence and status - return mf.Values.Network.External -} - -//GetMEXNetwork returns default MEX network, internal and prepped -func GetMEXNetwork(mf *Manifest) string { - //TODO validate existence and status - return mf.Values.Network.Name -} - -func GetMEXImageName(mf *Manifest) string { - return mf.Values.Cluster.OSImage -} - -func GetMEXUserData(mf *Manifest) string { - return MEXDir() + "/userdata.txt" -} -func GetKubeManifest(mf *Manifest) (string, error) { +func GetKubeManifest(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) (string, error) { var kubeManifest string - rootLB, err := getRootLB(mf.Spec.RootLB) - if err != nil { - return "", fmt.Errorf("cannot get rootlb, while getting kubemanifest, %v", err) - } + base := rootLB.PlatConf.Base if base == "" { log.DebugLog(log.DebugLevelMexos, "base is empty, using default") @@ -383,7 +66,7 @@ func GetKubeManifest(mf *Manifest) (string, error) { deployment := mf.Config.ConfigDetail.Deployment //XXX controlling pass full yaml text in parameter of another yaml log.DebugLog(log.DebugLevelMexos, "getting kubernetes manifest", "base", base, "manifest", mani) - if deployment != cloudcommon.AppDeploymentTypeHelm && !kubeManifestValidApiVersion(mani) { + if deployment != cloudcommon.AppDeploymentTypeHelm && !strings.HasPrefix(mani, "apiVersion: v1") { fn := fmt.Sprintf("%s/%s", base, mani) log.DebugLog(log.DebugLevelMexos, "getting manifest file", "uri", fn) res, err := GetURIFile(mf, fn) @@ -402,3 +85,4 @@ func GetKubeManifest(mf *Manifest) (string, error) { } return kubeManifest, nil } +*/ diff --git a/mexos/misc.go b/mexos/misc.go index 6234ff652..3559d39da 100644 --- a/mexos/misc.go +++ b/mexos/misc.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" "github.com/mobiledgex/edge-cloud/util" ) @@ -55,22 +56,23 @@ func NormalizeName(name string) string { return util.K8SSanitize(name) // XXX } -func SeedDockerSecret(mf *Manifest, rootLB *MEXRootLB) error { +func SeedDockerSecret(clusterName, rootLBName string) error { log.DebugLog(log.DebugLevelMexos, "seed docker secret") - name, err := FindClusterWithKey(mf, mf.Spec.Key) + + master, err := FindClusterMaster(clusterName) if err != nil { return err } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLBName, GetCloudletExternalNetwork(), sshUser) if err != nil { return fmt.Errorf("can't get ssh client for docker swarm, %v", err) } - masteraddr, err := FindNodeIP(mf, name) + masteraddr, err := FindNodeIP(master) if err != nil { return err } var out string - cmd := fmt.Sprintf("echo %s > .docker-pass", mexEnv(mf, "MEX_DOCKER_REG_PASS")) + cmd := fmt.Sprintf("echo %s > .docker-pass", GetCloudletDockerPass()) out, err = client.Output(cmd) if err != nil { return fmt.Errorf("can't store docker password, %s, %v", out, err) @@ -82,23 +84,23 @@ func SeedDockerSecret(mf *Manifest, rootLB *MEXRootLB) error { return fmt.Errorf("can't copy docker password to k8s-master, %s, %v", out, err) } log.DebugLog(log.DebugLevelMexos, "copied over docker password") - cmd = fmt.Sprintf("ssh -o %s -o %s -i id_rsa_mex %s 'cat .docker-pass| docker login -u mobiledgex --password-stdin %s'", sshOpts[0], sshOpts[1], masteraddr, mf.Values.Registry.Docker) + cmd = fmt.Sprintf("ssh -o %s -o %s -i id_rsa_mex %s 'cat .docker-pass| docker login -u mobiledgex --password-stdin %s'", sshOpts[0], sshOpts[1], masteraddr, GetCloudletDockerRegistry()) //TODO allow different docker registry as specified in the manifest out, err = client.Output(cmd) if err != nil { - return fmt.Errorf("can't docker login on k8s-master to %s, %s, %v", mf.Values.Registry.Docker, out, err) + return fmt.Errorf("can't docker login on k8s-master to %s, %s, %v", GetCloudletDockerRegistry(), out, err) } log.DebugLog(log.DebugLevelMexos, "docker login ok") return nil } -func GetHTPassword(mf *Manifest, rootLB *MEXRootLB) error { +func GetHTPassword(rootLBName string) error { log.DebugLog(log.DebugLevelMexos, "get htpasswd") - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLBName, GetCloudletExternalNetwork(), sshUser) if err != nil { return fmt.Errorf("can't get ssh client for docker swarm, %v", err) } - cmd := fmt.Sprintf("scp -o %s -o %s -i id_rsa_mex mobiledgex@%s:files-repo/mobiledgex/%s .", sshOpts[0], sshOpts[1], mf.Values.Registry.Name, HTPasswdFile) + cmd := fmt.Sprintf("scp -o %s -o %s -i id_rsa_mex mobiledgex@%s:files-repo/mobiledgex/%s .", sshOpts[0], sshOpts[1], GetCloudletRegistryFileServer(), HTPasswdFile) out, err := client.Output(cmd) if err != nil { return fmt.Errorf("can't get htpasswd file, %v, %s", err, out) @@ -106,3 +108,7 @@ func GetHTPassword(mf *Manifest, rootLB *MEXRootLB) error { log.DebugLog(log.DebugLevelMexos, "downloaded htpasswd") return nil } + +func GetResourceGroupForCluster(clusterInst *edgeproto.ClusterInst) string { + return clusterInst.Key.CloudletKey.Name + "_" + clusterInst.Key.ClusterKey.Name +} diff --git a/mexos/network.go b/mexos/network.go index dacd8a5cc..7f8998300 100644 --- a/mexos/network.go +++ b/mexos/network.go @@ -13,8 +13,8 @@ import ( // network information. Using that it further gets subnet information. Inside that subnet information // there should be gateway IP if the network is set up correctly. // Not to be confused with GetRouterDetailExternalGateway. -func GetExternalGateway(mf *Manifest, extNetName string) (string, error) { - nd, err := GetNetworkDetail(mf, extNetName) +func GetExternalGateway(extNetName string) (string, error) { + nd, err := GetNetworkDetail(extNetName) if err != nil { return "", fmt.Errorf("can't get details for external network %s, %v", extNetName, err) } @@ -31,7 +31,7 @@ func GetExternalGateway(mf *Manifest, extNetName string) (string, error) { return "", fmt.Errorf("no subnets for %s", extNetName) } //XXX just use first subnet -- may not work in all cases, but there is no tagging done rightly yet - sd, err := GetSubnetDetail(mf, subnets[0]) + sd, err := GetSubnetDetail(subnets[0]) if err != nil { return "", fmt.Errorf("cannot get details for subnet %s, %v", subnets[0], err) } @@ -46,7 +46,7 @@ func GetExternalGateway(mf *Manifest, extNetName string) (string, error) { //GetNextSubnetRange will find the CIDR for the next range of subnet that can be created. For example, // if the subnet detail we get has 10.101.101.0/24 then the next one can be 10.101.102.0/24 func GetNextSubnetRange(mf *Manifest, subnetName string) (string, error) { - sd, err := GetSubnetDetail(mf, subnetName) + sd, err := GetSubnetDetail(subnetName) if err != nil { return "", err } @@ -70,7 +70,7 @@ func GetNextSubnetRange(mf *Manifest, subnetName string) (string, error) { // accessible from private networks to route packets to the external network. // The GetExternalGateway gets the gateway for the outside network. This is // for the packets to be routed out to the external network, i.e. internet. -func GetRouterDetailExternalGateway(mf *Manifest, rd *OSRouterDetail) (*OSExternalGateway, error) { +func GetRouterDetailExternalGateway(rd *OSRouterDetail) (*OSExternalGateway, error) { if rd.ExternalGatewayInfo == "" { return nil, fmt.Errorf("empty external gateway info") } @@ -85,7 +85,7 @@ func GetRouterDetailExternalGateway(mf *Manifest, rd *OSRouterDetail) (*OSExtern // GetRouterDetailInterfaces gets the list of interfaces on the router. For example, each private // subnet connected to the router will be listed here with own interface definition. -func GetRouterDetailInterfaces(mf *Manifest, rd *OSRouterDetail) ([]OSRouterInterface, error) { +func GetRouterDetailInterfaces(rd *OSRouterDetail) ([]OSRouterInterface, error) { if rd.InterfacesInfo == "" { return nil, fmt.Errorf("missing interfaces info in router details") } diff --git a/mexos/nginx.go b/mexos/nginx.go index b60813880..f64bae413 100644 --- a/mexos/nginx.go +++ b/mexos/nginx.go @@ -8,11 +8,11 @@ import ( "github.com/parnurzeal/gorequest" ) -func AddNginxProxy(mf *Manifest, rootLBName, name, ipaddr string, ports []PortDetail, network string) error { +func AddNginxProxy(rootLBName, name, ipaddr string, ports []PortDetail, network string) error { log.DebugLog(log.DebugLevelMexos, "add nginx proxy", "name", name, "network", network, "ports", ports) request := gorequest.New() - npURI := fmt.Sprintf("http://%s:%s/v1/nginx", rootLBName, mf.Values.Agent.Port) + npURI := fmt.Sprintf("http://%s:%s/v1/nginx", rootLBName, GetCloudletMexosAgentPort()) pl, err := FormNginxProxyRequest(ports, ipaddr, name, network) if err != nil { log.DebugLog(log.DebugLevelMexos, "cannot form nginx proxy request") @@ -73,10 +73,10 @@ func FormNginxProxyRequest(ports []PortDetail, ipaddr string, name string, netwo return &pl, nil } -func DeleteNginxProxy(mf *Manifest, rootLBName, name string) error { +func DeleteNginxProxy(rootLBName, name string) error { log.DebugLog(log.DebugLevelMexos, "delete nginx proxy", "name", name) request := gorequest.New() - npURI := fmt.Sprintf("http://%s:%s/v1/nginx", rootLBName, mf.Values.Agent.Port) + npURI := fmt.Sprintf("http://%s:%s/v1/nginx", rootLBName, GetCloudletMexosAgentPort()) pl := fmt.Sprintf(`{"message":"delete","name":"%s"}`, name) log.DebugLog(log.DebugLevelMexos, "nginx proxy delete request post", "request", pl) resp, body, errs := request.Post(npURI).Set("Content-Type", "application/json").Send(pl).End() @@ -91,10 +91,10 @@ func DeleteNginxProxy(mf *Manifest, rootLBName, name string) error { return fmt.Errorf("cannot delete nginx proxy, resp %v", resp) } -func AddNginxKubectlProxy(mf *Manifest, rootLBName, name string, portnum int) error { +func AddNginxKubectlProxy(rootLBName, name string, portnum int) error { log.DebugLog(log.DebugLevelMexos, "add nginx kubectl proxy", "name", name) request := gorequest.New() - npURI := fmt.Sprintf("http://%s:%s/v1/nginx-kcp", rootLBName, mf.Values.Agent.Port) + npURI := fmt.Sprintf("http://%s:%s/v1/nginx-kcp", rootLBName, GetCloudletMexosAgentPort()) pl, err := FormNginxKCProxyRequest(name, portnum) if err != nil { log.DebugLog(log.DebugLevelMexos, "cannot form nginx kubectl proxy request") @@ -113,9 +113,9 @@ func AddNginxKubectlProxy(mf *Manifest, rootLBName, name string, portnum int) er return fmt.Errorf("cannot add nginx kubectl proxy, resp %v", resp) } -func DeleteNginxKCProxy(mf *Manifest, rootLBName, name string) error { +func DeleteNginxKCProxy(rootLBName, name string) error { log.DebugLog(log.DebugLevelMexos, "deleting nginx kubectl proxy", "name", name) - client, err := GetSSHClient(mf, rootLBName, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLBName, GetCloudletExternalNetwork(), sshUser) if err != nil { return err } @@ -124,7 +124,7 @@ func DeleteNginxKCProxy(mf *Manifest, rootLBName, name string) error { log.DebugLog(log.DebugLevelMexos, "warning, cannot delete container", "name", name+kcproxySuffix, "error", err, "out", out) } request := gorequest.New() - npURI := fmt.Sprintf("http://%s:%s/v1/nginx-kcp", rootLBName, mf.Values.Agent.Port) + npURI := fmt.Sprintf("http://%s:%s/v1/nginx-kcp", rootLBName, GetCloudletMexosAgentPort()) pl := fmt.Sprintf(`{"message":"delete","name":"%s"}`, name) log.DebugLog(log.DebugLevelMexos, "nginx kubectl proxy delete request post", "request", pl) resp, body, errs := request.Post(npURI).Set("Content-Type", "application/json").Send(pl).End() diff --git a/mexos/oscli.go b/mexos/oscli.go index 6e85269f8..91e3bdd0e 100644 --- a/mexos/oscli.go +++ b/mexos/oscli.go @@ -11,7 +11,7 @@ import ( ) //ListServers returns list of servers, KVM instances, running on the system -func ListServers(mf *Manifest) ([]OSServer, error) { +func ListServers() ([]OSServer, error) { out, err := sh.Command("openstack", "server", "list", "-f", "json").Output() if err != nil { err = fmt.Errorf("cannot get server list, %v", err) @@ -45,7 +45,7 @@ func ListImages(mf *Manifest) ([]OSImage, error) { } //ListNetworks lists networks known to the platform. Some created by the operator, some by users. -func ListNetworks(mf *Manifest) ([]OSNetwork, error) { +func ListNetworks() ([]OSNetwork, error) { out, err := sh.Command("openstack", "network", "list", "-f", "json").Output() if err != nil { err = fmt.Errorf("cannot get network list, %v", err) @@ -62,7 +62,7 @@ func ListNetworks(mf *Manifest) ([]OSNetwork, error) { } //ListFlavors lists flavors known to the platform. These vary. On Buckhorn cloudlet the are m4. prefixed. -func ListFlavors(mf *Manifest) ([]OSFlavor, error) { +func ListFlavors() ([]OSFlavor, error) { out, err := sh.Command("openstack", "flavor", "list", "-f", "json").Output() if err != nil { err = fmt.Errorf("cannot get flavor list, %v", err) @@ -79,7 +79,7 @@ func ListFlavors(mf *Manifest) ([]OSFlavor, error) { } //CreateServer instantiates a new server instance, which is a KVM instance based on a qcow2 image from glance -func CreateServer(mf *Manifest, opts *OSServerOpt) error { +func CreateServer(opts *OSServerOpt) error { args := []string{ "server", "create", "--config-drive", "true", //XXX always @@ -102,8 +102,10 @@ func CreateServer(mf *Manifest, opts *OSServerOpt) error { for i, v := range args { iargs[i] = v } + log.DebugLog(log.DebugLevelMexos, "creating server with args", "iargs", iargs) + //log.DebugLog(log.DebugLevelMexos, "openstack create server", "opts", opts, "iargs", iargs) - out, err := sh.Command("openstack", iargs...).Output() + out, err := sh.Command("openstack", iargs...).CombinedOutput() if err != nil { err = fmt.Errorf("cannot create server, %v, '%s'", err, out) return err @@ -112,7 +114,7 @@ func CreateServer(mf *Manifest, opts *OSServerOpt) error { } // GetServerDetails returns details of the KVM instance -func GetServerDetails(mf *Manifest, name string) (*OSServerDetail, error) { +func GetServerDetails(name string) (*OSServerDetail, error) { active := false srvDetail := &OSServerDetail{} for i := 0; i < 10; i++ { @@ -143,9 +145,9 @@ func GetServerDetails(mf *Manifest, name string) (*OSServerDetail, error) { //DeleteServer destroys a KVM instance // sometimes it is not possible to destroy. Like most things in Openstack, try again. -func DeleteServer(mf *Manifest, id string) error { +func DeleteServer(id string) error { log.DebugLog(log.DebugLevelMexos, "deleting server", "id", id) - out, err := sh.Command("openstack", "server", "delete", id).Output() + out, err := sh.Command("openstack", "server", "delete", id).CombinedOutput() if err != nil { err = fmt.Errorf("can't delete server %s, %s, %v", id, out, err) return err @@ -154,7 +156,7 @@ func DeleteServer(mf *Manifest, id string) error { } // CreateNetwork creates a network with a name. -func CreateNetwork(mf *Manifest, name string) error { +func CreateNetwork(name string) error { log.DebugLog(log.DebugLevelMexos, "creating network", "network", name) out, err := sh.Command("openstack", "network", "create", name).Output() if err != nil { @@ -168,7 +170,7 @@ func CreateNetwork(mf *Manifest, name string) error { // Sometimes it will fail. Openstack will refuse if there are resources attached. func DeleteNetwork(mf *Manifest, name string) error { log.DebugLog(log.DebugLevelMexos, "deleting network", "network", name) - out, err := sh.Command("openstack", "network", "delete", name).Output() + out, err := sh.Command("openstack", "network", "delete", name).CombinedOutput() if err != nil { err = fmt.Errorf("can't delete network %s, %s, %v", name, out, err) return err @@ -177,7 +179,7 @@ func DeleteNetwork(mf *Manifest, name string) error { } //CreateSubnet creates a subnet within a network. A subnet is assigned ranges. Optionally DHCP can be enabled. -func CreateSubnet(mf *Manifest, netRange, networkName, gatewayAddr, subnetName string, dhcpEnable bool) error { +func CreateSubnet(netRange, networkName, gatewayAddr, subnetName string, dhcpEnable bool) error { var dhcpFlag string if dhcpEnable { dhcpFlag = "--dhcp" @@ -199,7 +201,7 @@ func CreateSubnet(mf *Manifest, netRange, networkName, gatewayAddr, subnetName s return err } if strings.Index(nerr.NeutronError.Message, "overlap") > 0 { - sd, serr := GetSubnetDetail(mf, subnetName) + sd, serr := GetSubnetDetail(subnetName) if serr != nil { return fmt.Errorf("cannot get subnet detail for %s, while fixing overlap error, %v", subnetName, serr) } @@ -218,7 +220,7 @@ func CreateSubnet(mf *Manifest, netRange, networkName, gatewayAddr, subnetName s } //DeleteSubnet deletes the subnet. If this fails, remove any attached resources, like router, and try again. -func DeleteSubnet(mf *Manifest, subnetName string) error { +func DeleteSubnet(subnetName string) error { log.DebugLog(log.DebugLevelMexos, "deleting subnet", "name", subnetName) out, err := sh.Command("openstack", "subnet", "delete", subnetName).Output() if err != nil { @@ -229,7 +231,7 @@ func DeleteSubnet(mf *Manifest, subnetName string) error { } //CreateRouter creates new router. A router can be attached to network and subnets. -func CreateRouter(mf *Manifest, routerName string) error { +func CreateRouter(routerName string) error { log.DebugLog(log.DebugLevelMexos, "creating router", "name", routerName) out, err := sh.Command("openstack", "router", "create", routerName).Output() if err != nil { @@ -253,9 +255,9 @@ func DeleteRouter(mf *Manifest, routerName string) error { //SetRouter assigns the router to a particular network. The network needs to be attached to // a real external network. This is intended only for routing to external network for now. No internal routers. // Sometimes, oftentimes, it will fail if the network is not external. -func SetRouter(mf *Manifest, routerName, networkName string) error { +func SetRouter(routerName, networkName string) error { log.DebugLog(log.DebugLevelMexos, "setting router to network", "router", routerName, "network", networkName) - out, err := sh.Command("openstack", "router", "set", routerName, "--external-gateway", networkName).Output() + out, err := sh.Command("openstack", "router", "set", routerName, "--external-gateway", networkName).CombinedOutput() if err != nil { err = fmt.Errorf("can't set router %s to %s, %s, %v", routerName, networkName, out, err) return err @@ -264,9 +266,9 @@ func SetRouter(mf *Manifest, routerName, networkName string) error { } //AddRouterSubnet will connect subnet to another network, possibly external, via a router -func AddRouterSubnet(mf *Manifest, routerName, subnetName string) error { +func AddRouterSubnet(routerName, subnetName string) error { log.DebugLog(log.DebugLevelMexos, "adding router to subnet", "router", routerName, "network", subnetName) - out, err := sh.Command("openstack", "router", "add", "subnet", routerName, subnetName).Output() + out, err := sh.Command("openstack", "router", "add", "subnet", routerName, subnetName).CombinedOutput() if err != nil { err = fmt.Errorf("can't add router %s to subnet %s, %s, %v", routerName, subnetName, out, err) return err @@ -276,9 +278,9 @@ func AddRouterSubnet(mf *Manifest, routerName, subnetName string) error { //RemoveRouterSubnet is useful to remove the router from the subnet before deletion. Otherwise subnet cannot // be deleted. -func RemoveRouterSubnet(mf *Manifest, routerName, subnetName string) error { +func RemoveRouterSubnet(routerName, subnetName string) error { log.DebugLog(log.DebugLevelMexos, "removing subnet from router", "router", routerName, "subnet", subnetName) - out, err := sh.Command("openstack", "router", "remove", "subnet", routerName, subnetName).Output() + out, err := sh.Command("openstack", "router", "remove", "subnet", routerName, subnetName).CombinedOutput() if err != nil { err = fmt.Errorf("can't remove router %s from subnet %s, %s, %v", routerName, subnetName, out, err) return err @@ -287,7 +289,7 @@ func RemoveRouterSubnet(mf *Manifest, routerName, subnetName string) error { } //ListSubnets returns a list of subnets available -func ListSubnets(mf *Manifest, netName string) ([]OSSubnet, error) { +func ListSubnets(netName string) ([]OSSubnet, error) { var err error var out []byte @@ -311,7 +313,7 @@ func ListSubnets(mf *Manifest, netName string) ([]OSSubnet, error) { } //ListRouters returns a list of routers available -func ListRouters(mf *Manifest) ([]OSRouter, error) { +func ListRouters() ([]OSRouter, error) { out, err := sh.Command("openstack", "router", "list", "-f", "json").Output() if err != nil { err = fmt.Errorf("can't get a list of routers, %s, %v", out, err) @@ -328,7 +330,7 @@ func ListRouters(mf *Manifest) ([]OSRouter, error) { } //GetRouterDetail returns details per router -func GetRouterDetail(mf *Manifest, routerName string) (*OSRouterDetail, error) { +func GetRouterDetail(routerName string) (*OSRouterDetail, error) { out, err := sh.Command("openstack", "router", "show", "-f", "json", routerName).Output() if err != nil { err = fmt.Errorf("can't get router details for %s, %s, %v", routerName, out, err) @@ -345,7 +347,7 @@ func GetRouterDetail(mf *Manifest, routerName string) (*OSRouterDetail, error) { } //CreateServerImage snapshots running service into a qcow2 image -func CreateServerImage(mf *Manifest, serverName, imageName string) error { +func CreateServerImage(serverName, imageName string) error { log.DebugLog(log.DebugLevelMexos, "creating image snapshot from server", "server", serverName, "image", imageName) out, err := sh.Command("openstack", "server", "image", "create", serverName, "--name", imageName).Output() if err != nil { @@ -400,7 +402,7 @@ func DeleteImage(mf *Manifest, imageName string) error { //GetSubnetDetail returns details for the subnet. This is useful when getting router/gateway // IP for a given subnet. The gateway info is used for creating a server. // Also useful in general, like other `detail` functions, to get the ID map for the name of subnet. -func GetSubnetDetail(mf *Manifest, subnetName string) (*OSSubnetDetail, error) { +func GetSubnetDetail(subnetName string) (*OSSubnetDetail, error) { out, err := sh.Command("openstack", "subnet", "show", "-f", "json", subnetName).Output() if err != nil { err = fmt.Errorf("can't get subnet details for %s, %s, %v", subnetName, out, err) @@ -416,7 +418,7 @@ func GetSubnetDetail(mf *Manifest, subnetName string) (*OSSubnetDetail, error) { } //GetNetworkDetail returns details about a network. It is used, for example, by GetExternalGateway. -func GetNetworkDetail(mf *Manifest, networkName string) (*OSNetworkDetail, error) { +func GetNetworkDetail(networkName string) (*OSNetworkDetail, error) { out, err := sh.Command("openstack", "network", "show", "-f", "json", networkName).Output() if err != nil { err = fmt.Errorf("can't get details for network %s, %s, %v", networkName, out, err) @@ -432,7 +434,7 @@ func GetNetworkDetail(mf *Manifest, networkName string) (*OSNetworkDetail, error } //SetServerProperty sets properties for the server -func SetServerProperty(mf *Manifest, name, property string) error { +func SetServerProperty(name, property string) error { if name == "" { return fmt.Errorf("empty name") } diff --git a/mexos/platform.go b/mexos/platform.go index 8c9f62b75..2b8dafa39 100644 --- a/mexos/platform.go +++ b/mexos/platform.go @@ -1,11 +1,6 @@ package mexos -import ( - "fmt" - - "github.com/mobiledgex/edge-cloud/log" -) - +/* func setPlatConfManifest(mf *Manifest) error { rootLB, err := getRootLB(mf.Spec.RootLB) if err != nil { @@ -18,6 +13,7 @@ func setPlatConfManifest(mf *Manifest) error { return nil } + func setPlatConf(rootLB *MEXRootLB, mf *Manifest) { log.DebugLog(log.DebugLevelMexos, "rootlb platconf set") if rootLB == nil { @@ -25,3 +21,4 @@ func setPlatConf(rootLB *MEXRootLB, mf *Manifest) { } rootLB.PlatConf = mf } +*/ diff --git a/mexos/ports.go b/mexos/ports.go index 3d83d393c..309ed1825 100644 --- a/mexos/ports.go +++ b/mexos/ports.go @@ -8,15 +8,13 @@ import ( "github.com/mobiledgex/edge-cloud/edgeproto" ) -func addPorts(mf *Manifest, appInst *edgeproto.AppInst) error { - for ii, _ := range appInst.MappedPorts { +func GetPortDetail(appInst *edgeproto.AppInst) ([]PortDetail, error) { + ports := make([]PortDetail, 0) + for ii := range appInst.MappedPorts { port := &appInst.MappedPorts[ii] - if mf.Spec.Ports == nil { - mf.Spec.Ports = make([]PortDetail, 0) - } mexproto, ok := dme.LProto_name[int32(port.Proto)] if !ok { - return fmt.Errorf("invalid LProto %d", port.Proto) + return nil, fmt.Errorf("invalid LProto %d", port.Proto) } proto := "UDP" if port.Proto != dme.LProto_LProtoUDP { @@ -30,7 +28,7 @@ func addPorts(mf *Manifest, appInst *edgeproto.AppInst) error { PublicPort: int(port.PublicPort), PublicPath: port.PublicPath, } - mf.Spec.Ports = append(mf.Spec.Ports, p) + ports = append(ports, p) } - return nil + return ports, nil } diff --git a/mexos/rootlb.go b/mexos/rootlb.go index 665d63c55..3411a2359 100644 --- a/mexos/rootlb.go +++ b/mexos/rootlb.go @@ -2,16 +2,16 @@ package mexos import ( "fmt" - "strings" "time" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) //MEXRootLB has rootLB data type MEXRootLB struct { - Name string - PlatConf *Manifest + Name string + // PlatConf *Manifest } var MEXRootLBMap = make(map[string]*MEXRootLB) @@ -27,20 +27,6 @@ func NewRootLB(rootLBName string) (*MEXRootLB, error) { return newRootLB, nil } -//NewRootLBManifest creates rootLB instance and sets Platform Config with manifest -func NewRootLBManifest(mf *Manifest) (*MEXRootLB, error) { - log.DebugLog(log.DebugLevelMexos, "getting new rootLB with manifest", "name", mf.Spec.RootLB) - rootLB, err := NewRootLB(mf.Spec.RootLB) - if err != nil { - return nil, err - } - setPlatConf(rootLB, mf) - if rootLB == nil { - log.DebugLog(log.DebugLevelMexos, "error, newrootlbmanifest, rootLB is null") - } - return rootLB, nil -} - //DeleteRootLB to be called by code that called NewRootLB func DeleteRootLB(rootLBName string) { delete(MEXRootLBMap, rootLBName) //no mutex because caller should be serializing New/Delete in a control loop @@ -70,25 +56,23 @@ var rootLBPorts = []int{ //EnableRootLB creates a seed presence node in cloudlet that also becomes first Agent node. // It also sets up first basic network router and subnet, ready for running first MEX agent. -func EnableRootLB(mf *Manifest, rootLB *MEXRootLB) error { +func EnableRootLB(rootLB *MEXRootLB, cloudletKey *edgeproto.CloudletKey) error { log.DebugLog(log.DebugLevelMexos, "enable rootlb", "name", rootLB.Name) if rootLB == nil { return fmt.Errorf("cannot enable rootLB, rootLB is null") } - if mf.Values.Network.External == "" { + if GetCloudletExternalNetwork() == "" { return fmt.Errorf("enable rootlb, missing external network in manifest") } - err := PrepNetwork(mf) + + err := PrepNetwork() if err != nil { return err } - sl, err := ListServers(mf) + sl, err := ListServers() if err != nil { return err } - if mf.Metadata.DNSZone == "" { - return fmt.Errorf("missing dns zone in manifest, metadata %v", mf.Metadata) - } found := 0 for _, s := range sl { if s.Name == rootLB.Name { @@ -98,22 +82,27 @@ func EnableRootLB(mf *Manifest, rootLB *MEXRootLB) error { } if found == 0 { log.DebugLog(log.DebugLevelMexos, "not found existing server", "name", rootLB.Name) - netspec := fmt.Sprintf("external-ip,%s", mf.Values.Network.External) - if strings.Contains(mf.Spec.Options, "dhcp") { - netspec = netspec + ",dhcp" - } - cf, err := GetClusterFlavor(mf.Spec.Flavor) + netspec := fmt.Sprintf("external-ip,%s", GetCloudletExternalNetwork()) + //if strings.Contains(mf.Spec.Options, "dhcp") { TODO + netspec = netspec + ",dhcp" + //} + flavor := GetCloudletRootLBFlavor() + cf, err := GetClusterFlavor(flavor) if err != nil { log.DebugLog(log.DebugLevelMexos, "invalid platform flavor, can't create rootLB") return fmt.Errorf("cannot create rootLB invalid platform flavor %v", err) } + + tags := cloudletKey.Name + "-tag" + log.DebugLog(log.DebugLevelMexos, "creating agent node kvm", "netspec", netspec) - err = CreateMEXKVM(mf, rootLB.Name, + err = CreateMEXKVM(rootLB.Name, "mex-agent-node", //important, don't change netspec, - mf.Metadata.Tags, - mf.Metadata.Tenant, + tags, + GetCloudletTenant(), 1, + nil, // cluster not needed for rootlb cf.PlatformFlavor, ) if err != nil { @@ -122,12 +111,12 @@ func EnableRootLB(mf *Manifest, rootLB *MEXRootLB) error { } log.DebugLog(log.DebugLevelMexos, "created kvm instance", "name", rootLB.Name) - //rootLBIPaddr, ierr := GetServerIPAddr(mf, mf.Values.Network.External, rootLB.Name) + //rootLBIPaddr, ierr := GetServerIPAddr(GetCloudletExternalNetwork(), rootLB.Name) // if ierr != nil { // log.DebugLog(log.DebugLevelMexos, "cannot get rootlb IP address", "error", ierr) // return fmt.Errorf("created rootlb but cannot get rootlb IP") // } - ruleName := GetMEXSecurityRule(mf) + ruleName := GetCloudletSecurityRule() //privateNetCIDR := strings.Replace(defaultPrivateNetRange, "X", "0", 1) allowedClientCIDR := GetAllowedClientCIDR() for _, p := range rootLBPorts { @@ -139,7 +128,7 @@ func EnableRootLB(mf *Manifest, rootLB *MEXRootLB) error { // } // }(cidr) // } - if err := AddSecurityRuleCIDR(mf, allowedClientCIDR, "tcp", ruleName, p); err != nil { + if err := AddSecurityRuleCIDR(allowedClientCIDR, "tcp", ruleName, p); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, cannot add security rule", "error", err, "cidr", allowedClientCIDR, "port", p, "rule", ruleName) } } @@ -155,16 +144,17 @@ func EnableRootLB(mf *Manifest, rootLB *MEXRootLB) error { //WaitForRootLB waits for the RootLB instance to be up and copies of SSH credentials for internal networks. // Idempotent, but don't call all the time. -func WaitForRootLB(mf *Manifest, rootLB *MEXRootLB) error { +func WaitForRootLB(rootLB *MEXRootLB) error { log.DebugLog(log.DebugLevelMexos, "wait for rootlb", "name", rootLB.Name) if rootLB == nil { return fmt.Errorf("cannot wait for lb, rootLB is null") } - if mf.Values.Network.External == "" { + extNet := GetCloudletExternalNetwork() + if extNet == "" { return fmt.Errorf("waiting for lb, missing external network in manifest") } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, extNet, sshUser) if err != nil { return err } @@ -175,7 +165,7 @@ func WaitForRootLB(mf *Manifest, rootLB *MEXRootLB) error { if err == nil { log.DebugLog(log.DebugLevelMexos, "rootlb is running", "name", rootLB.Name) running = true - //if err := CopySSHCredential(mf, rootLB.Name, mf.Values.Network.External, "root"); err != nil { + //if err := CopySSHCredential(mf, rootLB.Name, GetCloudletExternalNetwork(), "root"); err != nil { // return fmt.Errorf("can't copy ssh credential to RootLB, %v", err) //} break diff --git a/mexos/securityrule.go b/mexos/securityrule.go index 590ba2df7..443a21e72 100644 --- a/mexos/securityrule.go +++ b/mexos/securityrule.go @@ -6,20 +6,24 @@ import ( "strings" sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) // TODO service to periodically clean up the leftover rules -func AddProxySecurityRules(rootLB *MEXRootLB, mf *Manifest, masteraddr string) error { - // rootLBIPaddr, err := GetServerIPAddr(mf, mf.Values.Network.External, rootLB.Name) - // if err != nil { - // log.DebugLog(log.DebugLevelMexos, "cannot get rootlb IP address", "error", err) - // return fmt.Errorf("cannot deploy kubernetes app, cannot get rootlb IP") - // } - sr := GetMEXSecurityRule(mf) +func AddProxySecurityRules(rootLB *MEXRootLB, masteraddr string, appInst *edgeproto.AppInst) error { + + name := NormalizeName(appInst.Key.AppKey.Name) + + ports, err := GetPortDetail(appInst) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "GetPortDetail failed", "err", err) + return err + } + sr := GetCloudletSecurityRule() allowedClientCIDR := GetAllowedClientCIDR() - for _, port := range mf.Spec.Ports { + for _, port := range ports { for _, sec := range []struct { addr string port int @@ -33,35 +37,38 @@ func AddProxySecurityRules(rootLB *MEXRootLB, mf *Manifest, masteraddr string) e // log.DebugLog(log.DebugLevelMexos, "warning, error while adding security rule", "cidr", addr, "securityrule", sr, "port", port, "proto", proto) // } // }(sec.addr, sec.port, port.Proto) - if err := AddSecurityRuleCIDR(mf, sec.addr, strings.ToLower(port.Proto), sr, sec.port); err != nil { + if err := AddSecurityRuleCIDR(sec.addr, strings.ToLower(port.Proto), sr, sec.port); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, error while adding security rule", "addr", sec.addr, "port", sec.port, "proto", port.Proto) } } } - if len(mf.Spec.Ports) > 0 { - if err := AddNginxProxy(mf, rootLB.Name, mf.Metadata.Name, masteraddr, mf.Spec.Ports, ""); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports) + if len(ports) > 0 { + if err := AddNginxProxy(rootLB.Name, name, masteraddr, ports, ""); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", name) return err } } - log.DebugLog(log.DebugLevelMexos, "added nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports) + log.DebugLog(log.DebugLevelMexos, "added nginx proxy", "name", name, "ports", appInst.MappedPorts) return nil } -func DeleteProxySecurityRules(rootLB *MEXRootLB, mf *Manifest, ipaddr string) error { - log.DebugLog(log.DebugLevelMexos, "delete spec ports", "ports", mf.Spec.Ports) - err := DeleteNginxProxy(mf, rootLB.Name, mf.Metadata.Name) +func DeleteProxySecurityRules(rootLB *MEXRootLB, ipaddr string, appInst *edgeproto.AppInst) error { + appName := NormalizeName(appInst.Key.AppKey.Name) + + log.DebugLog(log.DebugLevelMexos, "delete proxy rules", "name", appName) + err := DeleteNginxProxy(rootLB.Name, appName) + if err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", mf.Metadata.Name, "rootlb", rootLB.Name, "error", err) + log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", appName, "rootlb", rootLB.Name, "error", err) } - if err := DeleteSecurityRule(mf, ipaddr); err != nil { + if err := DeleteSecurityRule(ipaddr); err != nil { return err } // TODO - implement the clean up of security rules return nil } -func AddSecurityRuleCIDR(mf *Manifest, cidr string, proto string, name string, port int) error { +func AddSecurityRuleCIDR(cidr string, proto string, name string, port int) error { portStr := fmt.Sprintf("%d", port) out, err := sh.Command("openstack", "security", "group", "rule", "create", "--remote-ip", cidr, "--proto", proto, "--dst-port", portStr, "--ingress", name).Output() if err != nil { @@ -78,7 +85,7 @@ type SecurityRule struct { Proto string `json:"IP Protocol"` } -func DeleteSecurityRule(mf *Manifest, sip string) error { +func DeleteSecurityRule(sip string) error { sr := []SecurityRule{} dat, err := sh.Command("openstack", "security", "group", "rule", "list", "-f", "json").Output() if err != nil { diff --git a/mexos/ssh.go b/mexos/ssh.go index db4716b5a..7d525be04 100644 --- a/mexos/ssh.go +++ b/mexos/ssh.go @@ -15,7 +15,7 @@ var sshUser = "ubuntu" func CopySSHCredential(mf *Manifest, serverName, networkName, userName string) error { //TODO multiple keys to be copied and added to authorized_keys if needed log.DebugLog(log.DebugLevelMexos, "copying ssh credentials", "server", serverName, "network", networkName, "user", userName) - addr, err := GetServerIPAddr(mf, networkName, serverName) + addr, err := GetServerIPAddr(networkName, serverName) if err != nil { return err } @@ -28,9 +28,9 @@ func CopySSHCredential(mf *Manifest, serverName, networkName, userName string) e } //GetSSHClient returns ssh client handle for the server -func GetSSHClient(mf *Manifest, serverName, networkName, userName string) (ssh.Client, error) { +func GetSSHClient(serverName, networkName, userName string) (ssh.Client, error) { auth := ssh.Auth{Keys: []string{PrivateSSHKey()}} - addr, err := GetServerIPAddr(mf, networkName, serverName) + addr, err := GetServerIPAddr(networkName, serverName) if err != nil { return nil, err } @@ -42,7 +42,7 @@ func GetSSHClient(mf *Manifest, serverName, networkName, userName string) (ssh.C return client, nil } -func GetSSHClientIP(mf *Manifest, ipaddr, userName string) (ssh.Client, error) { +func GetSSHClientIP(ipaddr, userName string) (ssh.Client, error) { auth := ssh.Auth{Keys: []string{PrivateSSHKey()}} client, err := ssh.NewNativeClient(userName, ipaddr, "SSH-2.0-mobiledgex-ssh-client-1.0", 22, &auth, nil) if err != nil { @@ -51,9 +51,9 @@ func GetSSHClientIP(mf *Manifest, ipaddr, userName string) (ssh.Client, error) { return client, nil } -func SetupSSHUser(mf *Manifest, rootLB *MEXRootLB, user string) error { +func SetupSSHUser(rootLB *MEXRootLB, user string) error { log.DebugLog(log.DebugLevelMexos, "setting up ssh user", "user", user) - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, user) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), user) if err != nil { return err } diff --git a/mexos/stats.go b/mexos/stats.go index d01c4b0d5..0291fa199 100644 --- a/mexos/stats.go +++ b/mexos/stats.go @@ -9,8 +9,8 @@ import ( ) //GetLimits is used to retrieve tenant level platform stats -func GetLimits(mf *Manifest) ([]OSLimit, error) { - log.DebugLog(log.DebugLevelMexos, "GetLimits", "mf value name", mf.Values.Name) +func GetLimits() ([]OSLimit, error) { + log.DebugLog(log.DebugLevelMexos, "GetLimits") //err := sh.Command("openstack", "limits", "show", "--absolute", "-f", "json", sh.Dir("/tmp")).WriteStdout("os-out.txt") out, err := sh.Command("openstack", "limits", "show", "--absolute", "-f", "json", sh.Dir("/tmp")).Output() if err != nil { diff --git a/mexos/swarm.go b/mexos/swarm.go index 64eb36603..eeac38682 100644 --- a/mexos/swarm.go +++ b/mexos/swarm.go @@ -1,26 +1,18 @@ package mexos -import ( - "fmt" - "strconv" - "strings" - - "github.com/ghodss/yaml" - "github.com/mobiledgex/edge-cloud/log" -) - -func CreateDockerSwarm(mf *Manifest, rootLB *MEXRootLB) error { +/* +func CreateDockerSwarm(clusterName, rootLBName) error { //TODO independent swarm cluster without k8s - log.DebugLog(log.DebugLevelMexos, "creating docker swarm", "name", mf.Metadata.Swarm) - name, err := FindClusterWithKey(mf, mf.Spec.Key) + log.DebugLog(log.DebugLevelMexos, "creating docker swarm", "name", clusterName) + name, err := FindClusterWithKey(clusterName) if err != nil { return err } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return fmt.Errorf("can't get ssh client for docker swarm, %v", err) } - masteraddr, err := FindNodeIP(mf, name) + masteraddr, err := FindNodeIP(name) if err != nil { return err } @@ -69,6 +61,7 @@ func CreateDockerSwarm(mf *Manifest, rootLB *MEXRootLB) error { return nil } + //TODO make it support full docker-compose file spec. type DockerService struct { @@ -82,6 +75,7 @@ type DockerCompose struct { Services map[string]DockerService `json:"services"` } + func CreateDockerSwarmAppManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "create docker-swarm app") rootLB, err := getRootLB(mf.Spec.RootLB) @@ -95,15 +89,15 @@ func CreateDockerSwarmAppManifest(mf *Manifest) error { if err != nil { return fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) } - masteraddr, err := FindNodeIP(mf, name) + masteraddr, err := FindNodeIP(name) if err != nil { return err } var cmd string - if mexEnv(mf, "MEX_DOCKER_REG_PASS") == "" { + if GetCloudletDockerPass() == "" { return fmt.Errorf("empty docker registry password environment variable") } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return err } @@ -199,15 +193,15 @@ func DeleteDockerSwarmAppManifest(mf *Manifest) error { if err != nil { return fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) } - masteraddr, err := FindNodeIP(mf, name) + masteraddr, err := FindNodeIP(name) if err != nil { return err } var cmd string - if mexEnv(mf, "MEX_DOCKER_REG_PASS") == "" { + if GetCloudletDockerPass() == "" { return fmt.Errorf("empty docker registry password environment variable") } - client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return err } @@ -236,7 +230,7 @@ func DeleteDockerSwarmAppManifest(mf *Manifest) error { return err } log.DebugLog(log.DebugLevelMexos, "removing proxy and security rules", "name", mf.Metadata.Name) - if err = DeleteProxySecurityRules(rootLB, mf, masteraddr); err != nil { + if err = DeleteProxySecurityRules(rootLB, mf, masteraddr, appInst); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) return err } @@ -244,3 +238,4 @@ func DeleteDockerSwarmAppManifest(mf *Manifest) error { // TODO add custom DNS entries per app service endpoints return nil } +*/ diff --git a/mexos/templates.go b/mexos/templates.go index c89d3090e..904e37b01 100644 --- a/mexos/templates.go +++ b/mexos/templates.go @@ -1,30 +1,18 @@ package mexos -import ( - "bytes" - "encoding/json" - "fmt" - "strings" - "text/template" - - "github.com/ghodss/yaml" - "github.com/mobiledgex/edge-cloud/cloudcommon" - "github.com/mobiledgex/edge-cloud/edgeproto" - "github.com/mobiledgex/edge-cloud/log" -) - +/* type templateFill struct { - Name, Kind, Flavor, Tags, Tenant, Region, Zone, DNSZone string - ImageFlavor, Location, RootLB, Resource, ResourceKind, ResourceGroup string - StorageSpec, NetworkScheme, MasterFlavor, Topology string - NodeFlavor, Operator, Key, Image, Options string - ImageType, AppURI, ProxyPath, AgentImage string - ExternalNetwork, Project string - ExternalRouter, Flags, IPAccess, Swarm string - NumMasters, NumNodes int - Config templateConfig - Command []string - SpecPorts []PortDetail + Name, Kind, Flavor, Tags, Tenant, DNSZone string + ImageFlavor, RootLB, Resource, ResourceKind, ResourceGroup string + StorageSpec, NetworkScheme string + NodeFlavor, Operator, Key, Image, Options string + ImageType, ProxyPath string + ExternalNetwork, Project string + ExternalRouter, Flags, IPAccess, Swarm string + NumMasters, NumNodes int + Config templateConfig + Command []string + SpecPorts []PortDetail } type templateConfig struct { @@ -40,9 +28,6 @@ metadata: kind: {{.Kind}} tenant: {{.Tenant}} operator: {{.Operator}} - region: {{.Region}} - zone: {{.Zone}} - location: {{.Location}} project: {{.Project}} dnszone: {{.DNSZone}} swarm: {{.Swarm}} @@ -66,7 +51,7 @@ metadata: spec: flags: {{.Flags}} flavor: {{.Name}} - flavordetail: + flavordetail: name: {{.Name}} nodes: {{.NumNodes}} masters: {{.NumMasters}} @@ -85,9 +70,6 @@ metadata: name: {{.Name}} tags: {{.Tags}} tenant: {{.Tenant}} - region: {{.Region}} - zone: {{.Zone}} - location: {{.Location}} openrc: ~/.mobiledgex/openrc dnszone: {{.DNSZone}} operator: {{.Operator}} @@ -101,9 +83,6 @@ spec: networkscheme: {{.NetworkScheme}} externalrouter: {{.ExternalRouter}} options: {{.Options}} - agent: - image: {{.AgentImage}} - status: active ` var yamlMEXApp = `apiVersion: v1 @@ -135,7 +114,6 @@ spec: imageflavor: {{.ImageFlavor}} proxypath: {{.ProxyPath}} flavor: {{.Flavor}} - uri: {{.AppURI}} ipaccess: {{.IPAccess}} networkscheme: {{.NetworkScheme}} ports: @@ -155,18 +133,13 @@ spec: func fillPlatformTemplateCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) (*Manifest, error) { log.DebugLog(log.DebugLevelMexos, "fill template cloudletkeystr", "cloudletkeystr", cloudletKeyStr) - clk := edgeproto.CloudletKey{} - err := json.Unmarshal([]byte(cloudletKeyStr), &clk) - if err != nil { - return nil, fmt.Errorf("can't unmarshal json cloudletkey %s, %v", cloudletKeyStr, err) - } - log.DebugLog(log.DebugLevelMexos, "unmarshalled cloudletkeystr", "cloudletkey", clk) - if clk.Name == "" || clk.OperatorKey.Name == "" { - log.DebugLog(log.DebugLevelMexos, "will not fill template with invalid cloudletkeystr", "cloudletkeystr", cloudletKeyStr) - return nil, fmt.Errorf("invalid cloudletkeystr %s", cloudletKeyStr) - } - log.DebugLog(log.DebugLevelMexos, "using external network", "extNet", GetMEXExternalNetwork(rootLB.PlatConf)) + + log.DebugLog(log.DebugLevelMexos, "using external network", "extNet", GetCloudletExternalNetwork()) + meximage := os.Getenv("MEX_OS_IMAGE") + if meximage == "" { + return nil, fmt.Errorf("Env variable MEX_OS_IMAGE not set") + } data := templateFill{ ResourceKind: "platform", @@ -176,16 +149,12 @@ func fillPlatformTemplateCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) ( Key: clk.Name + "-" + NormalizeName(clk.OperatorKey.Name), Flavor: "x1.medium", Operator: NormalizeName(clk.OperatorKey.Name), - Location: "buckhorn", - Region: "eu-central-1", - Zone: "eu-central-1c", RootLB: rootLB.Name, - AgentImage: "registry.mobiledgex.net:5000/mobiledgex/mexosagent", Kind: "mex-platform", - ExternalNetwork: GetMEXExternalNetwork(rootLB.PlatConf), - NetworkScheme: "priv-subnet,mex-k8s-net-1,10.101.X.0/24", - DNSZone: "mobiledgex.net", - ExternalRouter: "mex-k8s-router-1", + ExternalNetwork: GetCloudletExternalNetwork(), + NetworkScheme: GetCloudletNetworkScheme(), + DNSZone: GetCloudletDNSZone(), + ExternalRouter: GetCloudletExternalRouter(), Options: "dhcp", } mf, err := templateUnmarshal(&data, yamlMEXPlatform) @@ -195,97 +164,6 @@ func fillPlatformTemplateCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) ( return mf, nil } -func fillAppTemplate(rootLB *MEXRootLB, appInst *edgeproto.AppInst, app *edgeproto.App, clusterInst *edgeproto.ClusterInst) (*Manifest, error) { - var err error - var mf *Manifest - log.DebugLog(log.DebugLevelMexos, "fill app template", "appinst", appInst, "clusterInst", clusterInst) - imageType, ok := edgeproto.ImageType_name[int32(app.ImageType)] - if !ok { - return nil, fmt.Errorf("cannot find imagetype in map") - } - if clusterInst.Flavor.Name == "" { - return nil, fmt.Errorf("will not fill app template, invalid cluster flavor name") - } - if verr := ValidateClusterKind(clusterInst.Key.CloudletKey.OperatorKey.Name); verr != nil { - return nil, verr - } - if appInst.Key.AppKey.Name == "" { - return nil, fmt.Errorf("will not fill app template, invalid appkey name") - } - ipAccess, ok := edgeproto.IpAccess_name[int32(appInst.IpAccess)] - if !ok { - return nil, fmt.Errorf("cannot find ipaccess in map") - } - if len(appInst.Key.AppKey.Name) < 3 { - log.DebugLog(log.DebugLevelMexos, "warning, very short appkey name", "name", appInst.Key.AppKey.Name) - } - config, err := cloudcommon.ParseAppConfig(app.Config) - if err != nil { - return nil, fmt.Errorf("error parsing appinst config %s, %v", app.Config, err) - } - log.DebugLog(log.DebugLevelMexos, "appinst config", "config", config) - appDeploymentType := app.Deployment - if err != nil { - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "app deploying", "imageType", imageType, "deploymentType", appDeploymentType) - if !cloudcommon.IsValidDeploymentForImage(app.ImageType, appDeploymentType) { - return nil, fmt.Errorf("deployment is not valid for image type") - } - vp := &rootLB.PlatConf.Values - data := templateFill{ - ResourceKind: "application", - Resource: NormalizeName(appInst.Key.AppKey.Name), - Kind: vp.Application.Kind, //"kubernetes", - Name: NormalizeName(appInst.Key.AppKey.Name), - Tags: NormalizeName(appInst.Key.AppKey.Name), - Key: clusterInst.Key.ClusterKey.Name, - Tenant: NormalizeName(appInst.Key.AppKey.Name), - DNSZone: vp.Network.DNSZone, // "mobiledgex.net", - Operator: NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name), - RootLB: rootLB.Name, - Image: app.ImagePath, - ImageType: imageType, - ImageFlavor: appInst.Flavor.Name, - ProxyPath: NormalizeName(appInst.Key.AppKey.Name), - AppURI: appInst.Uri, - IPAccess: ipAccess, - NetworkScheme: vp.Network.Scheme, //XXX "external-ip," + GetMEXExternalNetwork(rootLB.PlatConf), - Config: templateConfig{ - Deployment: app.Deployment, //vp.Application.Deployment - //Resources: config.Resources, - //Manifest: app.DeploymentManifest, //XXX vp.Application.Manifest,controller passes entire YAML - Template: vp.Application.Template, - Base: vp.Application.Base, - Overlay: vp.Application.Overlay, - }, - SpecPorts: vp.Application.Ports, - Command: strings.Split(app.Command, " "), - } - mf, err = templateUnmarshal(&data, yamlMEXApp) - if err != nil { - return nil, err - } - //XXX these are passed in unmarshallable format from controller. it can contain entire YAML content - mf.Config.ConfigDetail.Resources = config.Resources - mf.Config.ConfigDetail.Manifest = app.DeploymentManifest - switch appDeploymentType { - case cloudcommon.AppDeploymentTypeKubernetes: - case cloudcommon.AppDeploymentTypeDockerSwarm: - case cloudcommon.AppDeploymentTypeKVM: - case cloudcommon.AppDeploymentTypeHelm: - default: - return nil, fmt.Errorf("unknown image type %s", imageType) - } - log.DebugLog(log.DebugLevelMexos, "filled app manifest") - //XXX inconsistent addPorts after template fill - err = addPorts(mf, appInst) - if err != nil { - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "added port to app manifest") - return mf, nil -} func templateUnmarshal(data *templateFill, yamltext string) (*Manifest, error) { //log.DebugLog(log.DebugLevelMexos, "template unmarshal", "yamltext", string, "data", data) @@ -307,3 +185,5 @@ func templateUnmarshal(data *templateFill, yamltext string) (*Manifest, error) { } return mf, nil } + +*/ diff --git a/mexos/types.go b/mexos/types.go index 4fb41edb4..a4d0cc284 100644 --- a/mexos/types.go +++ b/mexos/types.go @@ -2,9 +2,9 @@ package mexos //MetadataDetail has metadata type MetadataDetail struct { - Name string `json:"name"` - Tags string `json:"tags"` - Tenant string `json:"tenant"` + Name string `json:"name"` + Tags string `json:"tags"` + // Tenant string `json:"tenant"` Region string `json:"region"` Zone string `json:"zone"` Location string `json:"location"` @@ -70,25 +70,26 @@ type PortDetail struct { //SpecDetail holds spec block type SpecDetail struct { - Flavor string `json:"flavor"` // appInst flavor? - FlavorDetail FlavorDetailInfo `json:"flavordetail"` - Flags string `json:"flags"` - RootLB string `json:"rootlb"` - Image string `json:"image"` - ImageFlavor string `json:"imageflavor"` - ImageType string `json:"imagetype"` - DockerRegistry string `json:"dockerregistry"` - ExternalNetwork string `json:"externalnetwork"` - ExternalRouter string `json:"externalrouter"` - Options string `json:"options"` - ProxyPath string `json:"proxypath"` - Ports []PortDetail `json:"ports"` - Command []string `json:"command"` - IPAccess string `json:"ipaccess"` - URI string `json:"uri"` - Key string `json:"key"` - NetworkScheme string `json:"networkscheme"` - Agent AgentDetail `json:"agent"` + Flavor string `json:"flavor"` // appInst flavor? + FlavorDetail FlavorDetailInfo `json:"flavordetail"` + Flags string `json:"flags"` + RootLB string `json:"rootlb"` + Image string `json:"image"` + ImageFlavor string `json:"imageflavor"` + ImageType string `json:"imagetype"` + DockerRegistry string `json:"dockerregistry"` + // ExternalNetwork string `json:"externalnetwork"` + // ExternalRouter string `json:"externalrouter"` + Options string `json:"options"` + ProxyPath string `json:"proxypath"` + Ports []PortDetail `json:"ports"` + Command []string `json:"command"` + + IPAccess string `json:"ipaccess"` + URI string `json:"uri"` + Key string `json:"key"` + NetworkScheme string `json:"networkscheme"` + // Agent AgentDetail `json:"agent"` } type AppInstConfigDetail struct { @@ -132,7 +133,6 @@ type ClusterValue struct { Zone string `json:"zone"` Region string `json:"region"` Location string `json:"location"` - OSImage string `json:"osimage"` Tenant string `json:"tenant"` Swarm string `json:"swarm"` } @@ -161,7 +161,6 @@ type NetworkValue struct { type OperatorValue struct { Name string `json:"name"` - Kind string `json:"kind"` } type RegistryValue struct { @@ -176,18 +175,17 @@ type ResourceValue struct { } type ValueDetail struct { - Name string `json:"name"` - Kind string `json:"kind"` - Base string `json:"base"` + Name string `json:"name"` + Kind string `json:"kind"` + // Base string `json:"base"` Application AppValue `json:"application"` Agent AgentValue `json:"agent"` Cluster ClusterValue `json:"cluster"` + Operator OperatorValue `json:"operator"` Network NetworkValue `json:"network"` Registry RegistryValue `json:"registry"` - Operator OperatorValue `json:"operator"` Resource ResourceValue `json:"resource"` Environment EnvironmentValue `json:"environment"` - VaultEnvMap map[string]string } type EnvData struct { diff --git a/mexos/valid.go b/mexos/valid.go index a8aa25a99..614462f8b 100644 --- a/mexos/valid.go +++ b/mexos/valid.go @@ -2,7 +2,6 @@ package mexos import ( "fmt" - "os" "reflect" "strings" @@ -54,13 +53,6 @@ func CheckVals(vals interface{}) error { return nil } -func IsValidMEXOSEnv() bool { - if os.Getenv("MEX_CF_KEY") == "" { // XXX - return false - } - return validMEXOSEnv -} - //ValidateNetSpec parses and validates the netSpec func ValidateNetSpec(netSpec string) error { // TODO if _,err:=ParseNetSpec(netSpec); err!=nil{ return err} diff --git a/mexos/vault.go b/mexos/vault.go index a9433ae1f..0e44eeb92 100644 --- a/mexos/vault.go +++ b/mexos/vault.go @@ -8,6 +8,8 @@ import ( "strings" "github.com/ghodss/yaml" + "github.com/mobiledgex/edge-cloud/edgeproto" + "github.com/mobiledgex/edge-cloud/log" ) func GetVaultData(url string) ([]byte, error) { @@ -69,10 +71,9 @@ func internEnv(envs []EnvData) error { return nil } -func InternVaultEnv(mf *Manifest) error { - //log.DebugLog(log.DebugLevelMexos, "interning vault env var") - mf.Values.VaultEnvMap = make(map[string]string) - for _, u := range []string{mf.Values.Environment.OpenRC, mf.Values.Environment.MexEnv} { +func InternVaultEnv(openrc string, mexenv string, cloudletInfra *edgeproto.CloudletInfraProperties) error { + log.DebugLog(log.DebugLevelMexos, "interning vault", "openrc", openrc, "mexenv", mexenv) + for _, u := range []string{openrc, mexenv} { if u == "" { continue } @@ -84,8 +85,10 @@ func InternVaultEnv(mf *Manifest) error { if err != nil { return err } - for _, e := range vr.Data.Detail.Env { - mf.Values.VaultEnvMap[e.Name] = e.Value + if u == openrc { + for _, e := range vr.Data.Detail.Env { + cloudletInfra.OpenstackProperties.OpenRcVars[e.Name] = e.Value + } } //log.DebugLog(log.DebugLevelMexos, "interning vault data", "data", vr) err = internEnv(vr.Data.Detail.Env) @@ -106,10 +109,10 @@ func CheckPlatformEnv(platformType string) error { // name string // getter func() string // }{ - // {"MEX_EXT_NETWORK", GetMEXExternalNetwork}, - // {"MEX_EXT_ROUTER", GetMEXExternalRouter}, - // {"MEX_NETWORK", GetMEXNetwork}, - // {"MEX_SECURITY_RULE", GetMEXSecurityRule}, + // {"MEX_EXT_NETWORK", GetCloudletExternalNetwork}, + // {"MEX_EXT_ROUTER", GetCloudletExternalRouter}, + // {"MEX_NETWORK", GetCloudletNetwork}, + // {"MEX_SECURITY_RULE", GetCloudletSecurityRule}, // } { // ev := os.Getenv(n.name) // if ev == "" { @@ -127,21 +130,10 @@ func CheckPlatformEnv(platformType string) error { return nil } -func GetVaultEnv(mf *Manifest, uri string) error { - dat, err := GetURIFile(mf, mf.Base+"/"+uri) - if err != nil { - return err - } - err = yaml.Unmarshal(dat, mf) - if err != nil { - return err - } - //log.DebugLog(log.DebugLevelMexos, "about to intern vault env", "mf", mf) - if err := InternVaultEnv(mf); err != nil { - return err - } - if err := CheckPlatformEnv(mf.Values.Operator.Kind); err != nil { +func GetVaultEnv(openrc string, mexenv string, cloudletInfra *edgeproto.CloudletInfraProperties) error { + + if err := InternVaultEnv(openrc, mexenv, cloudletInfra); err != nil { return err } - return err + return nil } From 10b69092d6f00c517ec9af493fe57120db2f4241 Mon Sep 17 00:00:00 2001 From: Jim Date: Thu, 7 Feb 2019 14:57:10 -0600 Subject: [PATCH 02/10] tweaks --- mexos/agent.go | 69 +------------ mexos/cloudflare.go | 14 --- mexos/cloudletinfra.go | 4 +- mexos/cluster.go | 11 +- mexos/env.go | 30 ------ mexos/init.go | 9 -- mexos/inst.go | 11 -- mexos/kubeconfig.go | 32 ------ mexos/kubectl.go | 20 ---- mexos/kubeproxy.go | 4 +- mexos/kubernetes.go | 2 +- mexos/kvm.go | 36 ++++--- mexos/manifest.go | 88 ---------------- mexos/mexctl/main.go | 160 ----------------------------- mexos/network.go | 2 +- mexos/oscli.go | 12 +-- mexos/platform.go | 24 ----- mexos/ports.go | 9 ++ mexos/revproxy.go | 4 +- mexos/ssh.go | 2 +- mexos/types.go | 224 ----------------------------------------- mexos/uri.go | 12 +-- mexos/valid.go | 163 ------------------------------ mexos/vault.go | 17 ++++ 24 files changed, 76 insertions(+), 883 deletions(-) delete mode 100644 mexos/cloudflare.go delete mode 100644 mexos/env.go delete mode 100644 mexos/manifest.go delete mode 100644 mexos/mexctl/main.go delete mode 100644 mexos/platform.go delete mode 100644 mexos/types.go delete mode 100644 mexos/valid.go diff --git a/mexos/agent.go b/mexos/agent.go index 025dc02c4..a9dae7a25 100644 --- a/mexos/agent.go +++ b/mexos/agent.go @@ -40,7 +40,7 @@ func RunMEXAgent(rootLBName string, cloudletKey *edgeproto.CloudletKey) error { if err != nil { return fmt.Errorf("cannot find rootlb %s", fqdn) } - //return RunMEXOSAgentContainer(mf, rootLB) + //return RunMEXOSAgentContainer(rootLB) return RunMEXOSAgentService(rootLB) } } @@ -131,7 +131,7 @@ func RunMEXOSAgentService(rootLB *MEXRootLB) error { return nil } -func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { +func RunMEXOSAgentContainer(rootLB *MEXRootLB) error { client, err := GetSSHClient(rootLB.Name, GetCloudletExternalNetwork(), sshUser) if err != nil { return err @@ -151,12 +151,9 @@ func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { return fmt.Errorf("can't store docker pass, %s, %v", out, err) } log.DebugLog(log.DebugLevelMexos, "seeded docker registry password") - dockerinstanceName := fmt.Sprintf("%s-%s", mf.Metadata.Name, rootLB.Name) - if mf.Spec.DockerRegistry == "" { - log.DebugLog(log.DebugLevelMexos, "warning, empty docker registry spec, using default.") - mf.Spec.DockerRegistry = GetCloudletDockerRegistry() - } - cmd = fmt.Sprintf("cat .docker-pass| docker login -u mobiledgex --password-stdin %s", mf.Spec.DockerRegistry) + dockerinstanceName := fmt.Sprintf("%s-%s", "mexos", rootLB.Name) + + cmd = fmt.Sprintf("cat .docker-pass| docker login -u mobiledgex --password-stdin %s", GetCloudletDockerRegistry()) out, err = client.Output(cmd) if err != nil { return fmt.Errorf("error docker login at %s, %s, %s, %v", rootLB.Name, cmd, out, err) @@ -177,19 +174,6 @@ func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { return nil } -/* -//UpdateMEXAgentManifest upgrades the mex agent -func UpdateMEXAgentManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "update mex agent") - err := RemoveMEXAgentManifest(mf) - if err != nil { - return err - } - // Force pulling a potentially newer docker image - return RunMEXAgentManifest(mf) -} -*/ - //RunMEXAgentCloudletKey calls MEXPlatformInit with templated manifest func RunMEXAgentCloudletKey(rootLBName string, cloudletKeyStr string) error { @@ -204,46 +188,3 @@ func RunMEXAgentCloudletKey(rootLBName string, cloudletKeyStr string) error { } return RunMEXAgent(rootLBName, &clk) } - -/* -//RemoveMEXAgentManifest deletes mex agent docker instance -func RemoveMEXAgentManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "deleting mex agent") - //XXX we are deleting server kvm!!! - err := DeleteServer(mf.Spec.RootLB) - force := strings.Contains(mf.Spec.Flags, "force") - if err != nil { - if !force { - return err - } - log.DebugLog(log.DebugLevelMexos, "forced to continue, deleting mex agent error", "error", err, "rootLB", mf.Spec.RootLB) - } - log.DebugLog(log.DebugLevelMexos, "removed rootlb", "name", mf.Spec.RootLB) - sip, err := GetServerIPAddr(GetCloudletExternalNetwork(), mf.Spec.RootLB) - if err := DeleteSecurityRule(sip); err != nil { - log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rule", "error", err, "server ip", sip) - } - if mf.Metadata.DNSZone == "" { - return fmt.Errorf("missing dns zone in manifest, metadata %v", mf.Metadata) - } - if cerr := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); cerr != nil { - return fmt.Errorf("cannot init cloudflare api, %v", cerr) - } - recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) - fqdn := mf.Spec.RootLB - if derr != nil { - return fmt.Errorf("can not get dns records for %s, %v", fqdn, derr) - } - for _, rec := range recs { - if rec.Type == "A" && rec.Name == fqdn { - err = cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID) - if err != nil { - return fmt.Errorf("cannot delete dns record id %s Zone %s, %v", rec.ID, mf.Metadata.DNSZone, err) - } - } - } - log.DebugLog(log.DebugLevelMexos, "removed DNS A record", "FQDN", fqdn) - //TODO remove mex-k8s internal nets and router - return nil -} -*/ diff --git a/mexos/cloudflare.go b/mexos/cloudflare.go deleted file mode 100644 index a7486fbb6..000000000 --- a/mexos/cloudflare.go +++ /dev/null @@ -1,14 +0,0 @@ -package mexos - -/* -//CheckCredentialsCF checks for Cloudflare -func CheckCredentialsCF(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "check for cloudflare credentials") - for _, envname := range []string{"MEX_CF_KEY", "MEX_CF_USER"} { - if v := mexEnv(mf, envname); v == "" { - return fmt.Errorf("no env var for %s", envname) - } - } - return nil -} -*/ diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index f344a584d..cb6e0b764 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -116,7 +116,7 @@ func InitializeCloudletInfra(fakecloudlet bool) error { } case cloudcommon.CloudletKindGCP: - CloudletInfra.GcpProperties.Project = os.Getenv("MEX_GCP_LOCATION") + CloudletInfra.GcpProperties.Project = os.Getenv("MEX_GCP_PROJECT") if CloudletInfra.GcpProperties.Project == "" { //default CloudletInfra.OpenstackProperties.OSImageName = "still-entity-201400" @@ -171,7 +171,7 @@ func GetCloudletExternalNetwork() string { // Utility functions that used to be within manifest. //GetCloudletNetwork returns default MEX network, internal and prepped -func GetCloudletNetwork() string { +func GetCloudletMexNetwork() string { //TODO validate existence and status return CloudletInfra.OpenstackProperties.OSMexNetwork } diff --git a/mexos/cluster.go b/mexos/cluster.go index 979f0f8cd..7ab1c6755 100644 --- a/mexos/cluster.go +++ b/mexos/cluster.go @@ -107,15 +107,8 @@ func mexCreateClusterKubernetes(clusterInst *edgeproto.ClusterInst, rootLBName s //TODO allow more net types //TODO validate CIDR, etc. - // this does nothing currently - err = ValidateTenant(tenantName) - - if err != nil { - return fmt.Errorf("can't validate tenant, %v", err) - } - err = ValidateTags(tags) - if err != nil { - return fmt.Errorf("invalid tag, %v", err) + if tags == "" { + return fmt.Errorf("invalid tags") } tags += "," + cf.PlatformFlavor //TODO add whole manifest yaml->json into stringified property of the kvm instance for later diff --git a/mexos/env.go b/mexos/env.go deleted file mode 100644 index 3d1cd86e2..000000000 --- a/mexos/env.go +++ /dev/null @@ -1,30 +0,0 @@ -package mexos - -/* -//MEXCheckEnvVars sets up environment vars and checks for credentials required for running -func MEXCheckEnvVars(mf *Manifest) error { - // secrets to be passed via Env var still : MEX_CF_KEY, MEX_CF_USER, MEX_DOCKER_REG_PASS - // TODO: use `secrets` or `vault` - for _, evar := range []string{ - "MEX_CF_KEY", - "MEX_CF_USER", - "MEX_DOCKER_REG_PASS", - } { - if v := mexEnv(mf, evar); v == "" { - return fmt.Errorf("missing env var %s", evar) - } - } - return nil -} - - -func mexEnv(mf *Manifest, name string) string { - v, ok := mf.Values.VaultEnvMap[name] - if !ok { - log.DebugLog(log.DebugLevelMexos, "error, env var not exist in vault", "name", name) - return "" - } - return v -} - -*/ diff --git a/mexos/init.go b/mexos/init.go index 88c9c9a38..b3eb87258 100644 --- a/mexos/init.go +++ b/mexos/init.go @@ -1,11 +1,2 @@ package mexos -/* -//MEXInit initializes MEX API -func MEXInit(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "mex init") - if err := MEXCheckEnvVars(mf); err != nil { - return err - } - return nil -}*/ diff --git a/mexos/inst.go b/mexos/inst.go index 62500a4a1..2743df7bc 100644 --- a/mexos/inst.go +++ b/mexos/inst.go @@ -202,14 +202,3 @@ func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return fmt.Errorf("unknown deployment type %s", appDeploymentType) } } - -/* -func fixValuesInst(mf *Manifest, rootLB *MEXRootLB) error { - if mf.Values.Kind == "" { - mf.Values = rootLB.PlatConf.Values - } - if mf.Values.Kind == "" { - log.DebugLog(log.DebugLevelMexos, "warning, missing mf values") - } - return nil -}*/ diff --git a/mexos/kubeconfig.go b/mexos/kubeconfig.go index 562964bcf..1710a1f33 100644 --- a/mexos/kubeconfig.go +++ b/mexos/kubeconfig.go @@ -135,35 +135,3 @@ func CopyKubeConfig(clusterInst *edgeproto.ClusterInst, rootLBName, name string) //return ProcessKubeconfig(mf, rootLB, name, port, []byte(out)) return nil } - -/* -//ProcessKubeconfig validates kubeconfig and saves it and creates a copy for proxy access -func ProcessKubeconfig(mf *Manifest, rootLB *MEXRootLB, name string, port int, dat []byte) error { - log.DebugLog(log.DebugLevelMexos, "process kubeconfig file", "name", name) - if rootLB == nil { - return fmt.Errorf("cannot process kubeconfig, rootLB is null") - } - kc := &clusterKubeconfig{} - err := yaml.Unmarshal(dat, kc) - if err != nil { - return fmt.Errorf("can't unmarshal kubeconfig %s, %v", name, err) - } - if len(kc.Clusters) < 1 { - return fmt.Errorf("insufficient clusters info in kubeconfig %s", name) - } - kconfname := GetLocalKconfName(mf) - log.DebugLog(log.DebugLevelMexos, "writing local kubeconfig file", "name", kconfname) - //TODO per cluster password has to come from vault - kc.Clusters[0].Cluster.Server = fmt.Sprintf("https://testuser314159:testpassword271828@%s:%d", rootLB.Name, port) - dat, err = yaml.Marshal(kc) - if err != nil { - return fmt.Errorf("can't marshal kubeconfig proxy edit %s, %v", name, err) - } - err = ioutil.WriteFile(kconfname, dat, 0666) - if err != nil { - return fmt.Errorf("can't write kubeconfig file %s, %v", kconfname, err) - } - log.DebugLog(log.DebugLevelMexos, "kubeconfig file saved", "file", kconfname) - return nil -} -*/ diff --git a/mexos/kubectl.go b/mexos/kubectl.go index 7be837574..10a429fbb 100644 --- a/mexos/kubectl.go +++ b/mexos/kubectl.go @@ -15,26 +15,6 @@ import ( "github.com/mobiledgex/edge-cloud/log" ) -/* -func RunKubectl(clusterInst *edgeproto.ClusterInst, params string, rootLBName string) (*string, error) { - log.DebugLog(log.DebugLevelMexos, "run kubectl", "params", params) - - //name, err := FindClusterWithKey(mf, mf.Spec.Key) - //if err != nil { - // return nil, fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) - //} - client, err := GetSSHClient(rootLBName, GetCloudletExternalNetwork(), sshUser) - if err != nil { - return nil, fmt.Errorf("can't get ssh client, %v", err) - } - cmd := fmt.Sprintf("kubectl --kubeconfig %s.kubeconfig %s", rootLBName, params) - out, err := client.Output(cmd) - if err != nil { - return nil, fmt.Errorf("kubectl failed, %v, %s", err, out) - } - return &out, nil -} -*/ func CreateDockerRegistrySecret(clusterInst *edgeproto.ClusterInst, rootLBName string) error { log.DebugLog(log.DebugLevelMexos, "creating docker registry secret in kubernetes") diff --git a/mexos/kubeproxy.go b/mexos/kubeproxy.go index 982faf53a..d0bd5eaa8 100644 --- a/mexos/kubeproxy.go +++ b/mexos/kubeproxy.go @@ -12,7 +12,7 @@ var kcproxySuffix = "-kcproxy" //StartKubectlProxy starts kubectl proxy on the rootLB to handle kubectl commands remotely. // To be called after copying over the kubeconfig file from cluster to rootLB. -func StartKubectlProxy(mf *Manifest, rootLB *MEXRootLB, name, kubeconfig string) (int, error) { +func StartKubectlProxy(rootLB *MEXRootLB, name, kubeconfig string) (int, error) { log.DebugLog(log.DebugLevelMexos, "start kubectl proxy", "name", name, "kubeconfig", kubeconfig) if rootLB == nil { return 0, fmt.Errorf("cannot kubectl proxy, rootLB is null") @@ -25,7 +25,7 @@ func StartKubectlProxy(mf *Manifest, rootLB *MEXRootLB, name, kubeconfig string) return 0, err } //TODO check /home/ubuntu/.docker-pass file - cmd := fmt.Sprintf("echo %s | docker login -u mobiledgex --password-stdin %s", GetCloudletDockerPass(), mf.Spec.DockerRegistry) + cmd := fmt.Sprintf("echo %s | docker login -u mobiledgex --password-stdin %s", GetCloudletDockerPass(), GetCloudletDockerRegistry()) out, err := client.Output(cmd) if err != nil { return 0, fmt.Errorf("can't docker login, %s, %v", out, err) diff --git a/mexos/kubernetes.go b/mexos/kubernetes.go index 6c6146308..aaeb320e1 100644 --- a/mexos/kubernetes.go +++ b/mexos/kubernetes.go @@ -154,7 +154,7 @@ type KAPINodes struct { //TODO metadata } -/* +/* TODO: fix for swarm func GetKubernetesNodes(mf *Manifest, rootLB *MEXRootLB) ([]KubernetesNode, error) { log.DebugLog(log.DebugLevelMexos, "getting kubernetes nodes") clusterName := clusterInst.Key.ClusterKey.Name diff --git a/mexos/kvm.go b/mexos/kvm.go index 17ffe24e0..3fbe69d50 100644 --- a/mexos/kvm.go +++ b/mexos/kvm.go @@ -3,21 +3,24 @@ package mexos import ( "fmt" "net" - "os" "strings" - "time" - sh "github.com/codeskyblue/go-sh" "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ) +type NetSpecInfo struct { + Kind, Name, CIDR, Options string + Extra []string +} + func GetK8sNodeNameSuffix(clusterInst *edgeproto.ClusterInst) string { cloudletName := clusterInst.Key.CloudletKey.Name clusterName := clusterInst.Key.ClusterKey.Name return NormalizeName(cloudletName + "-" + clusterName) } +/* TODO: Fix for swarm //CreateQCOW2AppManifest creates qcow2 app func CreateQCOW2AppManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "create qcow2 vm based app") @@ -121,6 +124,9 @@ func CreateQCOW2AppManifest(mf *Manifest) error { log.DebugLog(log.DebugLevelMexos, "created openstack kvm server", "opts", opts) return nil } +*/ + +/* func DeleteQCOW2AppManifest(mf *Manifest) error { if mf.Metadata.Name == "" { @@ -131,6 +137,7 @@ func DeleteQCOW2AppManifest(mf *Manifest) error { } return nil } +*/ //CreateFlavorMEXVM creates basic KVM for mobiledgex applications // with proper initial bootstrap scripts installed on the base image that understands @@ -187,17 +194,18 @@ func CreateFlavorMEXVM(name, image, flavor, netID, userdata, role, edgeproxy, sk props = append(props, "tenant="+tenant) /* TODO: holepunch has not been used anywhere but need to investigate if we will want this - if GetCloudletExternalNetwork().HolePunch != "" { - props = append(props, "holepunch="+GetCloudletExternalNetwork().HolePunch) - } + if GetCloudletHolePunch() != "" { + props = append(props, "holepunch="+GetCloudletHolePunch() + } */ /* TODO: update has code for it in the init scripts, but has not been used because the cloudlet-specific files - are not present on the registry and nobody knew this existed. This is for Venky to study. - if mf.Values.Registry.Update != "" { - props = append(props, "update="+mf.Values.Registry.Update) - } + are not present on the registry and nobody knew this existed. This is for Venky to study. + if mf.Values.Registry.Update != "" { + props = append(props, "update="+mf.Values.Registry.Update) + } */ + opts.Properties = props //log.DebugLog(log.DebugLevelMexos, "create flavor MEX KVM", "flavor", flavor, "server opts", opts) log.DebugLog(log.DebugLevelMexos, "create flavor MEX KVM", "flavor", flavor) @@ -257,8 +265,8 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, clusterInst if ni.CIDR == "" { return fmt.Errorf("missing CIDR spec in %v", ni) } - if ni.Name != GetCloudletNetwork() { //XXX for now - return fmt.Errorf("netspec net name %s not equal to default MEX net %s", ni.Name, GetCloudletNetwork()) + if ni.Name != GetCloudletMexNetwork() { //XXX for now + return fmt.Errorf("netspec net name %s not equal to default MEX net %s", ni.Name, GetCloudletMexNetwork()) } //XXX openstack bug - subnet does not take tags but description field can be used to tag stuff // Use tag as part of name @@ -402,7 +410,7 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, clusterInst if role == k8smasterRole { if snd == nil { log.DebugLog(log.DebugLevelMexos, "k8s master, no existing subnet, creating subnet", "name", sn) - err = CreateSubnet(ni.CIDR, GetCloudletNetwork(), edgeProxy, sn, false) + err = CreateSubnet(ni.CIDR, GetCloudletMexNetwork(), edgeProxy, sn, false) if err != nil { return err } @@ -422,7 +430,7 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, clusterInst //master node num is 1 //so, master node will always have .2 //XXX master always at X.X.X.2 - netID = GetCloudletNetwork() + ",v4-fixed-ip=" + ipaddr.String() + netID = GetCloudletMexNetwork() + ",v4-fixed-ip=" + ipaddr.String() masteripaddr := net.IPv4(v4[0], v4[1], v4[2], byte(2)) masterIP = masteripaddr.String() log.DebugLog(log.DebugLevelMexos, "k8s master ip addr", "netID", netID, "ipaddr", ipaddr, "masterip", masterIP) diff --git a/mexos/manifest.go b/mexos/manifest.go deleted file mode 100644 index 4f59b5cee..000000000 --- a/mexos/manifest.go +++ /dev/null @@ -1,88 +0,0 @@ -package mexos - -/* -//MEXPlatformCleanCloudletKey calls MEXPlatformClean with templated manifest -func MEXPlatformCleanCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) error { - mf, err := fillPlatformTemplateCloudletKey(rootLB, cloudletKeyStr) - if err != nil { - return err - } - return MEXPlatformCleanManifest(mf) -} - -//MEXPlatformInitManifest initializes platform -func MEXPlatformInitManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "init platform") - switch mf.Metadata.Operator { - case cloudcommon.OperatorGCP: - return nil //nothing to do - case cloudcommon.OperatorAzure: - return nil //nothing to do - default: - //TODO validate all mf content against platform data - if err := RunMEXAgentManifest(mf); err != nil { - return err - } - } - return nil -} - -//MEXPlatformCleanManifest cleans up the platform -func MEXPlatformCleanManifest(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "clean platform") - switch mf.Metadata.Operator { - case "gcp": - return nil //nothing to do - case "azure": - return nil - default: - if err := RemoveMEXAgentManifest(mf); err != nil { - return err - } - } - return nil -} - -//MEXAppCreateAppManifest creates app instances on the cluster platform -func GetDefaultRegistryBase(mf *Manifest, base string) string { - mf.Base = base - if mf.Base == "" { - mf.Base = fmt.Sprintf("scp://%s/files-repo/mobiledgex", GetCloudletRegistryFileServer()) - } - log.DebugLog(log.DebugLevelMexos, "default registry base", "base", mf.Base) - return mf.Base -} - - -func GetKubeManifest(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) (string, error) { - var kubeManifest string - - base := rootLB.PlatConf.Base - if base == "" { - log.DebugLog(log.DebugLevelMexos, "base is empty, using default") - base = GetDefaultRegistryBase(mf, base) - } - mani := mf.Config.ConfigDetail.Manifest - deployment := mf.Config.ConfigDetail.Deployment - //XXX controlling pass full yaml text in parameter of another yaml - log.DebugLog(log.DebugLevelMexos, "getting kubernetes manifest", "base", base, "manifest", mani) - if deployment != cloudcommon.AppDeploymentTypeHelm && !strings.HasPrefix(mani, "apiVersion: v1") { - fn := fmt.Sprintf("%s/%s", base, mani) - log.DebugLog(log.DebugLevelMexos, "getting manifest file", "uri", fn) - res, err := GetURIFile(mf, fn) - if err != nil { - return "", err - } - kubeManifest = string(res) - } else { - //XXX controller is passing full yaml as a string. - log.DebugLog(log.DebugLevelMexos, "getting deployment from cloudcommon", "base", mf.Base, "manifest", mani) - //XXX again it seems to download yaml but already yaml full string is passed from controller - kubeManifest, err = cloudcommon.GetDeploymentManifest(mf.Config.ConfigDetail.Manifest) - if err != nil { - return "", err - } - } - return kubeManifest, nil -} -*/ diff --git a/mexos/mexctl/main.go b/mexos/mexctl/main.go deleted file mode 100644 index e539f1e61..000000000 --- a/mexos/mexctl/main.go +++ /dev/null @@ -1,160 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "reflect" - - sh "github.com/codeskyblue/go-sh" - "github.com/mobiledgex/edge-cloud-infra/mexos" - "github.com/mobiledgex/edge-cloud/log" -) - -var clusterOps = map[string]func(*mexos.Manifest) error{ - "create": mexos.MEXClusterCreateManifest, - "remove": mexos.MEXClusterRemoveManifest, -} - -var platformOps = map[string]func(*mexos.Manifest) error{ - "create": mexos.MEXPlatformInitManifest, - "remove": mexos.MEXPlatformCleanManifest, -} - -var applicationOps = map[string]func(*mexos.Manifest) error{ - "create": mexos.MEXAppCreateAppManifest, - "remove": mexos.MEXAppDeleteAppManifest, -} - -var openstackOps = map[string]func(*mexos.Manifest) error{} -var kubectlOps = map[string]func(*mexos.Manifest) error{} - -var categories = map[string]map[string]func(*mexos.Manifest) error{ - "cluster": clusterOps, - "platform": platformOps, - "application": applicationOps, - "openstack": openstackOps, - "kubectl": kubectlOps, -} - -var mainflag = flag.NewFlagSet(os.Args[0], flag.ExitOnError) - -func printUsage() { - originalUsage() - fmt.Println("mex -manifest myvals.yaml {platform|cluster|application} {create|remove}") - fmt.Println("mex -manifest myvals.yaml openstack ...") -} - -var originalUsage func() - -func main() { - var err error - help := mainflag.Bool("help", false, "help") - debugLevels := mainflag.String("d", "", fmt.Sprintf("comma separated list of %v", log.DebugLevelStrings)) - base := mainflag.String("base", ".", "base containing templates, directory path or URI") - manifest := mainflag.String("manifest", "", "manifest") - originalUsage = mainflag.Usage - mainflag.Usage = printUsage - if err = mainflag.Parse(os.Args[1:]); err != nil { - log.InfoLog("parse error", "error", err) - os.Exit(1) - } - if *help { - printUsage() - os.Exit(0) - } - log.SetDebugLevelStrs(*debugLevels) - //XXX TODO make log to a remote server / aggregator - args := mainflag.Args() - if len(args) < 2 { - printUsage() - fmt.Println("insufficient args") - os.Exit(1) - } - _, ok := categories[args[0]] - if !ok { - printUsage() - fmt.Println("valid categories are", "categories", reflect.ValueOf(categories).MapKeys()) - os.Exit(1) - } - if *manifest == "" { - printUsage() - fmt.Println("missing manifest") - os.Exit(1) - } - if len(args) < 2 { - printUsage() - fmt.Println("insufficient args") - os.Exit(1) - } - log.DebugLog(log.DebugLevelMexos, "getting mf from manifest", "file", *manifest, "base", *base) - mf := &mexos.Manifest{Base: *base} - if err := mexos.GetVaultEnv(mf, *manifest); err != nil { - log.InfoLog("cannot get mf", "uri", *manifest, "error", err) - os.Exit(1) - } - kind := args[0] - if err := mexos.FillManifestValues(mf, kind, *base); err != nil { - log.InfoLog("cannot fill manifest", "error", err, "kind", kind, "base", *base) - os.Exit(1) - } - if err := mexos.CheckManifest(mf); err != nil { - log.InfoLog("incorrect manifest", "error", err) - os.Exit(1) - } - if _, err := mexos.NewRootLBManifest(mf); err != nil { - log.InfoLog("can't get new rootLB", "error", err) - os.Exit(1) - } - if err := mexos.MEXInit(mf); err != nil { - log.InfoLog("cannot init mex", "error", err) - os.Exit(1) - } - ops := args[1:] - log.DebugLog(log.DebugLevelMexos, "call", "kind", kind, "ops", ops) - err = callOps(mf, kind, ops...) - if err != nil { - log.InfoLog("ops failure", "kind", kind, "ops", ops, "error", err) - os.Exit(1) - } - os.Exit(0) -} - -func callOps(mf *mexos.Manifest, kind string, ops ...string) error { - if kind == "openstack" { - vs := make([]interface{}, len(ops)) - for i, v := range ops { - vs[i] = v - } - out, err := sh.Command(kind, vs...).Output() - if err != nil { - return fmt.Errorf("error, %s %v, %v, %s", kind, ops, err, out) - } - fmt.Println(string(out)) - return nil - } - if kind == "kubectl" { - vs := "" - for _, v := range ops { - vs = vs + v + " " - } - out, err := mexos.RunKubectl(mf, vs) - if err != nil { - return fmt.Errorf("error, %s %v, %v", kind, ops, err) - } - fmt.Println(string(*out)) - return nil - } - if _, ok := categories[kind]; !ok { - return fmt.Errorf("invalid category %s", kind) - } - op := ops[0] - if _, ok := categories[kind][op]; !ok { - return fmt.Errorf("invalid category op %s", op) - } - err := categories[kind][op](mf) - if err != nil { - return err - } - return nil -} diff --git a/mexos/network.go b/mexos/network.go index 7f8998300..eea09d94d 100644 --- a/mexos/network.go +++ b/mexos/network.go @@ -45,7 +45,7 @@ func GetExternalGateway(extNetName string) (string, error) { //GetNextSubnetRange will find the CIDR for the next range of subnet that can be created. For example, // if the subnet detail we get has 10.101.101.0/24 then the next one can be 10.101.102.0/24 -func GetNextSubnetRange(mf *Manifest, subnetName string) (string, error) { +func GetNextSubnetRange(subnetName string) (string, error) { sd, err := GetSubnetDetail(subnetName) if err != nil { return "", err diff --git a/mexos/oscli.go b/mexos/oscli.go index 91e3bdd0e..9de3fbc4f 100644 --- a/mexos/oscli.go +++ b/mexos/oscli.go @@ -28,7 +28,7 @@ func ListServers() ([]OSServer, error) { } //ListImages lists avilable images in glance -func ListImages(mf *Manifest) ([]OSImage, error) { +func ListImages() ([]OSImage, error) { out, err := sh.Command("openstack", "image", "list", "-f", "json").Output() if err != nil { err = fmt.Errorf("cannot get image list, %v", err) @@ -168,7 +168,7 @@ func CreateNetwork(name string) error { //DeleteNetwork destroys a named network // Sometimes it will fail. Openstack will refuse if there are resources attached. -func DeleteNetwork(mf *Manifest, name string) error { +func DeleteNetwork( name string) error { log.DebugLog(log.DebugLevelMexos, "deleting network", "network", name) out, err := sh.Command("openstack", "network", "delete", name).CombinedOutput() if err != nil { @@ -242,7 +242,7 @@ func CreateRouter(routerName string) error { } //DeleteRouter removes the named router. The router needs to not be in use at the time of deletion. -func DeleteRouter(mf *Manifest, routerName string) error { +func DeleteRouter( routerName string) error { log.DebugLog(log.DebugLevelMexos, "deleting router", "name", routerName) out, err := sh.Command("openstack", "router", "delete", routerName).Output() if err != nil { @@ -358,7 +358,7 @@ func CreateServerImage(serverName, imageName string) error { } //CreateImage puts images into glance -func CreateImage(mf *Manifest, imageName, qcowFile string) error { +func CreateImage( imageName, qcowFile string) error { log.DebugLog(log.DebugLevelMexos, "creating image in glance", "image", imageName, "qcow", qcowFile) out, err := sh.Command("openstack", "image", "create", imageName, @@ -376,7 +376,7 @@ func CreateImage(mf *Manifest, imageName, qcowFile string) error { // It will then save that into a local file. The image transfer happens from glance into your own laptop // or whatever. // This can take a while, transferring all the data. -func SaveImage(mf *Manifest, saveName, imageName string) error { +func SaveImage( saveName, imageName string) error { log.DebugLog(log.DebugLevelMexos, "saving image", "save name", saveName, "image name", imageName) out, err := sh.Command("openstack", "image", "save", "--file", saveName, imageName).Output() if err != nil { @@ -389,7 +389,7 @@ func SaveImage(mf *Manifest, saveName, imageName string) error { //DeleteImage deletes the named image from glance. Sometimes backing store is still busy and // will refuse to honor the request. Like most things in Openstack, wait for a while and try // again. -func DeleteImage(mf *Manifest, imageName string) error { +func DeleteImage( imageName string) error { log.DebugLog(log.DebugLevelMexos, "deleting image", "name", imageName) out, err := sh.Command("openstack", "image", "delete", imageName).Output() if err != nil { diff --git a/mexos/platform.go b/mexos/platform.go deleted file mode 100644 index 2b8dafa39..000000000 --- a/mexos/platform.go +++ /dev/null @@ -1,24 +0,0 @@ -package mexos - -/* -func setPlatConfManifest(mf *Manifest) error { - rootLB, err := getRootLB(mf.Spec.RootLB) - if err != nil { - return err - } - if rootLB == nil { - return fmt.Errorf("cannot set platform config manifest, rootLB is null") - } - setPlatConf(rootLB, mf) - return nil -} - - -func setPlatConf(rootLB *MEXRootLB, mf *Manifest) { - log.DebugLog(log.DebugLevelMexos, "rootlb platconf set") - if rootLB == nil { - log.DebugLog(log.DebugLevelMexos, "cannot set platconf, rootLB is null") - } - rootLB.PlatConf = mf -} -*/ diff --git a/mexos/ports.go b/mexos/ports.go index 309ed1825..59184c495 100644 --- a/mexos/ports.go +++ b/mexos/ports.go @@ -8,6 +8,15 @@ import ( "github.com/mobiledgex/edge-cloud/edgeproto" ) +type PortDetail struct { + Name string `json:"name"` + MexProto string `json:"mexproto"` + Proto string `json:"proto"` + InternalPort int `json:"internalport"` + PublicPort int `json:"publicport"` + PublicPath string `json:"publicpath"` +} + func GetPortDetail(appInst *edgeproto.AppInst) ([]PortDetail, error) { ports := make([]PortDetail, 0) for ii := range appInst.MappedPorts { diff --git a/mexos/revproxy.go b/mexos/revproxy.go index e0ff51a81..9e893c702 100644 --- a/mexos/revproxy.go +++ b/mexos/revproxy.go @@ -9,7 +9,7 @@ import ( ) //AddPathReverseProxy adds a new route to origin on the reverse proxy -func AddPathReverseProxy(mf *Manifest, rootLBName, path, origin string) []error { +func AddPathReverseProxy(rootLBName, path, origin string) []error { log.DebugLog(log.DebugLevelMexos, "add path to reverse proxy", "rootlbname", rootLBName, "path", path, "origin", origin) if path == "" { return []error{fmt.Errorf("empty path")} @@ -18,7 +18,7 @@ func AddPathReverseProxy(mf *Manifest, rootLBName, path, origin string) []error return []error{fmt.Errorf("empty origin")} } request := gorequest.New() - maURI := fmt.Sprintf("http://%s:%s/v1/proxy", rootLBName, mf.Values.Agent.Port) + maURI := fmt.Sprintf("http://%s:%s/v1/proxy", rootLBName, GetCloudletMexosAgentPort()) // The L7 reverse proxy terminates TLS at the RootLB and uses path routing to get to the service at a IP:port pl := fmt.Sprintf(`{ "message": "add", "proxies": [ { "path": "/%s/*catchall", "origin": "%s" } ] }`, path, origin) resp, body, errs := request.Post(maURI).Set("Content-Type", "application/json").Send(pl).End() diff --git a/mexos/ssh.go b/mexos/ssh.go index 7d525be04..19db8a495 100644 --- a/mexos/ssh.go +++ b/mexos/ssh.go @@ -12,7 +12,7 @@ var sshOpts = []string{"StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null var sshUser = "ubuntu" //CopySSHCredential copies over the ssh credential for mex to LB -func CopySSHCredential(mf *Manifest, serverName, networkName, userName string) error { +func CopySSHCredential(serverName, networkName, userName string) error { //TODO multiple keys to be copied and added to authorized_keys if needed log.DebugLog(log.DebugLevelMexos, "copying ssh credentials", "server", serverName, "network", networkName, "user", userName) addr, err := GetServerIPAddr(networkName, serverName) diff --git a/mexos/types.go b/mexos/types.go deleted file mode 100644 index a4d0cc284..000000000 --- a/mexos/types.go +++ /dev/null @@ -1,224 +0,0 @@ -package mexos - -//MetadataDetail has metadata -type MetadataDetail struct { - Name string `json:"name"` - Tags string `json:"tags"` - // Tenant string `json:"tenant"` - Region string `json:"region"` - Zone string `json:"zone"` - Location string `json:"location"` - Project string `json:"project"` - ResourceGroup string `json:"resourcegroup"` - OpenRC string `json:"openrc"` - DNSZone string `json:"dnszone"` - Kind string `json:"kind"` - Operator string `json:"operator"` - Swarm string `json:"swarm"` -} - -//NetworkDetail has network data -type NetworkDetail struct { - Name string `json:"name"` - Kind string `json:"kind"` - CIDR string `json:"cidr"` - Options string `json:"options"` - Extra string `json:"extra"` -} - -//AgentDetail has data on agent -type AgentDetail struct { - Image string `json:"image"` - Status string `json:"status"` -} - -//FlavorDetail has data on flavor -type FlavorDetail struct { - Name string `json:"name"` - Favorite string `json:"favorite"` - Memory string `json:"memory"` - Topology string `json:"topology"` - NodeFlavor string `json:"nodeflavor"` - MasterFlavor string `json:"masterflavor"` - NetworkScheme string `json:"networkscheme"` - Storage string `json:"storage"` - StorageScheme string `json:"storagescheme"` - CPUs int `json:"cpus"` - Masters int `json:"masters"` - Nodes int `json:"nodes"` -} - -type FlavorDetailInfo struct { - Name string `json:"name"` - Nodes int `json:"nodes"` - Masters int `json:"masters"` - NetworkScheme string `json:"networkscheme"` - MasterFlavor string `json:"masterflavor"` - NodeFlavor string `json:"nodeflavor"` - StorageScheme string `json:"storagescheme"` - Topology string `json:"topology"` -} - -type PortDetail struct { - Name string `json:"name"` - MexProto string `json:"mexproto"` - Proto string `json:"proto"` - InternalPort int `json:"internalport"` - PublicPort int `json:"publicport"` - PublicPath string `json:"publicpath"` -} - -//SpecDetail holds spec block -type SpecDetail struct { - Flavor string `json:"flavor"` // appInst flavor? - FlavorDetail FlavorDetailInfo `json:"flavordetail"` - Flags string `json:"flags"` - RootLB string `json:"rootlb"` - Image string `json:"image"` - ImageFlavor string `json:"imageflavor"` - ImageType string `json:"imagetype"` - DockerRegistry string `json:"dockerregistry"` - // ExternalNetwork string `json:"externalnetwork"` - // ExternalRouter string `json:"externalrouter"` - Options string `json:"options"` - ProxyPath string `json:"proxypath"` - Ports []PortDetail `json:"ports"` - Command []string `json:"command"` - - IPAccess string `json:"ipaccess"` - URI string `json:"uri"` - Key string `json:"key"` - NetworkScheme string `json:"networkscheme"` - // Agent AgentDetail `json:"agent"` -} - -type AppInstConfigDetail struct { - Deployment string `json:"deployment"` - Resources string `json:"resources"` - Template string `json:"template"` - Manifest string `json:"manifest"` - Base string `json:"base"` -} - -type AppInstConfig struct { - Kind string `json:"kind"` - Source string `json:"source"` - ConfigDetail AppInstConfigDetail `json:"detail"` -} - -type AgentValue struct { - Image string `json:"image"` - Port string `json:"port"` - Status string `json:"status"` -} - -type AppValue struct { - Deployment string `json:"deployment"` - Name string `json:"name"` - Kind string `json:"kind"` - Manifest string `json:"manifest"` - Image string `json:"image"` - ImageType string `json:"imagetype"` - ProxyPath string `json:"proxypath"` - Template string `json:"template"` - Base string `json:"base"` - Overlay string `json:"overlay"` - Ports []PortDetail `json:"ports"` -} - -type ClusterValue struct { - Flavor string `json:"flavor"` - Kind string `json:"kind"` - Name string `json:"name"` - Zone string `json:"zone"` - Region string `json:"region"` - Location string `json:"location"` - Tenant string `json:"tenant"` - Swarm string `json:"swarm"` -} - -type StorageValue struct { - Name string `json:"name"` - Scheme string `json:"scheme"` -} - -type EnvironmentValue struct { - OpenRC string `json:"openrc"` - MexEnv string `json:"mexenv"` -} - -type NetworkValue struct { - Router string `json:"router"` - Options string `json:"options"` - Name string `json:"name"` - External string `json:"external"` - IPAccess string `json:"ipaccess"` - SecurityRule string `json:"securityrule"` - Scheme string `json:"scheme"` - DNSZone string `json:"dnszone"` - HolePunch string `json:"holepunch"` -} - -type OperatorValue struct { - Name string `json:"name"` -} - -type RegistryValue struct { - Name string `json:"name"` - Docker string `json:"docker"` - Update string `json:"update"` -} - -type ResourceValue struct { - Group string `json:"group"` - Project string `json:"project"` -} - -type ValueDetail struct { - Name string `json:"name"` - Kind string `json:"kind"` - // Base string `json:"base"` - Application AppValue `json:"application"` - Agent AgentValue `json:"agent"` - Cluster ClusterValue `json:"cluster"` - Operator OperatorValue `json:"operator"` - Network NetworkValue `json:"network"` - Registry RegistryValue `json:"registry"` - Resource ResourceValue `json:"resource"` - Environment EnvironmentValue `json:"environment"` -} - -type EnvData struct { - Name string `json:"name"` - Value string `json:"value"` -} - -type VaultDataDetail struct { - Env []EnvData `json:"env"` -} - -type VaultData struct { - Detail VaultDataDetail `json:"data"` -} - -type VaultResponse struct { - Data VaultData `json:"data"` -} - -//Manifest is general container for the manifest yaml used by `mex` -type Manifest struct { - Name string `json:"name"` - APIVersion string `json:"apiVersion"` - Base string `json:"base"` - Kind string `json:"kind"` - Resource string `json:"resource"` - Metadata MetadataDetail `json:"metadata"` - Spec SpecDetail `json:"spec"` - Config AppInstConfig `json:"config"` - Values ValueDetail `json:"values"` -} - -type NetSpecInfo struct { - Kind, Name, CIDR, Options string - Extra []string -} diff --git a/mexos/uri.go b/mexos/uri.go index 48f43ad8e..fbdbb7750 100644 --- a/mexos/uri.go +++ b/mexos/uri.go @@ -23,13 +23,13 @@ func validateDomain(uri string) error { return fmt.Errorf("URI %s is not a valid domain name", uri) } -func GetURIFile(mf *Manifest, uri string) ([]byte, error) { +func GetURIFile(uri string) ([]byte, error) { log.DebugLog(log.DebugLevelMexos, "attempt to get uri file", "uri", uri) // if _, err := url.ParseRequestURI(uri); err != nil { // return nil, err // } if strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://") { - res, err := GetHTTPFile(mf, uri) + res, err := GetHTTPFile(uri) if err != nil { log.DebugLog(log.DebugLevelMexos, "error getting http uri file", "uri", uri, "error", err) return nil, err @@ -37,7 +37,7 @@ func GetURIFile(mf *Manifest, uri string) ([]byte, error) { return res, nil } if strings.HasPrefix(uri, "scp://") { - res, err := GetSCPFile(mf, uri) + res, err := GetSCPFile(uri) if err != nil { log.DebugLog(log.DebugLevelMexos, "error getting scp uri file", "uri", uri, "error", err) return nil, err @@ -59,7 +59,7 @@ func GetURIFile(mf *Manifest, uri string) ([]byte, error) { return res, nil } -func GetHTTPFile(mf *Manifest, uri string) ([]byte, error) { +func GetHTTPFile(uri string) ([]byte, error) { log.DebugLog(log.DebugLevelMexos, "attempt to get http uri file", "uri", uri) resp, err := http.Get(uri) if err != nil { @@ -76,7 +76,7 @@ func GetHTTPFile(mf *Manifest, uri string) ([]byte, error) { return nil, fmt.Errorf("http status not OK, %v", resp.StatusCode) } -func GetSCPFile(mf *Manifest, uri string) ([]byte, error) { +func GetSCPFile(uri string) ([]byte, error) { log.DebugLog(log.DebugLevelMexos, "attempt to get scp uri file", "uri", uri) part1 := strings.Replace(uri, "scp://", "mobiledgex@", -1) slashindex := strings.Index(part1, "/") @@ -94,7 +94,7 @@ func GetSCPFile(mf *Manifest, uri string) ([]byte, error) { return sh.Command("ssh", "-o", sshOpts[0], "-o", sshOpts[1], "-i", PrivateSSHKey(), addr, "cat", fn).Output() } -// func CopyURIFile(mf *Manifest, uri string, fn string) error { +// func CopyURIFile(uri string, fn string) error { // res, err := GetURIFile(mf, uri) // if err != nil { // return err diff --git a/mexos/valid.go b/mexos/valid.go deleted file mode 100644 index 614462f8b..000000000 --- a/mexos/valid.go +++ /dev/null @@ -1,163 +0,0 @@ -package mexos - -import ( - "fmt" - "reflect" - "strings" - - "github.com/mobiledgex/edge-cloud/log" -) - -var validMEXOSEnv = false - -func CheckManifest(mf *Manifest) error { - if mf.APIVersion == "" { - return fmt.Errorf("mf apiversion not set") - } - if mf.APIVersion != APIversion { - return fmt.Errorf("invalid api version") - } - return CheckManifestValues(mf) -} - -func CheckManifestValues(mf *Manifest) error { - log.DebugLog(log.DebugLevelMexos, "checking manifest values") - return CheckVals(mf.Values) -} - -func CheckVals(vals interface{}) error { - m := reflect.ValueOf(vals) - if strings.HasPrefix(m.Type().String(), "map[") { - return nil - } - for i := 0; i < m.NumField(); i++ { - //n := m.Type().Field(i).Name - t := m.Type().Field(i).Type - v := m.Field(i).Interface() - //log.DebugLog(log.DebugLevelMexos, "checkvals", "name", n, "type", m.Type(), "field type", t, "value", v) - if t.String() == "[]mexos.PortDetail" { // XXX skip ports, TODO check !nil and validate - continue - } - if t.String() == "string" { - if v == "" { - //log.DebugLog(log.DebugLevelMexos, "checkvals, warning, empty field", "name", n, "type", m.Type(), "field type", t, "value", v) - } - //log.DebugLog(log.DebugLevelMexos, "ok", "name", n, "type", t, "value", v) - } else { - if err := CheckVals(m.Field(i).Interface()); err != nil { - return err - } - } - } - validMEXOSEnv = true - return nil -} - -//ValidateNetSpec parses and validates the netSpec -func ValidateNetSpec(netSpec string) error { - // TODO if _,err:=ParseNetSpec(netSpec); err!=nil{ return err} - if netSpec == "" { - return fmt.Errorf("empty netspec") - } - return nil -} - -//ValidateTags parses and validates tags -func ValidateTags(tags string) error { - // TODO a=b,c=d,... - if tags == "" { - return fmt.Errorf("empty tags") - } - return nil -} - -//ValidateTenant parses and validates tenant -func ValidateTenant(tenant string) error { - // TODO suffix -tenant - if tenant == "" { - return fmt.Errorf("emtpy tenant") - } - return nil -} - -func ValidateClusterKind(kind string) error { - // TODO list of acceptable cluster kinds to be provided elsewhere - log.DebugLog(log.DebugLevelMexos, "cluster kind", "kind", kind) - if kind == "" { - return fmt.Errorf("empty cluster kind") - } - //TODO: add more kinds of clusters - for _, k := range []string{ - "gcp", - "azure", - //"gddt", - } { - if kind == k { - return nil - } - } - // if strings.HasPrefix(kind, "mex-") { - // return nil - // } - // log.DebugLog(log.DebugLevelMexos, "warning, cluster kind, operator has no mex- prefix", "kind", kind) - return nil -} - -func ValidateMetadata(mf *Manifest) error { - //TODO acceptable name patterns to be provided elsewhere - if mf.Metadata.Name == "" { - return fmt.Errorf("missing name for the deployment") - } - return nil -} - -func ValidateKey(mf *Manifest) error { - // TODO use of spec key as cluster name may need to change - if mf.Spec.Key == "" { - return fmt.Errorf("empty spec key name") - } - return nil -} - -func ValidateProxyPath(mf *Manifest) error { - // XXX is it error if this is empty? - if mf.Spec.ProxyPath == "" { - return fmt.Errorf("empty proxy path") - } - return nil -} - -func ValidateImage(mf *Manifest) error { - // TODO check valid list of images, provided elsewhere - if mf.Spec.Image == "" { - return fmt.Errorf("empty image") - } - return nil -} - -func ValidateDNSZone(mf *Manifest) error { - // TODO check against mobiledgex.net and other zones acceptable as listed in DB - if mf.Metadata.DNSZone == "" { - return fmt.Errorf("missing DNS zone, metadata %v", mf.Metadata) - } - return nil -} - -func ValidateCommon(mf *Manifest) error { - if err := ValidateMetadata(mf); err != nil { - return err - } - if err := ValidateKey(mf); err != nil { - return err - } - if err := ValidateProxyPath(mf); err != nil { - return err - } - if err := ValidateImage(mf); err != nil { - return err - } - if err := ValidateDNSZone(mf); err != nil { - return err - } - return nil -} diff --git a/mexos/vault.go b/mexos/vault.go index 0e44eeb92..1c6978931 100644 --- a/mexos/vault.go +++ b/mexos/vault.go @@ -12,6 +12,23 @@ import ( "github.com/mobiledgex/edge-cloud/log" ) +type EnvData struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type VaultDataDetail struct { + Env []EnvData `json:"env"` +} + +type VaultData struct { + Detail VaultDataDetail `json:"data"` +} + +type VaultResponse struct { + Data VaultData `json:"data"` +} + func GetVaultData(url string) ([]byte, error) { vault_token := os.Getenv("VAULT_TOKEN") if vault_token == "" { From 215d762a53de531ed33984433c4c61d0046b2973 Mon Sep 17 00:00:00 2001 From: Jim Date: Thu, 7 Feb 2019 17:49:54 -0600 Subject: [PATCH 03/10] more fixing merge with Jon's PR --- mexos/agent.go | 5 +- mexos/cloudletinfra.go | 9 ++- mexos/cluster.go | 8 +-- mexos/dns.go | 90 ++++++++++++++-------------- mexos/helm.go | 71 ++++++++++++++++------ mexos/inst.go | 92 +++-------------------------- mexos/kubectl.go | 130 ++++++++++++++++++----------------------- mexos/svc.go | 32 +--------- 8 files changed, 178 insertions(+), 259 deletions(-) diff --git a/mexos/agent.go b/mexos/agent.go index a9dae7a25..4a3f39537 100644 --- a/mexos/agent.go +++ b/mexos/agent.go @@ -24,7 +24,7 @@ func runLocalMexAgent() error { func RunMEXAgent(rootLBName string, cloudletKey *edgeproto.CloudletKey) error { log.DebugLog(log.DebugLevelMexos, "run mex agent") - if IsLocalDIND() { + if CloudletIsLocalDIND() { return runLocalMexAgent() } fqdn := rootLBName @@ -52,9 +52,6 @@ func RunMEXAgent(rootLBName string, cloudletKey *edgeproto.CloudletKey) error { if rootLB == nil { return fmt.Errorf("cannot run mex agent manifest, rootLB is null") } - //if mf.Spec.ExternalNetwork == "" { - // return fmt.Errorf("missing external network") - //} if GetCloudletOSImage() == "" { return fmt.Errorf("missing agent image") } diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index cb6e0b764..1f394017a 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -137,10 +137,17 @@ func InitializeCloudletInfra(fakecloudlet bool) error { return nil } -func IsLocalDIND() bool { +func CloudletIsLocalDIND() bool { return CloudletInfra.CloudletKind == cloudcommon.CloudletKindDIND } +// returns true if kubectl can be run directly from the CRM rather than SSH jump thru LB +func CloudletIsDirectKubectlAccess() bool { + return CloudletInfra.CloudletKind == cloudcommon.CloudletKindDIND || + CloudletInfra.CloudletKind == cloudcommon.CloudletKindAzure || + CloudletInfra.CloudletKind == cloudcommon.CloudletKindGCP +} + func GetCloudletKind() string { return CloudletInfra.CloudletKind } diff --git a/mexos/cluster.go b/mexos/cluster.go index 7ab1c6755..8bf3e3886 100644 --- a/mexos/cluster.go +++ b/mexos/cluster.go @@ -364,10 +364,10 @@ func FindClusterMaster(key string) (string, error) { return "", fmt.Errorf("key %s not found", key) } -//MEXClusterCreateInst creates a cluster +//MEXClusterCreateInst creates a cluster. This was formerly MEXClusterCreateManifest func MEXClusterCreateClustInst(clusterInst *edgeproto.ClusterInst, rootLBName string) error { log.DebugLog(log.DebugLevelMexos, "creating cluster instance", "clusterInst", clusterInst, "rootLBName", rootLBName) - if IsLocalDIND() { + if CloudletIsLocalDIND() { return localCreateDIND(clusterInst) } operatorName := NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name) @@ -378,7 +378,6 @@ func MEXClusterCreateClustInst(clusterInst *edgeproto.ClusterInst, rootLBName st case cloudcommon.OperatorAzure: return azureCreateAKS(clusterInst) default: - //guid, err := mexCreateClusterKubernetes(mf) err := mexCreateClusterKubernetes(clusterInst, rootLBName) if err != nil { return fmt.Errorf("can't create cluster, %v", err) @@ -389,12 +388,13 @@ func MEXClusterCreateClustInst(clusterInst *edgeproto.ClusterInst, rootLBName st } } +//MEXClusterRemoveClustInst removes a cluster. This was formerly MEXClusterRemoveManifest func MEXClusterRemoveClustInst(clusterInst *edgeproto.ClusterInst, rootLBName string) error { log.DebugLog(log.DebugLevelMexos, "removing cluster") clusterName := clusterInst.Key.ClusterKey.Name - if IsLocalDIND() { + if CloudletIsLocalDIND() { return dind.DeleteDINDCluster(clusterName) } operatorName := NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name) diff --git a/mexos/dns.go b/mexos/dns.go index 4ed9554fe..95c37b9c7 100644 --- a/mexos/dns.go +++ b/mexos/dns.go @@ -1,22 +1,21 @@ package mexos import ( - "encoding/json" "fmt" "net" "os/exec" - "strings" "time" "github.com/mobiledgex/edge-cloud-infra/k8s-prov/dind" "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" "github.com/mobiledgex/edge-cloud/cloudcommon" "github.com/mobiledgex/edge-cloud/log" + "k8s.io/api/core/v1" ) var dnsRegisterRetryDelay time.Duration = 3 * time.Second -func createAppDNS(kconf string, uri string, name string) error { +func createAppDNS(kp *kubeParam, uri string, name string) error { log.DebugLog(log.DebugLevelMexos, "createAppDNS") @@ -30,23 +29,26 @@ func createAppDNS(kconf string, uri string, name string) error { if err != nil { return err } - - serviceNames, err := getSvcNames(name, kconf) + svcs, err := getServices(kp) if err != nil { return err } - if len(serviceNames) < 1 { - return fmt.Errorf("no service names starting with %s", name) + if len(svcs) < 1 { + return fmt.Errorf("no load balancer services for %s", name) } recs, derr := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if derr != nil { return fmt.Errorf("error getting dns records for %s, %v", GetCloudletDNSZone(), err) } fqdnBase := uri2fqdn(uri) - for _, sn := range serviceNames { + for _, svc := range svcs { + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { + continue + } + sn := svc.ObjectMeta.Name // for the DIND case we need to patch the service here externalIP := "" - if IsLocalDIND() { + if CloudletIsLocalDIND() { addr := dind.GetMasterAddr() err = KubePatchServiceLocal(sn, addr) if err != nil { @@ -54,7 +56,7 @@ func createAppDNS(kconf string, uri string, name string) error { } externalIP, err = dind.GetLocalAddr() } else { - externalIP, err = getSvcExternalIP(sn, kconf) + externalIP, err = getSvcExternalIP(sn, kp) } if err != nil { return err @@ -81,7 +83,7 @@ func createAppDNS(kconf string, uri string, name string) error { return nil } -func deleteAppDNS(kconf string, uri string, name string) error { +func deleteAppDNS(kp *kubeParam, uri string, name string) error { if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { return fmt.Errorf("cannot init cloudflare api, %v", err) @@ -93,19 +95,23 @@ func deleteAppDNS(kconf string, uri string, name string) error { if err != nil { return err } - serviceNames, err := getSvcNames(name, kconf) + svcs, err := getServices(kp) if err != nil { return err } - if len(serviceNames) < 1 { - return fmt.Errorf("no service names starting with %s", name) + if len(svcs) < 1 { + return fmt.Errorf("no services in cluster") } recs, derr := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if derr != nil { return fmt.Errorf("cannot get dns records for dns zone %s, error %v", GetCloudletDNSZone(), err) } fqdnBase := uri2fqdn(uri) - for _, sn := range serviceNames { + for _, svc := range svcs { + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { + continue + } + sn := svc.ObjectMeta.Name fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) for _, rec := range recs { if rec.Type == "A" && rec.Name == fqdn { @@ -134,6 +140,14 @@ func KubePatchServiceLocal(servicename string, ipaddr string) error { return nil } +// TODO: This function and createAppDNS share a lot of duplicate code, +// but are subtly different. It'd be good to consolidate and remove +// duplicate code and highlight what the different use cases are, +// since it's not clear when to use one or the other. +// This should be easier to consolidate now that kubeParam can issue +// commands locally for DIND or other cases. +// Same for KubeDeleteDNSRecords and deleteAppDNS. + func KubeAddDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name string) error { log.DebugLog(log.DebugLevelMexos, "adding dns records for kubenernets app", "name", name) rootLBIPaddr, err := GetServerIPAddr(GetCloudletExternalNetwork(), rootLB.Name) @@ -141,19 +155,11 @@ func KubeAddDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name string log.DebugLog(log.DebugLevelMexos, "cannot get rootlb IP address", "error", err) return fmt.Errorf("cannot deploy kubernetes app, cannot get rootlb IP") } - cmd := fmt.Sprintf("%s kubectl get svc -o json", kp.kubeconfig) - out, err := kp.client.Output(cmd) - if err != nil { - return fmt.Errorf("can not get list of services, %s, %v", out, err) - } - svcs := &svcItems{} - err = json.Unmarshal([]byte(out), svcs) + svcs, err := getServices(kp) if err != nil { - return fmt.Errorf("can not unmarshal svc json, %v", err) - } - if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { - return fmt.Errorf("cannot init cloudflare api, %v", err) + return err } + recs, err := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if err != nil { return fmt.Errorf("error getting dns records for %s, %v", GetCloudletDNSZone(), err) @@ -162,17 +168,17 @@ func KubeAddDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name string fqdnBase := uri2fqdn(uri) log.DebugLog(log.DebugLevelMexos, "kubernetes services", "services", svcs) processed := 0 - for _, item := range svcs.Items { - if !strings.HasPrefix(item.Metadata.Name, name) { + for _, svc := range svcs { + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { continue } - cmd = fmt.Sprintf(`%s kubectl patch svc %s -p '{"spec":{"externalIPs":["%s"]}}'`, kp.kubeconfig, item.Metadata.Name, kp.ipaddr) - out, err = kp.client.Output(cmd) + cmd := fmt.Sprintf(`%s kubectl patch svc %s -p '{"spec":{"externalIPs":["%s"]}}'`, kp.kubeconfig, svc.ObjectMeta.Name, kp.ipaddr) + out, err := kp.client.Output(cmd) if err != nil { return fmt.Errorf("error patching for kubernetes service, %s, %s, %v", cmd, out, err) } - log.DebugLog(log.DebugLevelMexos, "patched externalIPs on service", "service", item.Metadata.Name, "externalIPs", kp.ipaddr) - fqdn := cloudcommon.ServiceFQDN(item.Metadata.Name, fqdnBase) + log.DebugLog(log.DebugLevelMexos, "patched externalIPs on service", "service", svc.ObjectMeta.Name, "externalIPs", kp.ipaddr) + fqdn := cloudcommon.ServiceFQDN(svc.ObjectMeta.Name, fqdnBase) for _, rec := range recs { if rec.Type == "A" && rec.Name == fqdn { if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), rec.ID); err != nil { @@ -203,13 +209,9 @@ func KubeDeleteDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name str if err != nil { return fmt.Errorf("can not get list of services, %s, %v", out, err) } - svcs := &svcItems{} - err = json.Unmarshal([]byte(out), svcs) + svcs, err := getServices(kp) if err != nil { - return fmt.Errorf("can not unmarshal svc json, %v", err) - } - if cerr := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); cerr != nil { - return fmt.Errorf("cannot init cloudflare api, %v", cerr) + return err } recs, derr := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if derr != nil { @@ -217,18 +219,18 @@ func KubeDeleteDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name str } fqdnBase := uri2fqdn(uri) //FIXME use k8s manifest file to delete the whole services and deployments - for _, item := range svcs.Items { - if !strings.HasPrefix(item.Metadata.Name, name) { + for _, svc := range svcs { + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { continue } - // cmd := fmt.Sprintf("%s kubectl delete service %s", kp.kubeconfig, item.Metadata.Name) + // cmd := fmt.Sprintf("%s kubectl delete service %s", kp.kubeconfig, svc.ObjectMeta.Name) // out, err := kp.client.Output(cmd) // if err != nil { - // log.DebugLog(log.DebugLevelMexos, "error deleting kubernetes service", "name", item.Metadata.Name, "cmd", cmd, "out", out, "err", err) + // log.DebugLog(log.DebugLevelMexos, "error deleting kubernetes service", "name", svc.ObjectMeta.Name, "cmd", cmd, "out", out, "err", err) // } else { - // log.DebugLog(log.DebugLevelMexos, "deleted service", "name", item.Metadata.Name) + // log.DebugLog(log.DebugLevelMexos, "deleted service", "name", svc.ObjectMeta.Name) // } - fqdn := cloudcommon.ServiceFQDN(item.Metadata.Name, fqdnBase) + fqdn := cloudcommon.ServiceFQDN(svc.ObjectMeta.Name, fqdnBase) for _, rec := range recs { if rec.Type == "A" && rec.Name == fqdn { if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), rec.ID); err != nil { diff --git a/mexos/helm.go b/mexos/helm.go index 5367f87bd..006b03287 100644 --- a/mexos/helm.go +++ b/mexos/helm.go @@ -2,6 +2,7 @@ package mexos import ( "fmt" + "regexp" "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" @@ -10,21 +11,33 @@ import ( func DeleteHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { log.DebugLog(log.DebugLevelMexos, "delete kubernetes helm app") - clusterName := clusterInst.Key.ClusterKey.Name appName := NormalizeName(app.Key.Name) + clusterName := clusterInst.Key.ClusterKey.Name + var err error + if rootLB == nil { + return fmt.Errorf("cannot delete helm app, rootLB is null") + } kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) if err != nil { return err } - // remove DNS entries - if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { - log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) - } - // remove Security rules - if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { - log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) + if CloudletIsLocalDIND() { + // remove DNS entries + if err = deleteAppDNS(kp, appInst.Uri, appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) + } + } else { + // remove DNS entries + if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) + } + // remove Security rules + if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) + } } + cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, appName) out, err := kp.client.Output(cmd) if err != nil { @@ -40,7 +53,11 @@ func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku clusterName := clusterInst.Key.ClusterKey.Name appName := NormalizeName(app.Key.Name) appImage := app.ImagePath + var err error + if rootLB == nil { + return fmt.Errorf("cannot create helm app, rootLB is null") + } kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) if err != nil { return err @@ -75,21 +92,39 @@ func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku log.DebugLog(log.DebugLevelMexos, "helm tiller initialized") } - cmd = fmt.Sprintf("%s helm install %s --name %s", kp.kubeconfig, appImage, appName) + helmOpts := "" + // XXX This gets helm's prometheus able to query kubelet metrics. + // This can be removed once Lev passes in an option in the yaml to + // set the helm command line options. + prom, err := regexp.MatchString("prometheus", appName) + if err == nil && prom { + log.DebugLog(log.DebugLevelMexos, "setting helm prometheus option") + helmOpts = "--set kubelet.serviceMonitor.https=true" + } + cmd = fmt.Sprintf("%s helm install %s --name %s %s", kp.kubeconfig, appImage, appName, helmOpts) out, err = kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deploying helm chart, %s, %s, %v", cmd, out, err) } log.DebugLog(log.DebugLevelMexos, "applied helm chart") - // Add security rules - if err = AddProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) - return err - } - - if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) - return err + if CloudletIsLocalDIND() { + // Add DNS Zone + if err = createAppDNS(kp, appInst.Uri, appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) + return err + } + } else { + // Add security rules + if err = AddProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "done AddProxySecurityRules", "app", app) + // Add DNS Zone + if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) + return err + } } return nil } diff --git a/mexos/inst.go b/mexos/inst.go index 2743df7bc..f3e31c614 100644 --- a/mexos/inst.go +++ b/mexos/inst.go @@ -9,85 +9,7 @@ import ( "github.com/mobiledgex/edge-cloud/log" ) -/* -//MEXClusterRemoveClustInst calls MEXClusterRemove with a manifest created from the template -func MEXClusterRemoveClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) error { - mf, err := FillClusterTemplateClustInst(rootLB, clusterInst) - if err != nil { - return err - } - fixValuesInst(mf, rootLB) - return MEXClusterRemoveManifest(mf) -} -*/ -/* -func FillClusterTemplateClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) (*Manifest, error) { - log.DebugLog(log.DebugLevelMexos, "fill cluster template manifest cluster inst", "clustinst", clusterInst) - if clusterInst.Key.ClusterKey.Name == "" { - log.DebugLog(log.DebugLevelMexos, "cannot create empty cluster manifest", "clustinst", clusterInst) - return nil, fmt.Errorf("invalid cluster inst %v", clusterInst) - } - if verr := ValidateClusterKind(clusterInst.Key.CloudletKey.OperatorKey.Name); verr != nil { - return nil, verr - } - vp := &rootLB.PlatConf.Values - data := templateFill{ - ResourceKind: "cluster", - Resource: clusterInst.Flavor.Name, - Name: clusterInst.Key.ClusterKey.Name, - Tags: clusterInst.Key.ClusterKey.Name + "-tag", - Tenant: clusterInst.Key.ClusterKey.Name + "-tenant", - Operator: NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name), - Key: clusterInst.Key.ClusterKey.Name, - Kind: vp.Cluster.Kind, //"kubernetes", - ResourceGroup: clusterInst.Key.CloudletKey.Name + "_" + clusterInst.Key.ClusterKey.Name, - Flavor: clusterInst.Flavor.Name, - DNSZone: GetCloudletDNSZone(), - RootLB: rootLB.Name, - NetworkScheme: GetCloudletNetworkScheme(), - Swarm: vp.Cluster.Swarm, - } - - mf, err := templateUnmarshal(&data, yamlMEXCluster) - if err != nil { - return nil, err - } - fixValuesInst(mf, rootLB) - return mf, nil -} */ - -/* this function never actually did anything -func MEXAddFlavorClusterInst(rootLB *MEXRootLB, flavor *edgeproto.ClusterFlavor) error { - log.DebugLog(log.DebugLevelMexos, "adding cluster inst flavor", "flavor", flavor) - - if flavor.Key.Name == "" { - log.DebugLog(log.DebugLevelMexos, "cannot add empty cluster inst flavor", "flavor", flavor) - return fmt.Errorf("will not add empty cluster inst %v", flavor) - } - vp := &rootLB.PlatConf.Values - data := templateFill{ - ResourceKind: "flavor", - Resource: flavor.Key.Name, - Name: flavor.Key.Name, - Tags: flavor.Key.Name + "-tag", - Kind: vp.Cluster.Kind, - Flags: flavor.Key.Name + "-flags", - NumNodes: int(flavor.NumNodes), - NumMasters: int(flavor.NumMasters), - NetworkScheme: GetCloudletNetworkScheme(), - NodeFlavor: flavor.NodeFlavor.Name, - StorageSpec: "default", //XXX - } - mf, err := templateUnmarshal(&data, yamlMEXFlavor) - if err != nil { - return err - } - fixValuesInst(mf, rootLB) - return MEXAddFlavor(mf) -} -*/ - -//MEXAppDeleteAppInst deletes app with templated manifest +//MEXAppDeleteAppInst deletes app instance func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, app *edgeproto.App, appInst *edgeproto.AppInst) error { log.DebugLog(log.DebugLevelMexos, "mex create app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) @@ -98,7 +20,7 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, //TODO values.application.template - if IsLocalDIND() { + if CloudletIsLocalDIND() { masteraddr := dind.GetMasterAddr() log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind") @@ -112,7 +34,7 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return err } log.DebugLog(log.DebugLevelMexos, "call runKubectlCreateApp for dind") - err = runKubectlCreateApp(clusterInst, appInst, app.DeploymentManifest) + err = runKubectlCreateApp(clusterInst, appInst, rootLB, app.DeploymentManifest) if err != nil { log.DebugLog(log.DebugLevelMexos, "error creating dind app") return err @@ -125,7 +47,7 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, fallthrough case cloudcommon.OperatorAzure: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return runKubectlCreateApp(clusterInst, appInst, app.DeploymentManifest) + return runKubectlCreateApp(clusterInst, appInst, rootLB, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { return fmt.Errorf("not yet supported") } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { @@ -157,9 +79,9 @@ func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, operatorName := NormalizeName(appInst.Key.CloudletKey.OperatorKey.Name) appName := NormalizeName(app.Key.Name) - if IsLocalDIND() { + if CloudletIsLocalDIND() { log.DebugLog(log.DebugLevelMexos, "run kubectl delete app for dind") - err := runKubectlDeleteApp(clusterInst, appInst, app.DeploymentManifest) + err := runKubectlDeleteApp(clusterInst, appInst, rootLB, app.DeploymentManifest) if err != nil { return err } @@ -179,7 +101,7 @@ func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, fallthrough case cloudcommon.OperatorAzure: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return runKubectlDeleteApp(clusterInst, appInst, app.DeploymentManifest) + return runKubectlDeleteApp(clusterInst, appInst, rootLB, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { return fmt.Errorf("not yet supported") } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { diff --git a/mexos/kubectl.go b/mexos/kubectl.go index 10a429fbb..f0e2abfe5 100644 --- a/mexos/kubectl.go +++ b/mexos/kubectl.go @@ -9,10 +9,9 @@ import ( "time" sh "github.com/codeskyblue/go-sh" - "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" - "github.com/mobiledgex/edge-cloud/cloudcommon" "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" + "k8s.io/api/core/v1" ) func CreateDockerRegistrySecret(clusterInst *edgeproto.ClusterInst, rootLBName string) error { @@ -20,9 +19,11 @@ func CreateDockerRegistrySecret(clusterInst *edgeproto.ClusterInst, rootLBName s var out string var err error - log.DebugLog(log.DebugLevelMexos, "creating docker registry secret in kubernetes cluster") - if rootLBName == "" { - log.DebugLog(log.DebugLevelMexos, "CreateDockerRegistrySecret locally, no rootLB") + + log.DebugLog(log.DebugLevelMexos, "creating docker registry secret in kubrnetes cluster") + + if CloudletIsDirectKubectlAccess() { + log.DebugLog(log.DebugLevelMexos, "CreateDockerRegistrySecret locally") var o []byte o, err = sh.Command("kubectl", "create", "secret", "docker-registry", "mexregistrysecret", "--docker-server="+GetCloudletDockerRegistry(), "--docker-username=mobiledgex", "--docker-password="+GetCloudletDockerPass(), "--docker-email=mobiledgex@mobiledgex.com").CombinedOutput() out = string(o) @@ -32,7 +33,6 @@ func CreateDockerRegistrySecret(clusterInst *edgeproto.ClusterInst, rootLBName s return fmt.Errorf("can't get ssh client, %v", err) } cmd := fmt.Sprintf("kubectl create secret docker-registry mexregistrysecret --docker-server=%s --docker-username=mobiledgex --docker-password=%s --docker-email=mobiledgex@mobiledgex.com --kubeconfig=%s", GetCloudletDockerRegistry(), GetCloudletDockerPass(), GetKconfName(clusterInst)) - out, err = client.Output(cmd) } if err != nil { @@ -46,57 +46,66 @@ func CreateDockerRegistrySecret(clusterInst *edgeproto.ClusterInst, rootLBName s return nil } -func runKubectlCreateApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, kubeManifest string) error { +func runKubectlCreateApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, rootLB *MEXRootLB, kubeManifest string) error { log.DebugLog(log.DebugLevelMexos, "run kubectl create app", "kubeManifest", kubeManifest) appName := NormalizeName(appInst.Key.AppKey.Name) - + clusterName := clusterInst.Key.ClusterKey.Name kfile := appName + ".yaml" + if err := writeKubeManifest(kubeManifest, kfile); err != nil { return err } defer os.Remove(kfile) - kconf, err := GetKconf(clusterInst, false) + + kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) if err != nil { - return fmt.Errorf("error creating app due to kconf missing, %v, %v", clusterInst, err) + return err } - out, err := sh.Command("kubectl", "create", "-f", kfile, "--kubeconfig="+kconf).Output() + cmd := fmt.Sprintf("%s kubectl create -f %s", kp.kubeconfig, kfile) + out, err := kp.client.Output(cmd) if err != nil { - return fmt.Errorf("error creating app, %s, %v, %v", out, appName, err) + return fmt.Errorf("error creating app, %s, %v, %s", out, err, kubeManifest) } - err = createAppDNS(kconf, appInst.Uri, appName) + defer func() { + if err != nil { + cmd = fmt.Sprintf("%s kubectl delete -f %s", kp.kubeconfig, kfile) + out, undoerr := kp.client.Output(cmd) + if undoerr != nil { + log.DebugLog(log.DebugLevelMexos, "undo kubectl create app failed", "name", appName, "out", out, "err", undoerr) + } + } + }() + err = createAppDNS(kp, appInst.Uri, appName) if err != nil { - return fmt.Errorf("error creating dns entry for app, %v, %v", appName, err) + return fmt.Errorf("error creating dns entry for app, %v", err) } return nil } -func getSvcExternalIP(name string, kconf string) (string, error) { - log.DebugLog(log.DebugLevelMexos, "get service external IP", "name", name, "kconf", kconf) +func getSvcExternalIP(name string, kp *kubeParam) (string, error) { + log.DebugLog(log.DebugLevelMexos, "get service external IP", "name", name, "kp", kp) externalIP := "" - var out []byte - var err error //wait for Load Balancer to assign external IP address. It takes a variable amount of time. for i := 0; i < 100; i++ { - out, err = sh.Command("kubectl", "get", "svc", "--kubeconfig="+kconf, "-o", "json").Output() + cmd := fmt.Sprintf("%s kubectl get svc -o json", kp.kubeconfig) + out, err := kp.client.Output(cmd) if err != nil { return "", fmt.Errorf("error getting svc %s, %s, %v", name, out, err) } - svcs := &svcItems{} - err = json.Unmarshal(out, svcs) + svcs, err := getServices(kp) if err != nil { - return "", fmt.Errorf("error unmarshalling svc json, %v", err) + return "", err } log.DebugLog(log.DebugLevelMexos, "getting externalIP, examine list of services", "name", name, "svcs", svcs) - for _, item := range svcs.Items { - log.DebugLog(log.DebugLevelMexos, "svc item", "item", item, "name", name) - if item.Metadata.Name != name { - log.DebugLog(log.DebugLevelMexos, "service name mismatch", "name", name, "item.Metadata.Name", item.Metadata.Name) + for _, svc := range svcs { + log.DebugLog(log.DebugLevelMexos, "svc item", "item", svc, "name", name) + if svc.ObjectMeta.Name != name { + log.DebugLog(log.DebugLevelMexos, "service name mismatch", "name", name, "svc.ObjectMeta.Name", svc.ObjectMeta.Name) continue } - for _, ingress := range item.Status.LoadBalancer.Ingresses { - log.DebugLog(log.DebugLevelMexos, "found ingress ip", "ingress.IP", ingress.IP, "item.Metadata.Name", item.Metadata.Name) - + for _, ingress := range svc.Status.LoadBalancer.Ingress { + log.DebugLog(log.DebugLevelMexos, "found ingress ip", "ingress.IP", ingress.IP, "svc.ObjectMeta.Name", svc.ObjectMeta.Name) if ingress.IP != "" { externalIP = ingress.IP log.DebugLog(log.DebugLevelMexos, "got externaIP for app", "externalIP", externalIP) @@ -112,36 +121,28 @@ func getSvcExternalIP(name string, kconf string) (string, error) { return externalIP, nil } -func getSvcNames(name string, kconf string) ([]string, error) { - out, err := sh.Command("kubectl", "get", "svc", "--kubeconfig="+kconf, "-o", "json").Output() +func getServices(kp *kubeParam) ([]v1.Service, error) { + cmd := fmt.Sprintf("%s kubectl get svc -o json", kp.kubeconfig) + out, err := kp.client.Output(cmd) if err != nil { - return nil, fmt.Errorf("error getting svc %s, %s, %v", name, out, err) + return nil, fmt.Errorf("can not get list of services, %s, %v", out, err) } - svcs := &svcItems{} - err = json.Unmarshal(out, svcs) + svcs := svcItems{} + err = json.Unmarshal([]byte(out), &svcs) if err != nil { - return nil, fmt.Errorf("error unmarshalling svc json, %v", err) + log.DebugLog(log.DebugLevelMexos, "cannot unmarshal svc json", "out", out, "err", err) + return nil, fmt.Errorf("cannot unmarshal svc json, %s", err.Error()) } - var serviceNames []string - for _, item := range svcs.Items { - if strings.HasPrefix(item.Metadata.Name, name) { - serviceNames = append(serviceNames, item.Metadata.Name) - } - } - log.DebugLog(log.DebugLevelMexos, "service names", "names", serviceNames) - return serviceNames, nil + return svcs.Items, nil } -func runKubectlDeleteApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, kubeManifest string) error { - if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { - return fmt.Errorf("cannot init cloudflare api, %v", err) - } - +func runKubectlDeleteApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, rootLB *MEXRootLB, kubeManifest string) error { appName := NormalizeName(appInst.Key.AppKey.Name) + clusterName := clusterInst.Key.ClusterKey.Name - kconf, err := GetKconf(clusterInst, false) + kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) if err != nil { - return fmt.Errorf("error deleting app due to kconf missing, %v, %v", clusterInst, err) + return err } kfile := appName + ".yaml" err = writeKubeManifest(kubeManifest, kfile) @@ -149,33 +150,14 @@ func runKubectlDeleteApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto. return err } defer os.Remove(kfile) - serviceNames, err := getSvcNames(appName, kconf) - if err != nil { - return err - } - if len(serviceNames) < 1 { - return fmt.Errorf("no service names starting with %s", appName) - } - out, err := sh.Command("kubectl", "delete", "-f", kfile, "--kubeconfig="+kconf).CombinedOutput() + cmd := fmt.Sprintf("%s kubectl delete -f %s", kp.kubeconfig, kfile) + out, err := kp.client.Output(cmd) if err != nil { - return fmt.Errorf("error deleting app, %s, %v, %v", out, appName, err) + return fmt.Errorf("error deleting app, %s, %v", out, err) } - - fqdnBase := uri2fqdn(appInst.Uri) - dr, err := cloudflare.GetDNSRecords(GetCloudletDNSZone()) + err = deleteAppDNS(kp, appInst.Uri, appName) if err != nil { - return fmt.Errorf("cannot get dns records for %s, %v", GetCloudletDNSZone(), err) - } - for _, sn := range serviceNames { - fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) - for _, d := range dr { - if d.Type == "A" && d.Name == fqdn { - if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), d.ID); err != nil { - return fmt.Errorf("cannot delete DNS record, %v", d) - } - log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) - } - } + return fmt.Errorf("error deleting dns entry for app, %v, %v", appInst, err) } return nil } diff --git a/mexos/svc.go b/mexos/svc.go index 9c66bee14..7b82b9089 100644 --- a/mexos/svc.go +++ b/mexos/svc.go @@ -1,5 +1,7 @@ package mexos +import "k8s.io/api/core/v1" + type ksaPort struct { Name string `json:"name"` Protocol string `json:"protocol"` @@ -16,34 +18,6 @@ type kubernetesServiceAbbrev struct { Spec ksaSpec `json:"spec"` } -type ingressItem struct { - IP string `json:"ip"` -} - -type loadBalancerItem struct { - Ingresses []ingressItem `json:"ingress"` -} - -type statusItem struct { - LoadBalancer loadBalancerItem `json:"loadBalancer"` -} - -type metadataItem struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - CreationTimestamp string `json:"creationTimestamp"` - ResourceVersion string `json:"resourceVersion"` - UID string `json:"uid"` -} - -type svcItem struct { - APIVersion string `json:"apiVersion"` - Kind string `json:"kind"` - Metadata metadataItem `json:"metadata"` - Spec interface{} `json:"spec"` - Status statusItem `json:"status"` -} - type svcItems struct { - Items []svcItem `json:"items"` + Items []v1.Service `json:"items"` } From e0d0cfe73ded2e347e8076f9b45344fef7f72749 Mon Sep 17 00:00:00 2001 From: Jim Date: Thu, 7 Feb 2019 19:53:47 -0600 Subject: [PATCH 04/10] more fixes for helm --- k8s-prov/dind/dind.go | 20 ++++++++++++------- mexos/cloudletinfra.go | 9 ++++----- mexos/helm.go | 2 +- mexos/inst.go | 44 ++++++++++++++++++++++++++++++++---------- mexos/kubeadmdind.go | 3 ++- mexos/kubernetes.go | 13 +++++++++++++ 6 files changed, 67 insertions(+), 24 deletions(-) diff --git a/k8s-prov/dind/dind.go b/k8s-prov/dind/dind.go index 349f40ed9..79eccfbda 100644 --- a/k8s-prov/dind/dind.go +++ b/k8s-prov/dind/dind.go @@ -27,7 +27,6 @@ func GetLocalAddr() (string, error) { defer conn.Close() localAddr := conn.LocalAddr().(*net.UDPAddr) - return localAddr.IP.String(), nil } @@ -36,23 +35,30 @@ func GetDockerNetworkName(clusterName string) string { } //CreateDINDCluster creates kubernetes cluster on local mac -func CreateDINDCluster(name string) error { - os.Setenv("DIND_LABEL", name) +func CreateDINDCluster(clusterName, kconfName string) error { + + os.Setenv("DIND_LABEL", clusterName) os.Setenv("CLUSTER_ID", getClusterID()) - log.DebugLog(log.DebugLevelMexos, "CreateDINDCluster via dind-cluster-v1.13.sh", "name", name, "clusterid", getClusterID()) + log.DebugLog(log.DebugLevelMexos, "CreateDINDCluster via dind-cluster-v1.13.sh", "name", clusterName, "clusterid", getClusterID()) out, err := sh.Command("dind-cluster-v1.13.sh", "up").CombinedOutput() if err != nil { return fmt.Errorf("ERROR creating Dind Cluster: [%s] %v", out, err) } - log.DebugLog(log.DebugLevelMexos, "Finished CreateDINDCluster", "name", name) + log.DebugLog(log.DebugLevelMexos, "Finished CreateDINDCluster", "name", clusterName) //now set the k8s config - out, err = sh.Command("kubectl", "config", "use-context", "dind-"+name+"-"+getClusterID()).CombinedOutput() + out, err = sh.Command("kubectl", "config", "use-context", "dind-"+clusterName+"-"+getClusterID()).CombinedOutput() if err != nil { return fmt.Errorf("ERROR setting kube config context: [%s] %v", out, err) } - + //copy kubeconfig locally + log.DebugLog(log.DebugLevelMexos, "locally copying kubeconfig", "kconfName", kconfName) + home := os.Getenv("HOME") + out, err = sh.Command("cp", home+"/.kube/config", home+"/.kube/"+kconfName).CombinedOutput() + if err != nil { + return fmt.Errorf("%s %v", out, err) + } return nil } diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index 1f394017a..5dfc3e916 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -62,11 +62,6 @@ func InitializeCloudletInfra(fakecloudlet bool) error { return fmt.Errorf("Env variable MEXENV_URL not set") } - openRcURL = os.Getenv("OPENRC_URL") - if openRcURL == "" { - return fmt.Errorf("Env variable OPENRC_URL not set") - } - err := InternVaultEnv(openRcURL, mexEnvURL, CloudletInfra) if err != nil { return fmt.Errorf("failed to InternVaultEnv: %v", err) @@ -88,6 +83,10 @@ func InitializeCloudletInfra(fakecloudlet bool) error { switch CloudletInfra.CloudletKind { case cloudcommon.CloudletKindOpenStack: + openRcURL = os.Getenv("OPENRC_URL") + if openRcURL == "" { + return fmt.Errorf("Env variable OPENRC_URL not set") + } CloudletInfra.OpenstackProperties.OSExternalNetworkName = os.Getenv("MEX_EXT_NETWORK") if CloudletInfra.OpenstackProperties.OSExternalNetworkName == "" { diff --git a/mexos/helm.go b/mexos/helm.go index 006b03287..2adf73cf5 100644 --- a/mexos/helm.go +++ b/mexos/helm.go @@ -52,7 +52,7 @@ func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku clusterName := clusterInst.Key.ClusterKey.Name appName := NormalizeName(app.Key.Name) - appImage := app.ImagePath + appImage := NormalizeName(app.ImagePath) var err error if rootLB == nil { diff --git a/mexos/inst.go b/mexos/inst.go index f3e31c614..67900e348 100644 --- a/mexos/inst.go +++ b/mexos/inst.go @@ -29,12 +29,23 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, log.DebugLog(log.DebugLevelMexos, "GetPortDetail failed", "appInst", appInst, "err", err) return err } - if err := AddNginxProxy("localhost", appName, masteraddr, portDetail, dind.GetDockerNetworkName(clusterName)); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", appName, "ports", appInst.MappedPorts) - return err + + if len(portDetail) > 0 { + log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind", "ports", portDetail) + if err := AddNginxProxy("localhost", appName, masteraddr, portDetail, dind.GetDockerNetworkName(clusterName)); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", appName, "ports", portDetail) + return err + } } + log.DebugLog(log.DebugLevelMexos, "call runKubectlCreateApp for dind") - err = runKubectlCreateApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + err = runKubectlCreateApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + err = CreateHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + } else { + err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) + } if err != nil { log.DebugLog(log.DebugLevelMexos, "error creating dind app") return err @@ -81,21 +92,34 @@ func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, if CloudletIsLocalDIND() { log.DebugLog(log.DebugLevelMexos, "run kubectl delete app for dind") - err := runKubectlDeleteApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + + var err error + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + err = runKubectlDeleteApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + err = DeleteHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + } else { + err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) + } if err != nil { return err } log.DebugLog(log.DebugLevelMexos, "call DeleteNginxProxy for dind") - - if err = DeleteNginxProxy("localhost", appName); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", appName) + portDetail, err := GetPortDetail(appInst) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "GetPortDetail failed", "appInst", appInst, "err", err) return err } - + if len(portDetail) > 0 { + if err = DeleteNginxProxy("localhost", appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", appName) + return err + } + } return nil - } + switch operatorName { case cloudcommon.OperatorGCP: fallthrough diff --git a/mexos/kubeadmdind.go b/mexos/kubeadmdind.go index 2055eb36a..71ea85f62 100644 --- a/mexos/kubeadmdind.go +++ b/mexos/kubeadmdind.go @@ -15,7 +15,8 @@ func localCreateDIND(clusterInst *edgeproto.ClusterInst) error { clusterName := clusterInst.Key.ClusterKey.Name log.DebugLog(log.DebugLevelMexos, "creating local dind cluster", "clusterName", clusterName) - if err = dind.CreateDINDCluster(clusterName); err != nil { + kconfName := GetKconfName(clusterInst) + if err = dind.CreateDINDCluster(clusterName, kconfName); err != nil { return err } //race condition exists where the config file is not ready until just after the cluster create is done diff --git a/mexos/kubernetes.go b/mexos/kubernetes.go index aaeb320e1..1598bf2cb 100644 --- a/mexos/kubernetes.go +++ b/mexos/kubernetes.go @@ -72,6 +72,19 @@ func ValidateKubernetesParameters(clusterInst *edgeproto.ClusterInst, rootLB *ME log.DebugLog(log.DebugLevelMexos, "validate kubernetes parameters rootLB", "cluster", clustName) clusterName := clusterInst.Key.ClusterKey.Name + if CloudletIsDirectKubectlAccess() { + // No ssh jump host (rootlb) but kconf configures how to + // talk to remote kubernetes cluster. This includes DIND, AKS, GCP + kconf, err := GetKconf(clusterInst, true) + if err != nil { + return nil, fmt.Errorf("kconf missing, %v, %v", clustName, err) + } + kp := kubeParam{ + kubeconfig: fmt.Sprintf("KUBECONFIG=%s", kconf), + client: &sshLocal{}, + } + return &kp, nil + } if rootLB == nil { return nil, fmt.Errorf("cannot validate kubernetes parameters, rootLB is null") } From 49158698fe3ecd66c982b5f7bdb2f9ecc976952a Mon Sep 17 00:00:00 2001 From: Jim Date: Fri, 8 Feb 2019 12:25:06 -0600 Subject: [PATCH 05/10] fix dns error --- mexos/dns.go | 44 ++++++++++++++++++++++++++++---------------- mexos/kubeconfig.go | 1 - 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/mexos/dns.go b/mexos/dns.go index 95c37b9c7..e62fc5970 100644 --- a/mexos/dns.go +++ b/mexos/dns.go @@ -50,9 +50,13 @@ func createAppDNS(kp *kubeParam, uri string, name string) error { externalIP := "" if CloudletIsLocalDIND() { addr := dind.GetMasterAddr() - err = KubePatchServiceLocal(sn, addr) - if err != nil { - return err + if len(svc.Spec.ExternalIPs) > 0 && svc.Spec.ExternalIPs[0] == addr { + log.DebugLog(log.DebugLevelMexos, "external IP already present in DIND, no patch required", "addr", addr) + } else { + err = KubePatchServiceLocal(sn, addr) + if err != nil { + return err + } } externalIP, err = dind.GetLocalAddr() } else { @@ -62,23 +66,31 @@ func createAppDNS(kp *kubeParam, uri string, name string) error { return err } fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) + recordMatch := false for _, rec := range recs { if rec.Type == "A" && rec.Name == fqdn { - if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), rec.ID); err != nil { - return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) + if rec.Content == externalIP { + log.DebugLog(log.DebugLevelMexos, "DNS record already correct", "fqdn", fqdn, "addr", externalIP) + recordMatch = true + } else { + if err := cloudflare.DeleteDNSRecord(GetCloudletDNSZone(), rec.ID); err != nil { + return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) + } } - log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) } } - if err := cloudflare.CreateDNSRecord(GetCloudletDNSZone(), fqdn, "A", externalIP, 1, false); err != nil { - return fmt.Errorf("can't create DNS record for %s,%s, %v", fqdn, externalIP, err) + if !recordMatch { + if err := cloudflare.CreateDNSRecord(GetCloudletDNSZone(), fqdn, "A", externalIP, 1, false); err != nil { + return fmt.Errorf("can't create DNS record for %s,%s, %v", fqdn, externalIP, err) + } + + //log.DebugLog(log.DebugLevelMexos, "waiting for DNS record to be created on cloudflare...") + //err = WaitforDNSRegistration(fqdn) + //if err != nil { + // return err + //} + log.DebugLog(log.DebugLevelMexos, "registered DNS name, may still need to wait for propagation", "name", fqdn, "externalIP", externalIP) } - //log.DebugLog(log.DebugLevelMexos, "waiting for DNS record to be created on cloudflare...") - //err = WaitforDNSRegistration(fqdn) - //if err != nil { - // return err - //} - log.DebugLog(log.DebugLevelMexos, "registered DNS name, may still need to wait for propagation", "name", fqdn, "externalIP", externalIP) } return nil } @@ -128,13 +140,13 @@ func deleteAppDNS(kp *kubeParam, uri string, name string) error { // KubePatchServiceLocal updates the service to have the given external ip. This is done locally and not thru // an ssh client func KubePatchServiceLocal(servicename string, ipaddr string) error { - log.DebugLog(log.DebugLevelMexos, "KubePatchServiceLocal", "servicename", servicename, "ipaddr", ipaddr) ips := fmt.Sprintf(`{"spec":{"externalIPs":["%s"]}}'`, ipaddr) log.DebugLog(log.DebugLevelMexos, "KubePatchServiceLocal", "servicename", servicename, "ipaddr", ipaddr, "ipspec", ips) - _, err := exec.Command("kubectl", "patch", "svc", servicename, "-p", ips).Output() + out, err := exec.Command("kubectl", "patch", "svc", servicename, "-p", ips).CombinedOutput() if err != nil { + log.DebugLog(log.DebugLevelMexos, "patch svc failed", "servicename", servicename, "out", out, "err", err) return fmt.Errorf("error patching for kubernetes service ip: %s, name: %s, err: %v", ipaddr, servicename, err) } return nil diff --git a/mexos/kubeconfig.go b/mexos/kubeconfig.go index 1710a1f33..56148900a 100644 --- a/mexos/kubeconfig.go +++ b/mexos/kubeconfig.go @@ -30,7 +30,6 @@ func GetKconf(clusterInst *edgeproto.ClusterInst, createIfMissing bool) (string, log.DebugLog(log.DebugLevelMexos, "get kubeconfig name", "name", name) if createIfMissing { // XXX - log.DebugLog(log.DebugLevelMexos, "warning, creating missing kubeconfig", "name", name) if _, err := os.Stat(name); os.IsNotExist(err) { // if kubeconfig does not exist, optionally create it. It is possible it was // created on a different container or we had a restart of the container From 9b69dac65e1861ec19a77b5c5d0c068ef58edb82 Mon Sep 17 00:00:00 2001 From: Jim Date: Fri, 8 Feb 2019 19:00:56 -0600 Subject: [PATCH 06/10] more fixing after testing, plus add Lev's fix --- mexos/cloudletinfra.go | 5 ++--- mexos/dns.go | 14 +++++++------- mexos/kubernetes.go | 9 +++++++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index 5dfc3e916..020cbe207 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -61,12 +61,11 @@ func InitializeCloudletInfra(fakecloudlet bool) error { if mexEnvURL == "" { return fmt.Errorf("Env variable MEXENV_URL not set") } - + openRcURL = os.Getenv("OPENRC_URL") err := InternVaultEnv(openRcURL, mexEnvURL, CloudletInfra) if err != nil { return fmt.Errorf("failed to InternVaultEnv: %v", err) } - CloudletInfraCommon.CFKey = os.Getenv("MEX_CF_KEY") if CloudletInfraCommon.CFKey == "" { return fmt.Errorf("Env variable MEX_CF_KEY not set") @@ -83,7 +82,7 @@ func InitializeCloudletInfra(fakecloudlet bool) error { switch CloudletInfra.CloudletKind { case cloudcommon.CloudletKindOpenStack: - openRcURL = os.Getenv("OPENRC_URL") + if openRcURL == "" { return fmt.Errorf("Env variable OPENRC_URL not set") } diff --git a/mexos/dns.go b/mexos/dns.go index e62fc5970..c360b1dbc 100644 --- a/mexos/dns.go +++ b/mexos/dns.go @@ -167,11 +167,14 @@ func KubeAddDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name string log.DebugLog(log.DebugLevelMexos, "cannot get rootlb IP address", "error", err) return fmt.Errorf("cannot deploy kubernetes app, cannot get rootlb IP") } + svcs, err := getServices(kp) if err != nil { return err } - + if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } recs, err := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if err != nil { return fmt.Errorf("error getting dns records for %s, %v", GetCloudletDNSZone(), err) @@ -225,6 +228,9 @@ func KubeDeleteDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name str if err != nil { return err } + if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } recs, derr := cloudflare.GetDNSRecords(GetCloudletDNSZone()) if derr != nil { return fmt.Errorf("error getting dns records for %s, %v", GetCloudletDNSZone(), derr) @@ -252,12 +258,6 @@ func KubeDeleteDNSRecords(rootLB *MEXRootLB, kp *kubeParam, uri string, name str } } } - cmd = fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, name) - out, err = kp.client.Output(cmd) - if err != nil { - return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", name, cmd, out, err) - } - return nil } diff --git a/mexos/kubernetes.go b/mexos/kubernetes.go index 1598bf2cb..7d74aa298 100644 --- a/mexos/kubernetes.go +++ b/mexos/kubernetes.go @@ -3,6 +3,7 @@ package mexos import ( "fmt" + "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" ssh "github.com/nanobox-io/golang-ssh" @@ -122,11 +123,19 @@ func DeleteKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterIn if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot clean up security rules", "name", appName, "rootlb", rootLB.Name, "error", err) } + if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } // Clean up DNS entries if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot clean up DNS entries", "name", appName, "rootlb", rootLB.Name, "error", err) return err } + cmd := fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, appName) + out, err := kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", appName, cmd, out, err) + } log.DebugLog(log.DebugLevelMexos, "deleted deployment", "name", appName) return nil } From 2e8815d1c3701392d4013c1be1539e164eff8009 Mon Sep 17 00:00:00 2001 From: Jim Date: Fri, 8 Feb 2019 20:24:23 -0600 Subject: [PATCH 07/10] fix for azure --- mexos/agent.go | 4 ++++ mexos/cloudletinfra.go | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mexos/agent.go b/mexos/agent.go index 4a3f39537..b842fd745 100644 --- a/mexos/agent.go +++ b/mexos/agent.go @@ -27,6 +27,10 @@ func RunMEXAgent(rootLBName string, cloudletKey *edgeproto.CloudletKey) error { if CloudletIsLocalDIND() { return runLocalMexAgent() } + if CloudletIsPublicCloud() { + log.DebugLog(log.DebugLevelMexos, "skip mex agent for public cloud") //TODO: maybe later we will actually have agent on public cloud + return nil + } fqdn := rootLBName //fqdn is that of the machine/kvm-instance running the agent if !valid.IsDNSName(fqdn) { diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index 020cbe207..c6db82ecd 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -108,10 +108,12 @@ func InitializeCloudletInfra(fakecloudlet bool) error { return fmt.Errorf("Env variable MEX_AZURE_LOCATION not set") } - CloudletInfra.OpenstackProperties.OSImageName = os.Getenv("MEX_OS_IMAGE") - if CloudletInfra.OpenstackProperties.OSImageName == "" { - CloudletInfra.OpenstackProperties.OSImageName = "mobiledgex" + /** resource group currently derived from cloudletname + cluster name + CloudletInfra.AzureProperties.ResourceGroup = os.Getenv("MEX_AZURE_RESOURCE_GROUP") + if CloudletInfra.AzureProperties.ResourceGroup == "" { + return fmt.Errorf("Env variable MEX_AZURE_RESOURCE_GROUP not set") } + */ case cloudcommon.CloudletKindGCP: CloudletInfra.GcpProperties.Project = os.Getenv("MEX_GCP_PROJECT") @@ -139,6 +141,10 @@ func CloudletIsLocalDIND() bool { return CloudletInfra.CloudletKind == cloudcommon.CloudletKindDIND } +func CloudletIsPublicCloud() bool { + return (CloudletInfra.CloudletKind == cloudcommon.CloudletKindAzure) || (CloudletInfra.CloudletKind == cloudcommon.CloudletKindGCP) +} + // returns true if kubectl can be run directly from the CRM rather than SSH jump thru LB func CloudletIsDirectKubectlAccess() bool { return CloudletInfra.CloudletKind == cloudcommon.CloudletKindDIND || From ae5442650e0f93c9ce9f6b2c07a439189e05559c Mon Sep 17 00:00:00 2001 From: Jim Date: Fri, 8 Feb 2019 20:34:06 -0600 Subject: [PATCH 08/10] remove code that should not be there --- mexos/cloudletinfra.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index c6db82ecd..5f1e91096 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -17,25 +17,6 @@ import ( var CloudletInfra *edgeproto.CloudletInfraProperties var CloudletInfraCommon *edgeproto.CloudletInfraCommon -//this is possible actions and optional parameters -var actionChoices = map[string]string{ - "CLOUDLET_KIND": "procname", - "stop": "procname", - "status": "procname", - "ctrlapi": "procname", - "ctrlcli": "procname", - "ctrlinfo": "procname", - "dmeapi": "procname", - "deploy": "", - "cleanup": "", - "fetchlogs": "", - "createcluster": "", - "deletecluster": "", - "gencerts": "", - "cleancerts": "", - "sleep": "seconds", -} - func InitializeCloudletInfra(fakecloudlet bool) error { log.DebugLog(log.DebugLevelMexos, "InitializeCloudletInfra called") From 6dacb373c0f4160bd495c9e5138b98864b1310bb Mon Sep 17 00:00:00 2001 From: Jim Date: Sat, 9 Feb 2019 12:50:25 -0600 Subject: [PATCH 09/10] comments per review so far. Have not restested yet --- mexos/{inst.go => appinst.go} | 51 ++++----- mexos/azure.go | 10 +- mexos/cloudletinfra.go | 19 ++++ mexos/gcloud.go | 6 +- mexos/helm.go | 36 +++---- mexos/kubeadmdind.go | 6 +- mexos/kubeconfig.go | 47 +++++---- mexos/kubectl.go | 24 ++--- mexos/kubernetes.go | 82 +++++++++------ mexos/misc.go | 5 - mexos/oscli.go | 13 +-- mexos/rootlb.go | 1 - mexos/securityrule.go | 13 +-- mexos/templates.go | 189 ---------------------------------- mexos/vault.go | 1 - 15 files changed, 169 insertions(+), 334 deletions(-) rename mexos/{inst.go => appinst.go} (74%) delete mode 100644 mexos/templates.go diff --git a/mexos/inst.go b/mexos/appinst.go similarity index 74% rename from mexos/inst.go rename to mexos/appinst.go index 67900e348..d5d1bd981 100644 --- a/mexos/inst.go +++ b/mexos/appinst.go @@ -14,11 +14,12 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, log.DebugLog(log.DebugLevelMexos, "mex create app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) appDeploymentType := app.Deployment - clusterName := clusterInst.Key.ClusterKey.Name - appName := NormalizeName(app.Key.Name) - operatorName := NormalizeName(appInst.Key.CloudletKey.OperatorKey.Name) - - //TODO values.application.template + var kubeNames KubeNames + err := GetKubeNames(clusterInst, app, appInst, &kubeNames) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "GetKubeNames failed") + return err + } if CloudletIsLocalDIND() { masteraddr := dind.GetMasterAddr() @@ -32,17 +33,17 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, if len(portDetail) > 0 { log.DebugLog(log.DebugLevelMexos, "call AddNginxProxy for dind", "ports", portDetail) - if err := AddNginxProxy("localhost", appName, masteraddr, portDetail, dind.GetDockerNetworkName(clusterName)); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", appName, "ports", portDetail) + if err := AddNginxProxy("localhost", kubeNames.appName, masteraddr, portDetail, dind.GetDockerNetworkName(kubeNames.clusterName)); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "appName", kubeNames.appName, "ports", portDetail) return err } } log.DebugLog(log.DebugLevelMexos, "call runKubectlCreateApp for dind") if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - err = runKubectlCreateApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + err = runKubectlCreateApp(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - err = CreateHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + err = CreateHelmAppInst(rootLB, &kubeNames, appInst, clusterInst, app.DeploymentManifest) } else { err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) } @@ -53,12 +54,12 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return nil } - switch operatorName { + switch kubeNames.operatorName { case cloudcommon.OperatorGCP: fallthrough case cloudcommon.OperatorAzure: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return runKubectlCreateApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + return runKubectlCreateApp(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { return fmt.Errorf("not yet supported") } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { @@ -69,10 +70,10 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return fmt.Errorf("unknown deployment type %s", appDeploymentType) default: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return CreateKubernetesAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + return CreateKubernetesAppInst(rootLB, &kubeNames, clusterInst, appInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - return CreateHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + return CreateHelmAppInst(rootLB, &kubeNames, appInst, clusterInst, app.DeploymentManifest) } //TODO -- support these later //} else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { @@ -87,17 +88,21 @@ func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, app *edgeproto.App, appInst *edgeproto.AppInst) error { log.DebugLog(log.DebugLevelMexos, "mex delete app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) appDeploymentType := app.Deployment - operatorName := NormalizeName(appInst.Key.CloudletKey.OperatorKey.Name) - appName := NormalizeName(app.Key.Name) + var kubeNames KubeNames + err := GetKubeNames(clusterInst, app, appInst, &kubeNames) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "GetKubeNames failed") + return err + } if CloudletIsLocalDIND() { log.DebugLog(log.DebugLevelMexos, "run kubectl delete app for dind") var err error if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - err = runKubectlDeleteApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + err = runKubectlDeleteApp(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - err = DeleteHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + err = DeleteHelmAppInst(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else { err = fmt.Errorf("invalid deployment type %s for dind", appDeploymentType) } @@ -112,20 +117,20 @@ func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return err } if len(portDetail) > 0 { - if err = DeleteNginxProxy("localhost", appName); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", appName) + if err = DeleteNginxProxy("localhost", kubeNames.appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", kubeNames.appName) return err } } return nil } - switch operatorName { + switch kubeNames.operatorName { case cloudcommon.OperatorGCP: fallthrough case cloudcommon.OperatorAzure: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return runKubectlDeleteApp(clusterInst, appInst, rootLB, app.DeploymentManifest) + return runKubectlDeleteApp(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { return fmt.Errorf("not yet supported") } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { @@ -136,9 +141,9 @@ func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, return fmt.Errorf("unknown image type %s", appDeploymentType) default: if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { - return DeleteKubernetesAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + return DeleteKubernetesAppInst(rootLB, &kubeNames, clusterInst) } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { - return DeleteHelmAppInst(rootLB, clusterInst, app.DeploymentManifest, app, appInst) + return DeleteHelmAppInst(rootLB, &kubeNames, clusterInst, app.DeploymentManifest) } //TODO //} else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { diff --git a/mexos/azure.go b/mexos/azure.go index f167df70c..57ec54ad1 100644 --- a/mexos/azure.go +++ b/mexos/azure.go @@ -9,6 +9,10 @@ import ( "github.com/mobiledgex/edge-cloud/log" ) +func GetResourceGroupForCluster(clusterInst *edgeproto.ClusterInst) string { + return clusterInst.Key.CloudletKey.Name + "_" + clusterInst.Key.ClusterKey.Name +} + func azureCreateAKS(clusterInst *edgeproto.ClusterInst) error { var err error resourceGroup := GetResourceGroupForCluster(clusterInst) @@ -26,10 +30,8 @@ func azureCreateAKS(clusterInst *edgeproto.ClusterInst) error { if err = azure.GetAKSCredentials(resourceGroup, clusterName); err != nil { return err } - kconf, err := GetKconf(clusterInst, false) // XXX - if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v, %v", clusterInst, kconf, err) - } + kconf := GetKconfName(clusterInst) // XXX + log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX //XXX watch out for multiple cluster contexts if err = copyFile(defaultKubeconfig(), kconf); err != nil { diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index 5f1e91096..c6db82ecd 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -17,6 +17,25 @@ import ( var CloudletInfra *edgeproto.CloudletInfraProperties var CloudletInfraCommon *edgeproto.CloudletInfraCommon +//this is possible actions and optional parameters +var actionChoices = map[string]string{ + "CLOUDLET_KIND": "procname", + "stop": "procname", + "status": "procname", + "ctrlapi": "procname", + "ctrlcli": "procname", + "ctrlinfo": "procname", + "dmeapi": "procname", + "deploy": "", + "cleanup": "", + "fetchlogs": "", + "createcluster": "", + "deletecluster": "", + "gencerts": "", + "cleancerts": "", + "sleep": "seconds", +} + func InitializeCloudletInfra(fakecloudlet bool) error { log.DebugLog(log.DebugLevelMexos, "InitializeCloudletInfra called") diff --git a/mexos/gcloud.go b/mexos/gcloud.go index 7e092e56c..31299d9bb 100644 --- a/mexos/gcloud.go +++ b/mexos/gcloud.go @@ -30,10 +30,8 @@ func gcloudCreateGKE(clusterInst *edgeproto.ClusterInst) error { if err = gcloud.GetGKECredentials(clusterName); err != nil { return err } - kconf, err := GetKconf(clusterInst, false) //XXX - if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v", clusterInst, err) - } + kconf := GetKconfName(clusterInst) //XXX + log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX if err = copyFile(defaultKubeconfig(), kconf); err != nil { return fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) diff --git a/mexos/helm.go b/mexos/helm.go index 2adf73cf5..6c4fb1e88 100644 --- a/mexos/helm.go +++ b/mexos/helm.go @@ -8,37 +8,34 @@ import ( "github.com/mobiledgex/edge-cloud/log" ) -func DeleteHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { +func DeleteHelmAppInst(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst, kubeManifest string) error { log.DebugLog(log.DebugLevelMexos, "delete kubernetes helm app") - appName := NormalizeName(app.Key.Name) - clusterName := clusterInst.Key.ClusterKey.Name - var err error if rootLB == nil { return fmt.Errorf("cannot delete helm app, rootLB is null") } - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } if CloudletIsLocalDIND() { // remove DNS entries - if err = deleteAppDNS(kp, appInst.Uri, appName); err != nil { + if err = deleteAppDNS(kp, kubeNames.appURI, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) } } else { // remove DNS entries - if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + if err = KubeDeleteDNSRecords(rootLB, kp, kubeNames.appURI, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) } // remove Security rules - if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) } } - cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, appName) + cmd := fmt.Sprintf("%s helm delete --purge %s", kp.kubeconfig, kubeNames.appName) out, err := kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deleting helm chart, %s, %s, %v", cmd, out, err) @@ -47,18 +44,15 @@ func DeleteHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku return nil } -func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { - log.DebugLog(log.DebugLevelMexos, "create kubernetes helm app", "clusterInst", clusterInst, "appInst", appInst) +func CreateHelmAppInst(rootLB *MEXRootLB, kubeNames *KubeNames, appInst *edgeproto.AppInst, clusterInst *edgeproto.ClusterInst, kubeManifest string) error { + log.DebugLog(log.DebugLevelMexos, "create kubernetes helm app", "clusterInst", clusterInst, "kubeNames", kubeNames) - clusterName := clusterInst.Key.ClusterKey.Name - appName := NormalizeName(app.Key.Name) - appImage := NormalizeName(app.ImagePath) var err error if rootLB == nil { return fmt.Errorf("cannot create helm app, rootLB is null") } - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } @@ -96,12 +90,12 @@ func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku // XXX This gets helm's prometheus able to query kubelet metrics. // This can be removed once Lev passes in an option in the yaml to // set the helm command line options. - prom, err := regexp.MatchString("prometheus", appName) + prom, err := regexp.MatchString("prometheus", kubeNames.appName) if err == nil && prom { log.DebugLog(log.DebugLevelMexos, "setting helm prometheus option") helmOpts = "--set kubelet.serviceMonitor.https=true" } - cmd = fmt.Sprintf("%s helm install %s --name %s %s", kp.kubeconfig, appImage, appName, helmOpts) + cmd = fmt.Sprintf("%s helm install %s --name %s %s", kp.kubeconfig, kubeNames.appImage, kubeNames.appName, helmOpts) out, err = kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deploying helm chart, %s, %s, %v", cmd, out, err) @@ -109,19 +103,19 @@ func CreateHelmAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, ku log.DebugLog(log.DebugLevelMexos, "applied helm chart") if CloudletIsLocalDIND() { // Add DNS Zone - if err = createAppDNS(kp, appInst.Uri, appName); err != nil { + if err = createAppDNS(kp, kubeNames.appURI, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) return err } } else { // Add security rules - if err = AddProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + if err = AddProxySecurityRules(rootLB, kp.ipaddr, kubeNames.appName, appInst); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) return err } - log.DebugLog(log.DebugLevelMexos, "done AddProxySecurityRules", "app", app) + log.DebugLog(log.DebugLevelMexos, "done AddProxySecurityRules", "kubeNames.appName", kubeNames.appName) // Add DNS Zone - if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) return err } diff --git a/mexos/kubeadmdind.go b/mexos/kubeadmdind.go index 71ea85f62..2c117021f 100644 --- a/mexos/kubeadmdind.go +++ b/mexos/kubeadmdind.go @@ -22,10 +22,8 @@ func localCreateDIND(clusterInst *edgeproto.ClusterInst) error { //race condition exists where the config file is not ready until just after the cluster create is done time.Sleep(3 * time.Second) - kconf, err := GetKconf(clusterInst, false) // XXX - if err != nil { - return fmt.Errorf("cannot get kconf, %v, %v", kconf, err) - } + kconf := GetKconfName(clusterInst) // XXX + log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX //XXX watch out for multiple cluster contexts if err = copyFile(defaultKubeconfig(), kconf); err != nil { diff --git a/mexos/kubeconfig.go b/mexos/kubeconfig.go index 56148900a..5b69a7d1c 100644 --- a/mexos/kubeconfig.go +++ b/mexos/kubeconfig.go @@ -23,38 +23,37 @@ func GetKconfName(clusterInst *edgeproto.ClusterInst) string { GetCloudletDNSZone()) } -func GetKconf(clusterInst *edgeproto.ClusterInst, createIfMissing bool) (string, error) { +func GetKconf(clusterInst *edgeproto.ClusterInst) (string, error) { name := GetLocalKconfName(clusterInst) operatorName := clusterInst.Key.CloudletKey.OperatorKey.Name clusterName := clusterInst.Key.ClusterKey.Name log.DebugLog(log.DebugLevelMexos, "get kubeconfig name", "name", name) - if createIfMissing { // XXX - if _, err := os.Stat(name); os.IsNotExist(err) { - // if kubeconfig does not exist, optionally create it. It is possible it was - // created on a different container or we had a restart of the container - log.DebugLog(log.DebugLevelMexos, "creating missing kconf file", "name", name) - switch operatorName { - case cloudcommon.OperatorGCP: - if err = gcloud.GetGKECredentials(clusterName); err != nil { - return "", fmt.Errorf("unable to get GKE credentials %v", err) - } - if err = copyFile(defaultKubeconfig(), name); err != nil { - return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) - } - case cloudcommon.OperatorAzure: - rg := GetResourceGroupForCluster(clusterInst) - if err = azure.GetAKSCredentials(rg, clusterName); err != nil { - return "", fmt.Errorf("unable to get AKS credentials %v", err) - } - if err = copyFile(defaultKubeconfig(), name); err != nil { - return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) - } - default: - log.DebugLog(log.DebugLevelMexos, "warning, not creating missing kubeconfig for operator", "operator", operatorName) + if _, err := os.Stat(name); os.IsNotExist(err) { + // if kubeconfig does not exist, optionally create it. It is possible it was + // created on a different container or we had a restart of the container + log.DebugLog(log.DebugLevelMexos, "creating missing kconf file", "name", name) + switch operatorName { + case cloudcommon.OperatorGCP: + if err = gcloud.GetGKECredentials(clusterName); err != nil { + return "", fmt.Errorf("unable to get GKE credentials %v", err) } + if err = copyFile(defaultKubeconfig(), name); err != nil { + return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) + } + case cloudcommon.OperatorAzure: + rg := GetResourceGroupForCluster(clusterInst) + if err = azure.GetAKSCredentials(rg, clusterName); err != nil { + return "", fmt.Errorf("unable to get AKS credentials %v", err) + } + if err = copyFile(defaultKubeconfig(), name); err != nil { + return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) + } + default: + log.DebugLog(log.DebugLevelMexos, "warning, not creating missing kubeconfig for operator", "operator", operatorName) } } + return name, nil } diff --git a/mexos/kubectl.go b/mexos/kubectl.go index f0e2abfe5..52781f702 100644 --- a/mexos/kubectl.go +++ b/mexos/kubectl.go @@ -46,19 +46,17 @@ func CreateDockerRegistrySecret(clusterInst *edgeproto.ClusterInst, rootLBName s return nil } -func runKubectlCreateApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, rootLB *MEXRootLB, kubeManifest string) error { +func runKubectlCreateApp(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst, kubeManifest string) error { log.DebugLog(log.DebugLevelMexos, "run kubectl create app", "kubeManifest", kubeManifest) - appName := NormalizeName(appInst.Key.AppKey.Name) - clusterName := clusterInst.Key.ClusterKey.Name - kfile := appName + ".yaml" + kfile := kubeNames.appName + ".yaml" if err := writeKubeManifest(kubeManifest, kfile); err != nil { return err } defer os.Remove(kfile) - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } @@ -72,11 +70,11 @@ func runKubectlCreateApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto. cmd = fmt.Sprintf("%s kubectl delete -f %s", kp.kubeconfig, kfile) out, undoerr := kp.client.Output(cmd) if undoerr != nil { - log.DebugLog(log.DebugLevelMexos, "undo kubectl create app failed", "name", appName, "out", out, "err", undoerr) + log.DebugLog(log.DebugLevelMexos, "undo kubectl create app failed", "kubeNames", kubeNames, "out", out, "err", undoerr) } } }() - err = createAppDNS(kp, appInst.Uri, appName) + err = createAppDNS(kp, kubeNames.appURI, kubeNames.appName) if err != nil { return fmt.Errorf("error creating dns entry for app, %v", err) } @@ -136,15 +134,13 @@ func getServices(kp *kubeParam) ([]v1.Service, error) { return svcs.Items, nil } -func runKubectlDeleteApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, rootLB *MEXRootLB, kubeManifest string) error { - appName := NormalizeName(appInst.Key.AppKey.Name) - clusterName := clusterInst.Key.ClusterKey.Name +func runKubectlDeleteApp(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst, kubeManifest string) error { - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } - kfile := appName + ".yaml" + kfile := kubeNames.appName + ".yaml" err = writeKubeManifest(kubeManifest, kfile) if err != nil { return err @@ -155,9 +151,9 @@ func runKubectlDeleteApp(clusterInst *edgeproto.ClusterInst, appInst *edgeproto. if err != nil { return fmt.Errorf("error deleting app, %s, %v", out, err) } - err = deleteAppDNS(kp, appInst.Uri, appName) + err = deleteAppDNS(kp, kubeNames.appURI, kubeNames.appName) if err != nil { - return fmt.Errorf("error deleting dns entry for app, %v, %v", appInst, err) + return fmt.Errorf("error deleting dns entry for app, %v, %v", kubeNames.appName, err) } return nil } diff --git a/mexos/kubernetes.go b/mexos/kubernetes.go index 7d74aa298..8f49d3215 100644 --- a/mexos/kubernetes.go +++ b/mexos/kubernetes.go @@ -9,21 +9,47 @@ import ( ssh "github.com/nanobox-io/golang-ssh" ) +type KubeNames struct { + appName string + appURI string + appImage string + clusterName string + operatorName string + kconfName string +} + +// GetKubeNames udpates kubeNames with normalized strings for the included clusterinst, app, and appisnt +func GetKubeNames(clusterInst *edgeproto.ClusterInst, app *edgeproto.App, appInst *edgeproto.AppInst, kubeNames *KubeNames) (err error) { + if clusterInst == nil { + return fmt.Errorf("nil cluster inst") + } + if app == nil { + return fmt.Errorf("nil app") + } + if appInst == nil { + return fmt.Errorf("nil app inst") + } + kubeNames.clusterName = clusterInst.Key.ClusterKey.Name + kubeNames.appName = NormalizeName(app.Key.Name) + kubeNames.appURI = appInst.Uri + kubeNames.appImage = NormalizeName(app.ImagePath) + kubeNames.operatorName = NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name) + kubeNames.kconfName = GetKconfName(clusterInst) + return nil + +} + //CreateKubernetesAppInst instantiates a new kubernetes deployment -func CreateKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { +func CreateKubernetesAppInst(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, kubeManifest string) error { log.DebugLog(log.DebugLevelMexos, "create kubernetes app") - clusterName := clusterInst.Key.ClusterKey.Name - appName := NormalizeName(app.Key.Name) - if rootLB == nil { return fmt.Errorf("cannot create kubernetes app manifest, rootLB is null") } - if appInst.Uri == "" { return fmt.Errorf("empty app URI") } - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } @@ -36,26 +62,26 @@ func CreateKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterIn // return err //} //TODO do not create yaml file but use remote yaml file over https - cmd = fmt.Sprintf("cat <<'EOF'> %s.yaml \n%s\nEOF", appName, kubeManifest) + cmd = fmt.Sprintf("cat <<'EOF'> %s.yaml \n%s\nEOF", kubeNames.appName, kubeManifest) out, err := kp.client.Output(cmd) if err != nil { return fmt.Errorf("error writing KubeManifest, %s, %s, %v", cmd, out, err) } log.DebugLog(log.DebugLevelMexos, "wrote kubernetes manifest file") - cmd = fmt.Sprintf("%s kubectl create -f %s.yaml", kp.kubeconfig, appName) + cmd = fmt.Sprintf("%s kubectl create -f %s.yaml", kp.kubeconfig, kubeNames.appName) out, err = kp.client.Output(cmd) if err != nil { return fmt.Errorf("error deploying kubernetes app, %s, %v", out, err) } log.DebugLog(log.DebugLevelMexos, "applied kubernetes manifest") // Add security rules - if err = AddProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { + if err = AddProxySecurityRules(rootLB, kp.ipaddr, kubeNames.appName, appInst); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) return err } log.DebugLog(log.DebugLevelMexos, "ok, added ports", "ports", appInst.MappedPorts) // Add DNS Zone - if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { + if err = KubeAddDNSRecords(rootLB, kp, appInst.Uri, kubeNames.appName); err != nil { log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) return err } @@ -69,16 +95,15 @@ type kubeParam struct { } //ValidateKubernetesParameters checks the kubernetes parameters and kubeconfig settings -func ValidateKubernetesParameters(clusterInst *edgeproto.ClusterInst, rootLB *MEXRootLB, clustName string) (*kubeParam, error) { - log.DebugLog(log.DebugLevelMexos, "validate kubernetes parameters rootLB", "cluster", clustName) - clusterName := clusterInst.Key.ClusterKey.Name +func ValidateKubernetesParameters(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst) (*kubeParam, error) { + log.DebugLog(log.DebugLevelMexos, "validate kubernetes parameters", "kubeNames", kubeNames) if CloudletIsDirectKubectlAccess() { // No ssh jump host (rootlb) but kconf configures how to // talk to remote kubernetes cluster. This includes DIND, AKS, GCP - kconf, err := GetKconf(clusterInst, true) + kconf, err := GetKconf(clusterInst) if err != nil { - return nil, fmt.Errorf("kconf missing, %v, %v", clustName, err) + return nil, fmt.Errorf("kconf missing, %v, %v", kubeNames.clusterName, err) } kp := kubeParam{ kubeconfig: fmt.Sprintf("KUBECONFIG=%s", kconf), @@ -96,47 +121,44 @@ func ValidateKubernetesParameters(clusterInst *edgeproto.ClusterInst, rootLB *ME if err != nil { return nil, err } - master, err := FindClusterMaster(clusterName) + master, err := FindClusterMaster(kubeNames.clusterName) if err != nil { - return nil, fmt.Errorf("can't find cluster with key %s, %v", clustName, err) + return nil, fmt.Errorf("can't find cluster with key %s, %v", kubeNames.clusterName, err) } ipaddr, err := FindNodeIP(master) if err != nil { return nil, err } //kubeconfig := fmt.Sprintf("KUBECONFIG=%s.kubeconfig", name[strings.LastIndex(name, "-")+1:]) - kubeconfig := fmt.Sprintf("KUBECONFIG=%s", GetKconfName(clusterInst)) + kubeconfig := fmt.Sprintf("KUBECONFIG=%s", kubeNames.kconfName) return &kubeParam{kubeconfig, client, ipaddr}, nil } -func DeleteKubernetesAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, kubeManifest string, app *edgeproto.App, appInst *edgeproto.AppInst) error { +func DeleteKubernetesAppInst(rootLB *MEXRootLB, kubeNames *KubeNames, clusterInst *edgeproto.ClusterInst) error { log.DebugLog(log.DebugLevelMexos, "delete kubernetes app") - clusterName := clusterInst.Key.ClusterKey.Name - appName := NormalizeName(app.Key.Name) - - kp, err := ValidateKubernetesParameters(clusterInst, rootLB, clusterName) + kp, err := ValidateKubernetesParameters(rootLB, kubeNames, clusterInst) if err != nil { return err } // Clean up security rules and nginx proxy - if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, appInst); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot clean up security rules", "name", appName, "rootlb", rootLB.Name, "error", err) + if err = DeleteProxySecurityRules(rootLB, kp.ipaddr, kubeNames.appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot clean up security rules", "name", kubeNames.appName, "rootlb", rootLB.Name, "error", err) } if err := cloudflare.InitAPI(GetCloudletCFUser(), GetCloudletCFKey()); err != nil { return fmt.Errorf("cannot init cloudflare api, %v", err) } // Clean up DNS entries - if err = KubeDeleteDNSRecords(rootLB, kp, appInst.Uri, appName); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot clean up DNS entries", "name", appName, "rootlb", rootLB.Name, "error", err) + if err = KubeDeleteDNSRecords(rootLB, kp, kubeNames.appURI, kubeNames.appName); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot clean up DNS entries", "name", kubeNames.appName, "rootlb", rootLB.Name, "error", err) return err } - cmd := fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, appName) + cmd := fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, kubeNames.appName) out, err := kp.client.Output(cmd) if err != nil { - return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", appName, cmd, out, err) + return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", kubeNames.appName, cmd, out, err) } - log.DebugLog(log.DebugLevelMexos, "deleted deployment", "name", appName) + log.DebugLog(log.DebugLevelMexos, "deleted deployment", "name", kubeNames.appName) return nil } diff --git a/mexos/misc.go b/mexos/misc.go index 3559d39da..bac1ba15e 100644 --- a/mexos/misc.go +++ b/mexos/misc.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "os" - "github.com/mobiledgex/edge-cloud/edgeproto" "github.com/mobiledgex/edge-cloud/log" "github.com/mobiledgex/edge-cloud/util" ) @@ -108,7 +107,3 @@ func GetHTPassword(rootLBName string) error { log.DebugLog(log.DebugLevelMexos, "downloaded htpasswd") return nil } - -func GetResourceGroupForCluster(clusterInst *edgeproto.ClusterInst) string { - return clusterInst.Key.CloudletKey.Name + "_" + clusterInst.Key.ClusterKey.Name -} diff --git a/mexos/oscli.go b/mexos/oscli.go index 9de3fbc4f..93e7ed6c1 100644 --- a/mexos/oscli.go +++ b/mexos/oscli.go @@ -46,8 +46,9 @@ func ListImages() ([]OSImage, error) { //ListNetworks lists networks known to the platform. Some created by the operator, some by users. func ListNetworks() ([]OSNetwork, error) { - out, err := sh.Command("openstack", "network", "list", "-f", "json").Output() + out, err := sh.Command("openstack", "network", "list", "-f", "json").CombinedOutput() if err != nil { + log.DebugLog(log.DebugLevelMexos, "network list failed", "out", out) err = fmt.Errorf("cannot get network list, %v", err) return nil, err } @@ -168,7 +169,7 @@ func CreateNetwork(name string) error { //DeleteNetwork destroys a named network // Sometimes it will fail. Openstack will refuse if there are resources attached. -func DeleteNetwork( name string) error { +func DeleteNetwork(name string) error { log.DebugLog(log.DebugLevelMexos, "deleting network", "network", name) out, err := sh.Command("openstack", "network", "delete", name).CombinedOutput() if err != nil { @@ -242,7 +243,7 @@ func CreateRouter(routerName string) error { } //DeleteRouter removes the named router. The router needs to not be in use at the time of deletion. -func DeleteRouter( routerName string) error { +func DeleteRouter(routerName string) error { log.DebugLog(log.DebugLevelMexos, "deleting router", "name", routerName) out, err := sh.Command("openstack", "router", "delete", routerName).Output() if err != nil { @@ -358,7 +359,7 @@ func CreateServerImage(serverName, imageName string) error { } //CreateImage puts images into glance -func CreateImage( imageName, qcowFile string) error { +func CreateImage(imageName, qcowFile string) error { log.DebugLog(log.DebugLevelMexos, "creating image in glance", "image", imageName, "qcow", qcowFile) out, err := sh.Command("openstack", "image", "create", imageName, @@ -376,7 +377,7 @@ func CreateImage( imageName, qcowFile string) error { // It will then save that into a local file. The image transfer happens from glance into your own laptop // or whatever. // This can take a while, transferring all the data. -func SaveImage( saveName, imageName string) error { +func SaveImage(saveName, imageName string) error { log.DebugLog(log.DebugLevelMexos, "saving image", "save name", saveName, "image name", imageName) out, err := sh.Command("openstack", "image", "save", "--file", saveName, imageName).Output() if err != nil { @@ -389,7 +390,7 @@ func SaveImage( saveName, imageName string) error { //DeleteImage deletes the named image from glance. Sometimes backing store is still busy and // will refuse to honor the request. Like most things in Openstack, wait for a while and try // again. -func DeleteImage( imageName string) error { +func DeleteImage(imageName string) error { log.DebugLog(log.DebugLevelMexos, "deleting image", "name", imageName) out, err := sh.Command("openstack", "image", "delete", imageName).Output() if err != nil { diff --git a/mexos/rootlb.go b/mexos/rootlb.go index 3411a2359..2df7d9a5c 100644 --- a/mexos/rootlb.go +++ b/mexos/rootlb.go @@ -11,7 +11,6 @@ import ( //MEXRootLB has rootLB data type MEXRootLB struct { Name string - // PlatConf *Manifest } var MEXRootLBMap = make(map[string]*MEXRootLB) diff --git a/mexos/securityrule.go b/mexos/securityrule.go index 443a21e72..acf24f7b2 100644 --- a/mexos/securityrule.go +++ b/mexos/securityrule.go @@ -12,9 +12,7 @@ import ( // TODO service to periodically clean up the leftover rules -func AddProxySecurityRules(rootLB *MEXRootLB, masteraddr string, appInst *edgeproto.AppInst) error { - - name := NormalizeName(appInst.Key.AppKey.Name) +func AddProxySecurityRules(rootLB *MEXRootLB, masteraddr string, appName string, appInst *edgeproto.AppInst) error { ports, err := GetPortDetail(appInst) if err != nil { @@ -43,17 +41,16 @@ func AddProxySecurityRules(rootLB *MEXRootLB, masteraddr string, appInst *edgepr } } if len(ports) > 0 { - if err := AddNginxProxy(rootLB.Name, name, masteraddr, ports, ""); err != nil { - log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", name) + if err := AddNginxProxy(rootLB.Name, appName, masteraddr, ports, ""); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "appName", appName) return err } } - log.DebugLog(log.DebugLevelMexos, "added nginx proxy", "name", name, "ports", appInst.MappedPorts) + log.DebugLog(log.DebugLevelMexos, "added nginx proxy", "appName", appName, "ports", appInst.MappedPorts) return nil } -func DeleteProxySecurityRules(rootLB *MEXRootLB, ipaddr string, appInst *edgeproto.AppInst) error { - appName := NormalizeName(appInst.Key.AppKey.Name) +func DeleteProxySecurityRules(rootLB *MEXRootLB, ipaddr string, appName string) error { log.DebugLog(log.DebugLevelMexos, "delete proxy rules", "name", appName) err := DeleteNginxProxy(rootLB.Name, appName) diff --git a/mexos/templates.go b/mexos/templates.go deleted file mode 100644 index 904e37b01..000000000 --- a/mexos/templates.go +++ /dev/null @@ -1,189 +0,0 @@ -package mexos - -/* -type templateFill struct { - Name, Kind, Flavor, Tags, Tenant, DNSZone string - ImageFlavor, RootLB, Resource, ResourceKind, ResourceGroup string - StorageSpec, NetworkScheme string - NodeFlavor, Operator, Key, Image, Options string - ImageType, ProxyPath string - ExternalNetwork, Project string - ExternalRouter, Flags, IPAccess, Swarm string - NumMasters, NumNodes int - Config templateConfig - Command []string - SpecPorts []PortDetail -} - -type templateConfig struct { - Base, Overlay, Deployment, Resources, Manifest, Template string -} - -var yamlMEXCluster = `apiVersion: v1 -kind: {{.ResourceKind}} -resource: {{.Resource}} -metadata: - name: {{.Name}} - tags: {{.Tags}} - kind: {{.Kind}} - tenant: {{.Tenant}} - operator: {{.Operator}} - project: {{.Project}} - dnszone: {{.DNSZone}} - swarm: {{.Swarm}} - resourcegroup: {{.ResourceGroup}} -spec: - flags: {{.Flags}} - flavor: {{.Flavor}} - key: {{.Key}} - dockerregistry: registry.mobiledgex.net:5000 - rootlb: {{.RootLB}} - networkscheme: {{.NetworkScheme}} -` - -var yamlMEXFlavor = `apiVersion: v1 -kind: {{.ResourceKind}} -resource: {{.Resource}} -metadata: - name: {{.Name}} - tags: {{.Tags}} - kind: {{.Kind}} -spec: - flags: {{.Flags}} - flavor: {{.Name}} - flavordetail: - name: {{.Name}} - nodes: {{.NumNodes}} - masters: {{.NumMasters}} - networkscheme: {{.NetworkScheme}} - masterflavor: {{.MasterFlavor}} - nodeflavor: {{.NodeFlavor}} - storagescheme: {{.StorageSpec}} - topology: {{.Topology}} -` - -var yamlMEXPlatform = `apiVersion: v1 -kind: {{.ResourceKind}} -resource: {{.Resource}} -metadata: - kind: {{.Kind}} - name: {{.Name}} - tags: {{.Tags}} - tenant: {{.Tenant}} - openrc: ~/.mobiledgex/openrc - dnszone: {{.DNSZone}} - operator: {{.Operator}} -spec: - flags: {{.Flags}} - flavor: {{.Flavor}} - rootlb: {{.RootLB}} - key: {{.Key}} - dockerregistry: registry.mobiledgex.net:5000 - externalnetwork: {{.ExternalNetwork}} - networkscheme: {{.NetworkScheme}} - externalrouter: {{.ExternalRouter}} - options: {{.Options}} -` - -var yamlMEXApp = `apiVersion: v1 -kind: {{.ResourceKind}} -resource: {{.Resource}} -metadata: - kind: {{.Kind}} - name: {{.Name}} - tags: {{.Tags}} - tenant: {{.Tenant}} - operator: {{.Operator}} - dnszone: {{.DNSZone}} -config: - kind: - source: - detail: - resources: {{.Config.Resources}} - deployment: {{.Config.Deployment}} - manifest: {{.Config.Manifest}} - template: {{.Config.Template}} - base: {{.Config.Base}} - overlay: {{.Config.Overlay}} -spec: - flags: {{.Flags}} - key: {{.Key}} - rootlb: {{.RootLB}} - image: {{.Image}} - imagetype: {{.ImageType}} - imageflavor: {{.ImageFlavor}} - proxypath: {{.ProxyPath}} - flavor: {{.Flavor}} - ipaccess: {{.IPAccess}} - networkscheme: {{.NetworkScheme}} - ports: -{{- range .SpecPorts}} - - {{.Name}} - {{.MexProto}} - {{.Proto}} - {{.InternalPort}} - {{.PublicPort}} - {{.PublicPath}} -{{- end}} - command: -{{- range .Command}} - - {{.}} -{{- end}} -` - -func fillPlatformTemplateCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) (*Manifest, error) { - log.DebugLog(log.DebugLevelMexos, "fill template cloudletkeystr", "cloudletkeystr", cloudletKeyStr) - - - log.DebugLog(log.DebugLevelMexos, "using external network", "extNet", GetCloudletExternalNetwork()) - meximage := os.Getenv("MEX_OS_IMAGE") - if meximage == "" { - return nil, fmt.Errorf("Env variable MEX_OS_IMAGE not set") - } - - data := templateFill{ - ResourceKind: "platform", - Resource: NormalizeName(clk.OperatorKey.Name), - Name: clk.Name, - Tags: clk.Name + "-tag", - Key: clk.Name + "-" + NormalizeName(clk.OperatorKey.Name), - Flavor: "x1.medium", - Operator: NormalizeName(clk.OperatorKey.Name), - RootLB: rootLB.Name, - Kind: "mex-platform", - ExternalNetwork: GetCloudletExternalNetwork(), - NetworkScheme: GetCloudletNetworkScheme(), - DNSZone: GetCloudletDNSZone(), - ExternalRouter: GetCloudletExternalRouter(), - Options: "dhcp", - } - mf, err := templateUnmarshal(&data, yamlMEXPlatform) - if err != nil { - return nil, err - } - return mf, nil -} - - -func templateUnmarshal(data *templateFill, yamltext string) (*Manifest, error) { - //log.DebugLog(log.DebugLevelMexos, "template unmarshal", "yamltext", string, "data", data) - tmpl, err := template.New("mex").Parse(yamltext) - if err != nil { - return nil, fmt.Errorf("can't create template for, %v", err) - } - var outbuffer bytes.Buffer - err = tmpl.Execute(&outbuffer, data) - if err != nil { - //log.DebugLog(log.DebugLevelMexos, "template data", "data", data) - return nil, fmt.Errorf("can't execute template, %v", err) - } - mf := &Manifest{} - err = yaml.Unmarshal(outbuffer.Bytes(), mf) - if err != nil { - log.DebugLog(log.DebugLevelMexos, "error yaml unmarshal, templated data") - return nil, fmt.Errorf("can't unmarshal templated data, %v, %s", err, outbuffer.String()) - } - return mf, nil -} - -*/ diff --git a/mexos/vault.go b/mexos/vault.go index 1c6978931..1bbf2ed66 100644 --- a/mexos/vault.go +++ b/mexos/vault.go @@ -148,7 +148,6 @@ func CheckPlatformEnv(platformType string) error { } func GetVaultEnv(openrc string, mexenv string, cloudletInfra *edgeproto.CloudletInfraProperties) error { - if err := InternVaultEnv(openrc, mexenv, cloudletInfra); err != nil { return err } From a1b5c326eb2f248df9df21f840464f4f057a5e39 Mon Sep 17 00:00:00 2001 From: Jim Date: Sat, 9 Feb 2019 12:52:09 -0600 Subject: [PATCH 10/10] remove code that got included accidentally again. --- mexos/cloudletinfra.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/mexos/cloudletinfra.go b/mexos/cloudletinfra.go index c6db82ecd..6c3fefad8 100644 --- a/mexos/cloudletinfra.go +++ b/mexos/cloudletinfra.go @@ -17,24 +17,6 @@ import ( var CloudletInfra *edgeproto.CloudletInfraProperties var CloudletInfraCommon *edgeproto.CloudletInfraCommon -//this is possible actions and optional parameters -var actionChoices = map[string]string{ - "CLOUDLET_KIND": "procname", - "stop": "procname", - "status": "procname", - "ctrlapi": "procname", - "ctrlcli": "procname", - "ctrlinfo": "procname", - "dmeapi": "procname", - "deploy": "", - "cleanup": "", - "fetchlogs": "", - "createcluster": "", - "deletecluster": "", - "gencerts": "", - "cleancerts": "", - "sleep": "seconds", -} func InitializeCloudletInfra(fakecloudlet bool) error { log.DebugLog(log.DebugLevelMexos, "InitializeCloudletInfra called")