From 84f2a8bb30e613a825dbc855676278399a0b8e34 Mon Sep 17 00:00:00 2001 From: bobmex <38759636+bobmex@users.noreply.github.com> Date: Fri, 28 Dec 2018 21:07:23 -0800 Subject: [PATCH] infra changes very big (#20) * new values * new values * new values * vault json * readme * major refactor * readme * refactor * working again * revised working versions * kustomize * broken network * still broken network but with update and holepunch * do update after network init * add del security rule. fix k8s network. * use ubuntu user not root. lock down sshd config to prevent root and password. * working with kubectl delete -f * working docker compose test * gcp and azure * working docker swarm * more examples --- .gitignore | 46 ++ .../examples/application/example/example.yaml | 62 ++ .../examples/cluster/azure/example.yaml | 22 + deployments/examples/cluster/gcp/example.yaml | 23 + .../docker-swarm/stackdemo/docker-compose.yml | 10 + .../docker-swarm/stackdemo/kustomization.yaml | 7 + .../base/kubernetes/testapp/app.yaml | 78 ++ .../kubernetes/testapp/kustomization.yaml | 13 + .../examples/kustomize/application/build.sh | 8 + .../application/output/mytest-app.yaml | 78 ++ .../application/output/stackdemo.yaml | 10 + .../overlays/mytest-app/deploy.yaml | 12 + .../overlays/mytest-app/kustomization.yaml | 15 + .../application/overlays/mytest-app/tcp.yaml | 9 + .../application/overlays/mytest-app/udp.yaml | 9 + .../overlays/stackdemo/kustomization.yaml | 8 + .../overlays/stackdemo/stackdemo.yaml | 4 + .../base/azure/kustomization.yaml | 13 + .../infrastructure/base/azure/values.yaml | 55 ++ .../base/gcp/kustomization.yaml | 13 + .../infrastructure/base/gcp/values.yaml | 55 ++ .../kustomize/infrastructure/build.sh | 5 + .../output/centralus-test.azure.yaml | 54 ++ .../output/us-central1-test.gcp.yaml | 55 ++ .../centralus-test.azure/kustomization.yaml | 13 + .../overlays/centralus-test.azure/values.yaml | 20 + .../us-central1-test.gcp/kustomization.yaml | 13 + .../overlays/us-central1-test.gcp/values.yaml | 22 + .../examples/platform/azure/example.yaml | 28 + .../examples/platform/gcp/example.yaml | 28 + examples/docker-swarm/stackdemo/Dockerfile | 5 + examples/docker-swarm/stackdemo/app.py | 15 + .../docker-swarm/stackdemo/docker-compose.yml | 10 + .../docker-swarm/stackdemo/requirements.txt | 3 + examples/mexexample/Dockerfile | 17 + examples/mexexample/Makefile | 63 ++ examples/mexexample/api/mexexample.pb.go | 658 +++++++++++++++ examples/mexexample/api/mexexample.pb.gw.go | 258 ++++++ examples/mexexample/api/mexexample.proto | 79 ++ .../mexexample/api/mexexample.swagger.json | 230 ++++++ examples/mexexample/docker-build.sh | 6 + examples/mexexample/mexexample.go | 94 +++ examples/mexexample/mexexample.yaml | 76 ++ examples/mexexample/server/handlers.go | 95 +++ examples/mexexample/server/server.go | 128 +++ k8s-prov/install-k8s-base.sh | 89 +- k8s-prov/install-k8s-master.sh | 78 +- k8s-prov/install-k8s-node.sh | 16 +- mexctl/.gitignore | 46 ++ mexctl/main.go | 146 ++++ mexctl/mexctl-all.sh | 25 + mexos/.gitignore | 46 ++ mexos/agent.go | 222 +++++ mexos/azure.go | 36 + mexos/basedata.go | 14 + mexos/cloudflare.go | 18 + mexos/cluster.go | 384 +++++++++ mexos/const.go | 20 + mexos/dns.go | 263 ++++++ mexos/env.go | 32 + mexos/flavor.go | 84 ++ mexos/fqdn.go | 111 +++ mexos/gcloud.go | 44 + mexos/helm.go | 85 ++ mexos/init.go | 14 + mexos/inst.go | 172 ++++ mexos/ip.go | 104 +++ mexos/items.go | 9 + mexos/kubeconfig.go | 173 ++++ mexos/kubectl.go | 218 +++++ mexos/kubeproxy.go | 73 ++ mexos/kubernetes.go | 326 ++++++++ .../oscliapi/mex.go => mexos/kvm.go | 250 ++++-- mexos/lb.go | 94 +++ mexos/letsencrypt.go | 135 +++ mexos/manifest.go | 328 ++++++++ mexos/misc.go | 91 ++ mexos/network.go | 99 +++ mexos/nginx.go | 83 ++ mexos/openstack.go | 171 ++++ mexos/oscli.go | 448 ++++++++++ mexos/platform.go | 27 + mexos/ports.go | 36 + mexos/revproxy.go | 41 + mexos/rootlb.go | 190 +++++ mexos/securityrule.go | 102 +++ mexos/ssh.go | 79 ++ mexos/stats.go | 28 + mexos/svc.go | 49 ++ mexos/swarm.go | 246 ++++++ mexos/templates.go | 306 +++++++ mexos/types.go | 226 +++++ mexos/uri.go | 107 +++ mexos/valid.go | 171 ++++ mexos/vault.go | 148 ++++ mgmt/registry/install-files.sh | 14 + openstack-prov/oscliapi/mex_test.go | 275 ------- openstack-prov/oscliapi/oscli.go | 775 ------------------ openstack-prov/oscliapi/oscli_test.go | 472 ----------- openstack-prov/oscliapi/userdata.txt | 13 - openstack-tenant/agent/README.md | 5 +- .../agent/mexosagent.service | 2 +- openstack-tenant/agent/server/handlers.go | 1 - openstack-tenant/agent/server/server.go | 2 +- .../packer/packer_template.mobiledgex.json | 6 +- openstack-tenant/packer/setup.sh | 83 +- .../mobiledgex-16.04-qcow2/mobiledgex-init.sh | 144 ++-- 107 files changed, 8849 insertions(+), 1771 deletions(-) create mode 100644 .gitignore create mode 100644 deployments/examples/application/example/example.yaml create mode 100644 deployments/examples/cluster/azure/example.yaml create mode 100644 deployments/examples/cluster/gcp/example.yaml create mode 100644 deployments/examples/kustomize/application/base/docker-swarm/stackdemo/docker-compose.yml create mode 100644 deployments/examples/kustomize/application/base/docker-swarm/stackdemo/kustomization.yaml create mode 100644 deployments/examples/kustomize/application/base/kubernetes/testapp/app.yaml create mode 100644 deployments/examples/kustomize/application/base/kubernetes/testapp/kustomization.yaml create mode 100755 deployments/examples/kustomize/application/build.sh create mode 100644 deployments/examples/kustomize/application/output/mytest-app.yaml create mode 100644 deployments/examples/kustomize/application/output/stackdemo.yaml create mode 100644 deployments/examples/kustomize/application/overlays/mytest-app/deploy.yaml create mode 100644 deployments/examples/kustomize/application/overlays/mytest-app/kustomization.yaml create mode 100644 deployments/examples/kustomize/application/overlays/mytest-app/tcp.yaml create mode 100644 deployments/examples/kustomize/application/overlays/mytest-app/udp.yaml create mode 100644 deployments/examples/kustomize/application/overlays/stackdemo/kustomization.yaml create mode 100644 deployments/examples/kustomize/application/overlays/stackdemo/stackdemo.yaml create mode 100644 deployments/examples/kustomize/infrastructure/base/azure/kustomization.yaml create mode 100644 deployments/examples/kustomize/infrastructure/base/azure/values.yaml create mode 100644 deployments/examples/kustomize/infrastructure/base/gcp/kustomization.yaml create mode 100644 deployments/examples/kustomize/infrastructure/base/gcp/values.yaml create mode 100755 deployments/examples/kustomize/infrastructure/build.sh create mode 100644 deployments/examples/kustomize/infrastructure/output/centralus-test.azure.yaml create mode 100644 deployments/examples/kustomize/infrastructure/output/us-central1-test.gcp.yaml create mode 100644 deployments/examples/kustomize/infrastructure/overlays/centralus-test.azure/kustomization.yaml create mode 100644 deployments/examples/kustomize/infrastructure/overlays/centralus-test.azure/values.yaml create mode 100644 deployments/examples/kustomize/infrastructure/overlays/us-central1-test.gcp/kustomization.yaml create mode 100644 deployments/examples/kustomize/infrastructure/overlays/us-central1-test.gcp/values.yaml create mode 100644 deployments/examples/platform/azure/example.yaml create mode 100644 deployments/examples/platform/gcp/example.yaml create mode 100644 examples/docker-swarm/stackdemo/Dockerfile create mode 100644 examples/docker-swarm/stackdemo/app.py create mode 100644 examples/docker-swarm/stackdemo/docker-compose.yml create mode 100644 examples/docker-swarm/stackdemo/requirements.txt create mode 100644 examples/mexexample/Dockerfile create mode 100644 examples/mexexample/Makefile create mode 100644 examples/mexexample/api/mexexample.pb.go create mode 100644 examples/mexexample/api/mexexample.pb.gw.go create mode 100644 examples/mexexample/api/mexexample.proto create mode 100644 examples/mexexample/api/mexexample.swagger.json create mode 100755 examples/mexexample/docker-build.sh create mode 100644 examples/mexexample/mexexample.go create mode 100644 examples/mexexample/mexexample.yaml create mode 100644 examples/mexexample/server/handlers.go create mode 100644 examples/mexexample/server/server.go create mode 100644 mexctl/.gitignore create mode 100644 mexctl/main.go create mode 100755 mexctl/mexctl-all.sh create mode 100644 mexos/.gitignore create mode 100644 mexos/agent.go create mode 100644 mexos/azure.go create mode 100644 mexos/basedata.go create mode 100644 mexos/cloudflare.go create mode 100644 mexos/cluster.go create mode 100644 mexos/const.go create mode 100644 mexos/dns.go create mode 100644 mexos/env.go create mode 100644 mexos/flavor.go create mode 100644 mexos/fqdn.go create mode 100644 mexos/gcloud.go create mode 100644 mexos/helm.go create mode 100644 mexos/init.go create mode 100644 mexos/inst.go create mode 100644 mexos/ip.go create mode 100644 mexos/items.go create mode 100644 mexos/kubeconfig.go create mode 100644 mexos/kubectl.go create mode 100644 mexos/kubeproxy.go create mode 100644 mexos/kubernetes.go rename openstack-prov/oscliapi/mex.go => mexos/kvm.go (67%) create mode 100644 mexos/lb.go create mode 100644 mexos/letsencrypt.go create mode 100644 mexos/manifest.go create mode 100644 mexos/misc.go create mode 100644 mexos/network.go create mode 100644 mexos/nginx.go create mode 100644 mexos/openstack.go create mode 100644 mexos/oscli.go create mode 100644 mexos/platform.go create mode 100644 mexos/ports.go create mode 100644 mexos/revproxy.go create mode 100644 mexos/rootlb.go create mode 100644 mexos/securityrule.go create mode 100644 mexos/ssh.go create mode 100644 mexos/stats.go create mode 100644 mexos/svc.go create mode 100644 mexos/swarm.go create mode 100644 mexos/templates.go create mode 100644 mexos/types.go create mode 100644 mexos/uri.go create mode 100644 mexos/valid.go create mode 100644 mexos/vault.go create mode 100755 mgmt/registry/install-files.sh delete mode 100644 openstack-prov/oscliapi/mex_test.go delete mode 100644 openstack-prov/oscliapi/oscli.go delete mode 100644 openstack-prov/oscliapi/oscli_test.go delete mode 100644 openstack-prov/oscliapi/userdata.txt rename mexosagent.service => openstack-tenant/agent/mexosagent.service (83%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..35f440a23 --- /dev/null +++ b/.gitignore @@ -0,0 +1,46 @@ +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el diff --git a/deployments/examples/application/example/example.yaml b/deployments/examples/application/example/example.yaml new file mode 100644 index 000000000..c972bd45c --- /dev/null +++ b/deployments/examples/application/example/example.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +name: {{.Name}} +kind: application +resource: {{.Resource.Group}} +metadata: + kind: {{.Application.Kind}} + name: {{.Application.Name}} + tags: {{.Application.Name}}-tag + tenant: {{.Application.Name}}-tenant + operator: {{.Operator.Name}} + dnszone: {{.Network.DNSZone}} +config: + kind: config + source: + detail: + base: {{.Application.Base}} + resources: + template: {{.Application.Template}} + deployment: {{.Application.Deployment}} + manifest: {{.Application.Manifest}} +spec: + flags: none + key: {{.Cluster.Name}} + rootlb: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}} + image: {{.Application.Image}} + imagetype: {{.Application.ImageType}} + proxypath: {{.Application.ProxyPath}} + flavor: {{.Cluster.Flavor}} + uri: https://{{.Application.Name}}.{{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}} + ipaccess: {{.Network.IPAccess}} + ports: + - name: lprotohttp1 + mexproto: LProtoHTTP + proto: TCP + internalport: 27272 + publicport: 27272 + publicpath: {{.Application.Name}}-grpc + - name: lprotohttp2 + mexproto: LProtoHTTP + proto: TCP + internalport: 27273 + publicport: 27273 + publicpath: {{.Application.Name}}-rest + - name: lprotohttp3 + mexproto: LProtoHTTP + proto: TCP + internalport: 27274 + publicport: 27274 + publicpath: {{.Application.Name}}-http + - name: lprototcp1 + mexproto: LProtoTCP + proto: TCP + internalport: 27275 + publicport: 27275 + publicpath: {{.Application.Name}}-tcp + - name: lprotoudp1 + mexproto: LProtoUDP + proto: UDP + internalport: 27276 + publicport: 27276 + publicpath: {{.Application.Name}}-udp + command: diff --git a/deployments/examples/cluster/azure/example.yaml b/deployments/examples/cluster/azure/example.yaml new file mode 100644 index 000000000..036295cc7 --- /dev/null +++ b/deployments/examples/cluster/azure/example.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +name: {{.Name}} +kind: cluster +resource: {{.Resource.Group}} +metadata: + name: {{.Cluster.Name}} + tags: {{.Cluster.Name}}-tag + kind: {{.Cluster.Kind}} + tenant: {{.Cluster.Name}}-tenant + region: {{.Cluster.Region}} + zone: {{.Cluster.Zone}} + location: {{.Cluster.Location}} + resourcegroup: {{.Resource.Group}} + operator: {{.Operator.Name}} + dnszone: {{.Network.DNSZone}} +spec: + flags: force + flavor: {{.Cluster.Flavor}} + key: {{.Cluster.Name}} + dockerregistry: {{.Registry.Docker}} + rootlb: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}} + networkscheme: {{.Network.Scheme}} diff --git a/deployments/examples/cluster/gcp/example.yaml b/deployments/examples/cluster/gcp/example.yaml new file mode 100644 index 000000000..a58b121b3 --- /dev/null +++ b/deployments/examples/cluster/gcp/example.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +name: {{.Name}} +kind: cluster +resource: {{.Resource.Group}} +metadata: + name: {{.Cluster.Name}} + tags: {{.Cluster.Name}}-tag + kind: {{.Cluster.Kind}} + tenant: {{.Cluster.Name}}-tenant + region: {{.Cluster.Region}} + zone: {{.Cluster.Zone}} + location: {{.Cluster.Location}} + resourcegroup: {{.Resource.Group}} + operator: {{.Operator.Name}} + dnszone: {{.Network.DNSZone}} + project: {{.Resource.Project}} +spec: + flags: force + flavor: {{.Cluster.Flavor}} + key: {{.Cluster.Name}} + dockerregistry: {{.Registry.Docker}} + rootlb: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}} + networkscheme: {{.Network.Scheme}} diff --git a/deployments/examples/kustomize/application/base/docker-swarm/stackdemo/docker-compose.yml b/deployments/examples/kustomize/application/base/docker-swarm/stackdemo/docker-compose.yml new file mode 100644 index 000000000..bb01c4c8d --- /dev/null +++ b/deployments/examples/kustomize/application/base/docker-swarm/stackdemo/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3' + +services: + web: + image: registry.mobiledgex.net:5000/stackdemo + build: . + ports: + - "8000:8000" + redis: + image: redis:alpine diff --git a/deployments/examples/kustomize/application/base/docker-swarm/stackdemo/kustomization.yaml b/deployments/examples/kustomize/application/base/docker-swarm/stackdemo/kustomization.yaml new file mode 100644 index 000000000..699801500 --- /dev/null +++ b/deployments/examples/kustomize/application/base/docker-swarm/stackdemo/kustomization.yaml @@ -0,0 +1,7 @@ +# this is a dummy. no kustomize for docker-compose +apiVersion: v1 +kind: Kustomization +#resources: +#- docker-compose.yml + + diff --git a/deployments/examples/kustomize/application/base/kubernetes/testapp/app.yaml b/deployments/examples/kustomize/application/base/kubernetes/testapp/app.yaml new file mode 100644 index 000000000..b47c1e224 --- /dev/null +++ b/deployments/examples/kustomize/application/base/kubernetes/testapp/app.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: Service +metadata: + name: mexexample-tcp-service + labels: + run: mexexample +spec: + type: LoadBalancer + ports: + - port: 27272 + targetPort: 27272 + protocol: TCP + name: grpc27272 + - port: 27273 + targetPort: 27273 + protocol: TCP + name: rest27273 + - port: 27274 + targetPort: 27274 + protocol: TCP + name: http27274 + - port: 27275 + targetPort: 27275 + protocol: TCP + name: tcp27275 + selector: + run: mexexample +--- +apiVersion: v1 +kind: Service +metadata: + name: mexexample-udp-service + labels: + run: mexexample +spec: + type: LoadBalancer + ports: + - port: 27276 + targetPort: 27276 + protocol: UDP + name: udp27276 + selector: + run: mexexample +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mexexample-deployment +spec: + selector: + matchLabels: + run: mexexample + replicas: 2 + template: + metadata: + labels: + run: mexexample + spec: + volumes: + - name: mexexample + emptyDir: {} + imagePullSecrets: + - name: mexregistrysecret + containers: + - name: mexexample + image: registry.mobiledgex.net:5000/mobiledgex/mexexample + imagePullPolicy: Always + ports: + - containerPort: 27272 + protocol: TCP + - containerPort: 27273 + protocol: TCP + - containerPort: 27274 + protocol: TCP + - containerPort: 27275 + protocol: TCP + - containerPort: 27276 + protocol: UDP diff --git a/deployments/examples/kustomize/application/base/kubernetes/testapp/kustomization.yaml b/deployments/examples/kustomize/application/base/kubernetes/testapp/kustomization.yaml new file mode 100644 index 000000000..109348a45 --- /dev/null +++ b/deployments/examples/kustomize/application/base/kubernetes/testapp/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Kustomization + +#namespace: example +#namePrefix: example- +#commonAnnotations: +# operator-contact: 800-555-1212 +#commonLabels: +# app: example + +resources: +- app.yaml + diff --git a/deployments/examples/kustomize/application/build.sh b/deployments/examples/kustomize/application/build.sh new file mode 100755 index 000000000..cd3708b1b --- /dev/null +++ b/deployments/examples/kustomize/application/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -x +for i in overlays/*; do + kustomize build $i > output/$(basename $i).yaml +done +# bogus 0 length output for docker-compose example +rm output/stackdemo.yaml +cp base/docker-swarm/stackdemo/docker-compose.yml output/stackdemo.yaml diff --git a/deployments/examples/kustomize/application/output/mytest-app.yaml b/deployments/examples/kustomize/application/output/mytest-app.yaml new file mode 100644 index 000000000..b6eec7a77 --- /dev/null +++ b/deployments/examples/kustomize/application/output/mytest-app.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + run: mytest-app + name: mytest-app-mexexample-tcp-service +spec: + ports: + - name: grpc27272 + port: 27272 + protocol: TCP + targetPort: 27272 + - name: rest27273 + port: 27273 + protocol: TCP + targetPort: 27273 + - name: http27274 + port: 27274 + protocol: TCP + targetPort: 27274 + - name: tcp27275 + port: 27275 + protocol: TCP + targetPort: 27275 + selector: + run: mytest-app + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: mytest-app + name: mytest-app-mexexample-udp-service +spec: + ports: + - name: udp27276 + port: 27276 + protocol: UDP + targetPort: 27276 + selector: + run: mytest-app + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mytest-app-mexexample-deployment +spec: + replicas: 2 + selector: + matchLabels: + run: mytest-app + template: + metadata: + labels: + run: mytest-app + spec: + containers: + - image: registry.mobiledgex.net:5000/mobiledgex/mexexample + imagePullPolicy: Always + name: mexexample + ports: + - containerPort: 27272 + protocol: TCP + - containerPort: 27273 + protocol: TCP + - containerPort: 27274 + protocol: TCP + - containerPort: 27275 + protocol: TCP + - containerPort: 27276 + protocol: UDP + imagePullSecrets: + - name: mexregistrysecret + volumes: + - emptyDir: {} + name: mexexample diff --git a/deployments/examples/kustomize/application/output/stackdemo.yaml b/deployments/examples/kustomize/application/output/stackdemo.yaml new file mode 100644 index 000000000..bb01c4c8d --- /dev/null +++ b/deployments/examples/kustomize/application/output/stackdemo.yaml @@ -0,0 +1,10 @@ +version: '3' + +services: + web: + image: registry.mobiledgex.net:5000/stackdemo + build: . + ports: + - "8000:8000" + redis: + image: redis:alpine diff --git a/deployments/examples/kustomize/application/overlays/mytest-app/deploy.yaml b/deployments/examples/kustomize/application/overlays/mytest-app/deploy.yaml new file mode 100644 index 000000000..601473717 --- /dev/null +++ b/deployments/examples/kustomize/application/overlays/mytest-app/deploy.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mexexample-deployment +spec: + selector: + matchLabels: + run: mytest-app + template: + metadata: + labels: + run: mytest-app \ No newline at end of file diff --git a/deployments/examples/kustomize/application/overlays/mytest-app/kustomization.yaml b/deployments/examples/kustomize/application/overlays/mytest-app/kustomization.yaml new file mode 100644 index 000000000..202716af2 --- /dev/null +++ b/deployments/examples/kustomize/application/overlays/mytest-app/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Kustomization +bases: +- ../../base/kubernetes/testapp +patchesStrategicMerge: +- tcp.yaml +- udp.yaml +- deploy.yaml +namePrefix: mytest-app- +#commonLabels: +# org: acme +# variant: testing +#commonAnnotations: +# note: Hello, I am testing! + diff --git a/deployments/examples/kustomize/application/overlays/mytest-app/tcp.yaml b/deployments/examples/kustomize/application/overlays/mytest-app/tcp.yaml new file mode 100644 index 000000000..440c5fd0c --- /dev/null +++ b/deployments/examples/kustomize/application/overlays/mytest-app/tcp.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: mexexample-tcp-service + labels: + run: mytest-app +spec: + selector: + run: mytest-app \ No newline at end of file diff --git a/deployments/examples/kustomize/application/overlays/mytest-app/udp.yaml b/deployments/examples/kustomize/application/overlays/mytest-app/udp.yaml new file mode 100644 index 000000000..b810bc4e6 --- /dev/null +++ b/deployments/examples/kustomize/application/overlays/mytest-app/udp.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: mexexample-udp-service + labels: + run: mytest-app +spec: + selector: + run: mytest-app \ No newline at end of file diff --git a/deployments/examples/kustomize/application/overlays/stackdemo/kustomization.yaml b/deployments/examples/kustomize/application/overlays/stackdemo/kustomization.yaml new file mode 100644 index 000000000..6a498527a --- /dev/null +++ b/deployments/examples/kustomize/application/overlays/stackdemo/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Kustomization +bases: +- ../../base/docker-swarm/stackdemo +#patchesStrategicMerge: +#- stackdemo.yaml + + diff --git a/deployments/examples/kustomize/application/overlays/stackdemo/stackdemo.yaml b/deployments/examples/kustomize/application/overlays/stackdemo/stackdemo.yaml new file mode 100644 index 000000000..39f8f374c --- /dev/null +++ b/deployments/examples/kustomize/application/overlays/stackdemo/stackdemo.yaml @@ -0,0 +1,4 @@ +#apiVersion: v1 +#kind: docker-swarm-app +#metadata: +# name: docker-swarm-example diff --git a/deployments/examples/kustomize/infrastructure/base/azure/kustomization.yaml b/deployments/examples/kustomize/infrastructure/base/azure/kustomization.yaml new file mode 100644 index 000000000..b7dc0ca71 --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/base/azure/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Kustomization + +#namespace: example +#namePrefix: example- +#commonAnnotations: +# operator-contact: 800-555-1212 +#commonLabels: +# app: example + +resources: +- values.yaml + diff --git a/deployments/examples/kustomize/infrastructure/base/azure/values.yaml b/deployments/examples/kustomize/infrastructure/base/azure/values.yaml new file mode 100644 index 000000000..c47c94f24 --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/base/azure/values.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +name: example-test +metadata: + name: azure +kind: values +resource: azure +values: + name: example-test + kind: static + base: example + operator: + name: azure + kind: azure + cluster: + name: example-test + flavor: x1.medium + kind: kubernetes + location: centralus + region: centralus + zone: centralus + osimage: mobiledgex + network: + name: mex-k8s-net-1 + dnszone: mobiledgex.net + ipaccess: IpAccessShared + external: external-network-shared + router: mex-k8s-router-1 + options: dhcp + securityrule: default + scheme: priv-subnet,mex-k8s-net-1,10.101.X.0/24 + holepunch: 22222 + agent: + image: registry.mobiledgex.net:5000/mobiledgex/mexosagent + port: 18889 + status: active + registry: + name: registry.mobiledgex.net + docker: registry.mobiledgex.net:5000 + update: example-test + resource: + group: azure-centralus-rg-1 + application: + name: + kind: kubernetes + deployment: kubernetes + base: + overlay: + manifest: + template: + image: registry.mobiledgex.net:5000/mobiledgex/mexexample + imagetype: ImageTypeDocker + proxypath: + environment: + openrc: + mexenv: https://vault.mobiledgex.net/v1/secret/data/cloudlet/openstack/mexenv.json diff --git a/deployments/examples/kustomize/infrastructure/base/gcp/kustomization.yaml b/deployments/examples/kustomize/infrastructure/base/gcp/kustomization.yaml new file mode 100644 index 000000000..b7dc0ca71 --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/base/gcp/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Kustomization + +#namespace: example +#namePrefix: example- +#commonAnnotations: +# operator-contact: 800-555-1212 +#commonLabels: +# app: example + +resources: +- values.yaml + diff --git a/deployments/examples/kustomize/infrastructure/base/gcp/values.yaml b/deployments/examples/kustomize/infrastructure/base/gcp/values.yaml new file mode 100644 index 000000000..f9abf89f4 --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/base/gcp/values.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +name: example-test +metadata: + name: gcp +kind: values +resource: gcp +values: + name: example-test + kind: static + base: example + operator: + name: gcp + kind: gcp + cluster: + name: example-test + flavor: x1.medium + kind: kubernetes + location: us-west + region: us-west1 + zone: us-west1-a + osimage: mobiledgex + network: + name: mex-k8s-net-1 + dnszone: mobiledgex.net + ipaccess: IpAccessShared + external: external-network-shared + router: mex-k8s-router-1 + options: dhcp + securityrule: default + scheme: priv-subnet,mex-k8s-net-1,10.101.X.0/24 + holepunch: 22222 + agent: + image: registry.mobiledgex.net:5000/mobiledgex/mexosagent + port: 18889 + status: active + registry: + name: registry.mobiledgex.net + docker: registry.mobiledgex.net:5000 + update: example-test + resource: + group: gcp-us-west1-a-still-entity-201400 + application: + name: + kind: kubernetes + deployment: kubernetes + base: + overlay: + manifest: + template: + image: registry.mobiledgex.net:5000/mobiledgex/mexexample + imagetype: ImageTypeDocker + proxypath: + environment: + openrc: + mexenv: https://vault.mobiledgex.net/v1/secret/data/cloudlet/openstack/mexenv.json diff --git a/deployments/examples/kustomize/infrastructure/build.sh b/deployments/examples/kustomize/infrastructure/build.sh new file mode 100755 index 000000000..c4a294517 --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -x +for i in overlays/*; do + kustomize build $i > output/$(basename $i).yaml +done diff --git a/deployments/examples/kustomize/infrastructure/output/centralus-test.azure.yaml b/deployments/examples/kustomize/infrastructure/output/centralus-test.azure.yaml new file mode 100644 index 000000000..892d6a0a6 --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/output/centralus-test.azure.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: values +metadata: + name: sunnydale-test-azure +name: sunnydale +resource: azure +values: + agent: + image: registry.mobiledgex.net:5000/mobiledgex/mexosagent + port: 18889 + status: active + application: + base: null + deployment: kubernetes + image: registry.mobiledgex.net:5000/mobiledgex/mexexample + imagetype: ImageTypeDocker + kind: kubernetes + manifest: null + name: null + overlay: null + proxypath: null + template: null + base: example + cluster: + flavor: x1.medium + kind: kubernetes + location: centralus + name: centralus-test + osimage: mobiledgex + region: centralus + zone: centralus + environment: + mexenv: https://vault.mobiledgex.net/v1/secret/data/cloudlet/openstack/mexenv.json + kind: static + name: centralus-test + network: + dnszone: mobiledgex.net + external: external-network-shared + holepunch: 22223 + ipaccess: IpAccessShared + name: mex-k8s-net-1 + options: dhcp + router: mex-k8s-router-1 + scheme: priv-subnet,mex-k8s-net-1,10.101.X.0/24 + securityrule: default + operator: + kind: azure + name: azure + registry: + docker: registry.mobiledgex.net:5000 + name: registry.mobiledgex.net + update: centralus-test + resource: + group: azure-centralus-rg-1 diff --git a/deployments/examples/kustomize/infrastructure/output/us-central1-test.gcp.yaml b/deployments/examples/kustomize/infrastructure/output/us-central1-test.gcp.yaml new file mode 100644 index 000000000..f6a542ce1 --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/output/us-central1-test.gcp.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +kind: values +metadata: + name: sunnydale-test-gcp +name: sunnydale +resource: gcp +values: + agent: + image: registry.mobiledgex.net:5000/mobiledgex/mexosagent + port: 18889 + status: active + application: + base: null + deployment: kubernetes + image: registry.mobiledgex.net:5000/mobiledgex/mexexample + imagetype: ImageTypeDocker + kind: kubernetes + manifest: null + name: null + overlay: null + proxypath: null + template: null + base: example + cluster: + flavor: x1.medium + kind: kubernetes + location: us-central1 + name: us-central1-test + osimage: mobiledgex + region: us-central1 + zone: us-central1-a + environment: + mexenv: https://vault.mobiledgex.net/v1/secret/data/cloudlet/openstack/mexenv.json + kind: static + name: us-central1-test + network: + dnszone: mobiledgex.net + external: external-network-shared + holepunch: 22223 + ipaccess: IpAccessShared + name: mex-k8s-net-1 + options: dhcp + router: mex-k8s-router-1 + scheme: priv-subnet,mex-k8s-net-1,10.101.X.0/24 + securityrule: default + operator: + kind: gcp + name: gcp + registry: + docker: registry.mobiledgex.net:5000 + name: registry.mobiledgex.net + update: us-central1-test + resource: + group: gcp-us-west1-a-still-entity-201400 + project: still-entity-201400 diff --git a/deployments/examples/kustomize/infrastructure/overlays/centralus-test.azure/kustomization.yaml b/deployments/examples/kustomize/infrastructure/overlays/centralus-test.azure/kustomization.yaml new file mode 100644 index 000000000..a42d34d72 --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/overlays/centralus-test.azure/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Kustomization +bases: +- ../../base/azure +patchesStrategicMerge: +- values.yaml +namePrefix: sunnydale-test- +#commonLabels: +# org: acme +# variant: testing +#commonAnnotations: +# note: Hello, I am testing! + diff --git a/deployments/examples/kustomize/infrastructure/overlays/centralus-test.azure/values.yaml b/deployments/examples/kustomize/infrastructure/overlays/centralus-test.azure/values.yaml new file mode 100644 index 000000000..6459ad75b --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/overlays/centralus-test.azure/values.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +name: sunnydale +metadata: + name: azure +kind: values +resource: azure +values: + name: centralus-test + cluster: + name: centralus-test + location: centralus + region: centralus + zone: centralus + network: + holepunch: 22223 + registry: + update: centralus-test + environment: + openrc: + mexenv: https://vault.mobiledgex.net/v1/secret/data/cloudlet/openstack/mexenv.json diff --git a/deployments/examples/kustomize/infrastructure/overlays/us-central1-test.gcp/kustomization.yaml b/deployments/examples/kustomize/infrastructure/overlays/us-central1-test.gcp/kustomization.yaml new file mode 100644 index 000000000..7c28fe60f --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/overlays/us-central1-test.gcp/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Kustomization +bases: +- ../../base/gcp +patchesStrategicMerge: +- values.yaml +namePrefix: sunnydale-test- +#commonLabels: +# org: acme +# variant: testing +#commonAnnotations: +# note: Hello, I am testing! + diff --git a/deployments/examples/kustomize/infrastructure/overlays/us-central1-test.gcp/values.yaml b/deployments/examples/kustomize/infrastructure/overlays/us-central1-test.gcp/values.yaml new file mode 100644 index 000000000..8a8d4571c --- /dev/null +++ b/deployments/examples/kustomize/infrastructure/overlays/us-central1-test.gcp/values.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +name: sunnydale +metadata: + name: gcp +kind: values +resource: gcp +values: + name: us-central1-test + resource: + project: still-entity-201400 + cluster: + name: us-central1-test + location: us-central1 + region: us-central1 + zone: us-central1-a + network: + holepunch: 22223 + registry: + update: us-central1-test + environment: + openrc: + mexenv: https://vault.mobiledgex.net/v1/secret/data/cloudlet/openstack/mexenv.json diff --git a/deployments/examples/platform/azure/example.yaml b/deployments/examples/platform/azure/example.yaml new file mode 100644 index 000000000..d3d8bc239 --- /dev/null +++ b/deployments/examples/platform/azure/example.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +name: {{.Name}} +kind: platform +resource: {{.Resource.Group}} +metadata: + kind: {{.Operator.Kind}} + name: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}} + tags: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}}-tag + tenant: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}}-tenant + region: {{.Cluster.Region}} + zone: {{.Cluster.Zone}} + location: {{.Cluster.Location}} + openrc: {{.Environment.OpenRC}} + dnszone: {{.Network.DNSZone}} + operator: gddt +spec: + flags: force + flavor: x1.medium + rootlb: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}} + key: {{.Name}} + dockerregistry: {{.Registry.Docker}} + externalnetwork: {{.Network.External}} + networkscheme: {{.Network.Scheme}} + externalrouter: {{.Network.Router}} + options: {{.Network.Options}} + agent: + image: {{.Agent.Image}} + status: {{.Agent.Status}} diff --git a/deployments/examples/platform/gcp/example.yaml b/deployments/examples/platform/gcp/example.yaml new file mode 100644 index 000000000..d3d8bc239 --- /dev/null +++ b/deployments/examples/platform/gcp/example.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +name: {{.Name}} +kind: platform +resource: {{.Resource.Group}} +metadata: + kind: {{.Operator.Kind}} + name: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}} + tags: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}}-tag + tenant: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}}-tenant + region: {{.Cluster.Region}} + zone: {{.Cluster.Zone}} + location: {{.Cluster.Location}} + openrc: {{.Environment.OpenRC}} + dnszone: {{.Network.DNSZone}} + operator: gddt +spec: + flags: force + flavor: x1.medium + rootlb: {{.Cluster.Name}}.{{.Operator.Name}}.{{.Network.DNSZone}} + key: {{.Name}} + dockerregistry: {{.Registry.Docker}} + externalnetwork: {{.Network.External}} + networkscheme: {{.Network.Scheme}} + externalrouter: {{.Network.Router}} + options: {{.Network.Options}} + agent: + image: {{.Agent.Image}} + status: {{.Agent.Status}} diff --git a/examples/docker-swarm/stackdemo/Dockerfile b/examples/docker-swarm/stackdemo/Dockerfile new file mode 100644 index 000000000..8e67d74e8 --- /dev/null +++ b/examples/docker-swarm/stackdemo/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.4-alpine +ADD . /code +WORKDIR /code +RUN pip install -r requirements.txt +CMD ["python", "app.py"] diff --git a/examples/docker-swarm/stackdemo/app.py b/examples/docker-swarm/stackdemo/app.py new file mode 100644 index 000000000..341db727f --- /dev/null +++ b/examples/docker-swarm/stackdemo/app.py @@ -0,0 +1,15 @@ +from flask import Flask +from redis import Redis +import requests + +app = Flask(__name__) +redis = Redis(host='redis', port=6379) + +@app.route('/') +def hello(): + count = redis.incr('hits') + response = requests.get("https://api.ipify.org?format=json") + return 'Hello World! I have been seen {0} times. IP address {1}\n'.format(count,response.json()["ip"]) + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8000, debug=True) diff --git a/examples/docker-swarm/stackdemo/docker-compose.yml b/examples/docker-swarm/stackdemo/docker-compose.yml new file mode 100644 index 000000000..bb01c4c8d --- /dev/null +++ b/examples/docker-swarm/stackdemo/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3' + +services: + web: + image: registry.mobiledgex.net:5000/stackdemo + build: . + ports: + - "8000:8000" + redis: + image: redis:alpine diff --git a/examples/docker-swarm/stackdemo/requirements.txt b/examples/docker-swarm/stackdemo/requirements.txt new file mode 100644 index 000000000..b8a0ee2e7 --- /dev/null +++ b/examples/docker-swarm/stackdemo/requirements.txt @@ -0,0 +1,3 @@ +flask +redis +requests diff --git a/examples/mexexample/Dockerfile b/examples/mexexample/Dockerfile new file mode 100644 index 000000000..3a91df5db --- /dev/null +++ b/examples/mexexample/Dockerfile @@ -0,0 +1,17 @@ +FROM registry.mobiledgex.net:5000/mobiledgex/build AS build + +WORKDIR /go/src/github.com/mobiledgex/edge-cloud/ +COPY . . +ENV CGO_ENABLED=0 +ENV GOPATH=/go +ENV PATH="/go/bin:${PATH}" +WORKDIR /go/src/github.com/mobiledgex/edge-cloud/cloud-resource-manager/example/mexexample +RUN go get -d -v ./... +RUN go install -v ./... + +FROM alpine:latest + +COPY --from=build /go/bin/mexexample /usr/local/bin + +ENTRYPOINT [ "mexexample" ] +CMD [] diff --git a/examples/mexexample/Makefile b/examples/mexexample/Makefile new file mode 100644 index 000000000..24a8a8724 --- /dev/null +++ b/examples/mexexample/Makefile @@ -0,0 +1,63 @@ +PKG = "mexexample" +GOPATH ?= $(shell go env GOPATH) +GO_PACKAGES := $(shell go list ./... | grep -v /vendor/) +PROGRAM = mexexample +API_FILES = api/mexexample.pb.go api/mexexample.pb.gw.go api/mexexample.swagger.json + +.PHONY: build api dep test race msan + +default: build + +docker-build: + ./docker-build.sh + +docker-push: + docker tag mobiledgex/mexexample registry.mobiledgex.net:5000/mobiledgex/mexexample + docker push registry.mobiledgex.net:5000/mobiledgex/mexexample + +build: api dep ## Build mexexample + go get ./... + mkdir -p build + CGO_ENABLED=0 go build -o build/${PROGRAM} . + +dep: api ## Fetch dependencies + @go get ./... + +api: $(API_FILES) ## Auto-generate gRPC/REST Go sources + +api/mexexample.pb.go: api/mexexample.proto + @protoc -I. \ + -I${GOPATH}/src \ + -I${GOPATH}/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \ + --go_out=plugins=grpc:. \ + api/mexexample.proto + +api/mexexample.pb.gw.go: api/mexexample.proto + @protoc -I. \ + -I${GOPATH}/src \ + -I${GOPATH}/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \ + --grpc-gateway_out=logtostderr=true:. \ + api/mexexample.proto + +api/mexexample.swagger.json: api/mexexample.proto + @protoc -I. \ + -I${GOPATH}/src \ + -I${GOPATH}/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \ + --swagger_out=logtostderr=true:. \ + api/mexexample.proto + +clean: ## Clean compiled binaries + @rm -f build/${PROGRAM} + +realclean: ## Clean compiled binaries and all generated files + @rm -f build/${PROGRAM} + @rm -f ${API_FILES} + +test: dep ## Run tests + @go test -short ${GO_PACKAGES} + +race: dep ## Run tests with race detector + @go test -race -short ${GO_PACKAGES} + +msan: dep ## Run tests with memory sanitizer + @go test -msan -short ${GO_PACKAGES} diff --git a/examples/mexexample/api/mexexample.pb.go b/examples/mexexample/api/mexexample.pb.go new file mode 100644 index 000000000..7c8293f3d --- /dev/null +++ b/examples/mexexample/api/mexexample.pb.go @@ -0,0 +1,658 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: api/mexexample.proto + +package api + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "google.golang.org/genproto/googleapis/api/annotations" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type DataTestRequest struct { + Numbytes int64 `protobuf:"varint,1,opt,name=numbytes" json:"numbytes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DataTestRequest) Reset() { *m = DataTestRequest{} } +func (m *DataTestRequest) String() string { return proto.CompactTextString(m) } +func (*DataTestRequest) ProtoMessage() {} +func (*DataTestRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{0} +} +func (m *DataTestRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DataTestRequest.Unmarshal(m, b) +} +func (m *DataTestRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DataTestRequest.Marshal(b, m, deterministic) +} +func (dst *DataTestRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataTestRequest.Merge(dst, src) +} +func (m *DataTestRequest) XXX_Size() int { + return xxx_messageInfo_DataTestRequest.Size(m) +} +func (m *DataTestRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DataTestRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DataTestRequest proto.InternalMessageInfo + +func (m *DataTestRequest) GetNumbytes() int64 { + if m != nil { + return m.Numbytes + } + return 0 +} + +type DataTestResponse struct { + Bytes string `protobuf:"bytes,1,opt,name=bytes" json:"bytes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DataTestResponse) Reset() { *m = DataTestResponse{} } +func (m *DataTestResponse) String() string { return proto.CompactTextString(m) } +func (*DataTestResponse) ProtoMessage() {} +func (*DataTestResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{1} +} +func (m *DataTestResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DataTestResponse.Unmarshal(m, b) +} +func (m *DataTestResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DataTestResponse.Marshal(b, m, deterministic) +} +func (dst *DataTestResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataTestResponse.Merge(dst, src) +} +func (m *DataTestResponse) XXX_Size() int { + return xxx_messageInfo_DataTestResponse.Size(m) +} +func (m *DataTestResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DataTestResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DataTestResponse proto.InternalMessageInfo + +func (m *DataTestResponse) GetBytes() string { + if m != nil { + return m.Bytes + } + return "" +} + +type EchoRequest struct { + Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EchoRequest) Reset() { *m = EchoRequest{} } +func (m *EchoRequest) String() string { return proto.CompactTextString(m) } +func (*EchoRequest) ProtoMessage() {} +func (*EchoRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{2} +} +func (m *EchoRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EchoRequest.Unmarshal(m, b) +} +func (m *EchoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EchoRequest.Marshal(b, m, deterministic) +} +func (dst *EchoRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_EchoRequest.Merge(dst, src) +} +func (m *EchoRequest) XXX_Size() int { + return xxx_messageInfo_EchoRequest.Size(m) +} +func (m *EchoRequest) XXX_DiscardUnknown() { + xxx_messageInfo_EchoRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_EchoRequest proto.InternalMessageInfo + +func (m *EchoRequest) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type EchoResponse struct { + Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EchoResponse) Reset() { *m = EchoResponse{} } +func (m *EchoResponse) String() string { return proto.CompactTextString(m) } +func (*EchoResponse) ProtoMessage() {} +func (*EchoResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{3} +} +func (m *EchoResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EchoResponse.Unmarshal(m, b) +} +func (m *EchoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EchoResponse.Marshal(b, m, deterministic) +} +func (dst *EchoResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_EchoResponse.Merge(dst, src) +} +func (m *EchoResponse) XXX_Size() int { + return xxx_messageInfo_EchoResponse.Size(m) +} +func (m *EchoResponse) XXX_DiscardUnknown() { + xxx_messageInfo_EchoResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_EchoResponse proto.InternalMessageInfo + +func (m *EchoResponse) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type StatusRequest struct { + Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StatusRequest) Reset() { *m = StatusRequest{} } +func (m *StatusRequest) String() string { return proto.CompactTextString(m) } +func (*StatusRequest) ProtoMessage() {} +func (*StatusRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{4} +} +func (m *StatusRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StatusRequest.Unmarshal(m, b) +} +func (m *StatusRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StatusRequest.Marshal(b, m, deterministic) +} +func (dst *StatusRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_StatusRequest.Merge(dst, src) +} +func (m *StatusRequest) XXX_Size() int { + return xxx_messageInfo_StatusRequest.Size(m) +} +func (m *StatusRequest) XXX_DiscardUnknown() { + xxx_messageInfo_StatusRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_StatusRequest proto.InternalMessageInfo + +func (m *StatusRequest) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type StatusResponse struct { + Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` + Status string `protobuf:"bytes,2,opt,name=status" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StatusResponse) Reset() { *m = StatusResponse{} } +func (m *StatusResponse) String() string { return proto.CompactTextString(m) } +func (*StatusResponse) ProtoMessage() {} +func (*StatusResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{5} +} +func (m *StatusResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StatusResponse.Unmarshal(m, b) +} +func (m *StatusResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StatusResponse.Marshal(b, m, deterministic) +} +func (dst *StatusResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_StatusResponse.Merge(dst, src) +} +func (m *StatusResponse) XXX_Size() int { + return xxx_messageInfo_StatusResponse.Size(m) +} +func (m *StatusResponse) XXX_DiscardUnknown() { + xxx_messageInfo_StatusResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_StatusResponse proto.InternalMessageInfo + +func (m *StatusResponse) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +func (m *StatusResponse) GetStatus() string { + if m != nil { + return m.Status + } + return "" +} + +type InfoRequest struct { + Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *InfoRequest) Reset() { *m = InfoRequest{} } +func (m *InfoRequest) String() string { return proto.CompactTextString(m) } +func (*InfoRequest) ProtoMessage() {} +func (*InfoRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{6} +} +func (m *InfoRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InfoRequest.Unmarshal(m, b) +} +func (m *InfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InfoRequest.Marshal(b, m, deterministic) +} +func (dst *InfoRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_InfoRequest.Merge(dst, src) +} +func (m *InfoRequest) XXX_Size() int { + return xxx_messageInfo_InfoRequest.Size(m) +} +func (m *InfoRequest) XXX_DiscardUnknown() { + xxx_messageInfo_InfoRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_InfoRequest proto.InternalMessageInfo + +func (m *InfoRequest) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type InfoResponse struct { + Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` + Info string `protobuf:"bytes,2,opt,name=info" json:"info,omitempty"` + Outbound string `protobuf:"bytes,3,opt,name=outbound" json:"outbound,omitempty"` + Hostname string `protobuf:"bytes,4,opt,name=hostname" json:"hostname,omitempty"` + Realoutbound string `protobuf:"bytes,5,opt,name=realoutbound" json:"realoutbound,omitempty"` + Totaltcp uint64 `protobuf:"varint,6,opt,name=totaltcp" json:"totaltcp,omitempty"` + Totaludp uint64 `protobuf:"varint,7,opt,name=totaludp" json:"totaludp,omitempty"` + Interfaces []*Interface `protobuf:"bytes,8,rep,name=interfaces" json:"interfaces,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *InfoResponse) Reset() { *m = InfoResponse{} } +func (m *InfoResponse) String() string { return proto.CompactTextString(m) } +func (*InfoResponse) ProtoMessage() {} +func (*InfoResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{7} +} +func (m *InfoResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InfoResponse.Unmarshal(m, b) +} +func (m *InfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InfoResponse.Marshal(b, m, deterministic) +} +func (dst *InfoResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_InfoResponse.Merge(dst, src) +} +func (m *InfoResponse) XXX_Size() int { + return xxx_messageInfo_InfoResponse.Size(m) +} +func (m *InfoResponse) XXX_DiscardUnknown() { + xxx_messageInfo_InfoResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_InfoResponse proto.InternalMessageInfo + +func (m *InfoResponse) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +func (m *InfoResponse) GetInfo() string { + if m != nil { + return m.Info + } + return "" +} + +func (m *InfoResponse) GetOutbound() string { + if m != nil { + return m.Outbound + } + return "" +} + +func (m *InfoResponse) GetHostname() string { + if m != nil { + return m.Hostname + } + return "" +} + +func (m *InfoResponse) GetRealoutbound() string { + if m != nil { + return m.Realoutbound + } + return "" +} + +func (m *InfoResponse) GetTotaltcp() uint64 { + if m != nil { + return m.Totaltcp + } + return 0 +} + +func (m *InfoResponse) GetTotaludp() uint64 { + if m != nil { + return m.Totaludp + } + return 0 +} + +func (m *InfoResponse) GetInterfaces() []*Interface { + if m != nil { + return m.Interfaces + } + return nil +} + +type Interface struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Addresses string `protobuf:"bytes,2,opt,name=addresses" json:"addresses,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Interface) Reset() { *m = Interface{} } +func (m *Interface) String() string { return proto.CompactTextString(m) } +func (*Interface) ProtoMessage() {} +func (*Interface) Descriptor() ([]byte, []int) { + return fileDescriptor_mexexample_3a5dff07cce77a14, []int{8} +} +func (m *Interface) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Interface.Unmarshal(m, b) +} +func (m *Interface) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Interface.Marshal(b, m, deterministic) +} +func (dst *Interface) XXX_Merge(src proto.Message) { + xxx_messageInfo_Interface.Merge(dst, src) +} +func (m *Interface) XXX_Size() int { + return xxx_messageInfo_Interface.Size(m) +} +func (m *Interface) XXX_DiscardUnknown() { + xxx_messageInfo_Interface.DiscardUnknown(m) +} + +var xxx_messageInfo_Interface proto.InternalMessageInfo + +func (m *Interface) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Interface) GetAddresses() string { + if m != nil { + return m.Addresses + } + return "" +} + +func init() { + proto.RegisterType((*DataTestRequest)(nil), "api.DataTestRequest") + proto.RegisterType((*DataTestResponse)(nil), "api.DataTestResponse") + proto.RegisterType((*EchoRequest)(nil), "api.EchoRequest") + proto.RegisterType((*EchoResponse)(nil), "api.EchoResponse") + proto.RegisterType((*StatusRequest)(nil), "api.StatusRequest") + proto.RegisterType((*StatusResponse)(nil), "api.StatusResponse") + proto.RegisterType((*InfoRequest)(nil), "api.InfoRequest") + proto.RegisterType((*InfoResponse)(nil), "api.InfoResponse") + proto.RegisterType((*Interface)(nil), "api.Interface") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for MexExample service + +type MexExampleClient interface { + DataTest(ctx context.Context, in *DataTestRequest, opts ...grpc.CallOption) (*DataTestResponse, error) + Echo(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) + Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) + Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error) +} + +type mexExampleClient struct { + cc *grpc.ClientConn +} + +func NewMexExampleClient(cc *grpc.ClientConn) MexExampleClient { + return &mexExampleClient{cc} +} + +func (c *mexExampleClient) DataTest(ctx context.Context, in *DataTestRequest, opts ...grpc.CallOption) (*DataTestResponse, error) { + out := new(DataTestResponse) + err := grpc.Invoke(ctx, "/api.MexExample/DataTest", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *mexExampleClient) Echo(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) { + out := new(EchoResponse) + err := grpc.Invoke(ctx, "/api.MexExample/Echo", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *mexExampleClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + out := new(StatusResponse) + err := grpc.Invoke(ctx, "/api.MexExample/Status", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *mexExampleClient) Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error) { + out := new(InfoResponse) + err := grpc.Invoke(ctx, "/api.MexExample/Info", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for MexExample service + +type MexExampleServer interface { + DataTest(context.Context, *DataTestRequest) (*DataTestResponse, error) + Echo(context.Context, *EchoRequest) (*EchoResponse, error) + Status(context.Context, *StatusRequest) (*StatusResponse, error) + Info(context.Context, *InfoRequest) (*InfoResponse, error) +} + +func RegisterMexExampleServer(s *grpc.Server, srv MexExampleServer) { + s.RegisterService(&_MexExample_serviceDesc, srv) +} + +func _MexExample_DataTest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DataTestRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MexExampleServer).DataTest(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.MexExample/DataTest", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MexExampleServer).DataTest(ctx, req.(*DataTestRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MexExample_Echo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EchoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MexExampleServer).Echo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.MexExample/Echo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MexExampleServer).Echo(ctx, req.(*EchoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MexExample_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MexExampleServer).Status(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.MexExample/Status", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MexExampleServer).Status(ctx, req.(*StatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MexExample_Info_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(InfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MexExampleServer).Info(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.MexExample/Info", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MexExampleServer).Info(ctx, req.(*InfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _MexExample_serviceDesc = grpc.ServiceDesc{ + ServiceName: "api.MexExample", + HandlerType: (*MexExampleServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "DataTest", + Handler: _MexExample_DataTest_Handler, + }, + { + MethodName: "Echo", + Handler: _MexExample_Echo_Handler, + }, + { + MethodName: "Status", + Handler: _MexExample_Status_Handler, + }, + { + MethodName: "Info", + Handler: _MexExample_Info_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "api/mexexample.proto", +} + +func init() { proto.RegisterFile("api/mexexample.proto", fileDescriptor_mexexample_3a5dff07cce77a14) } + +var fileDescriptor_mexexample_3a5dff07cce77a14 = []byte{ + // 468 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4b, 0x8e, 0xd3, 0x40, + 0x10, 0x55, 0xfe, 0x49, 0x4d, 0x08, 0xa1, 0x09, 0xc8, 0xb2, 0x66, 0x11, 0xf5, 0x06, 0x33, 0x12, + 0xb1, 0x34, 0xec, 0x46, 0x9a, 0x0d, 0x22, 0x8b, 0x59, 0xcc, 0xc6, 0x70, 0x81, 0x4e, 0x5c, 0x49, + 0x2c, 0xc5, 0xdd, 0x26, 0x5d, 0x96, 0xc2, 0x0e, 0x71, 0x05, 0x2e, 0xc0, 0x9d, 0xb8, 0x02, 0x07, + 0x41, 0xfd, 0xb1, 0xe3, 0x8c, 0x84, 0x32, 0x3b, 0xbf, 0x4f, 0xbf, 0xae, 0x6a, 0x3d, 0xc3, 0x4c, + 0x14, 0x59, 0x9c, 0xe3, 0x11, 0x8f, 0x22, 0x2f, 0xf6, 0xb8, 0x28, 0x0e, 0x8a, 0x14, 0xeb, 0x88, + 0x22, 0x0b, 0xaf, 0xb7, 0x4a, 0x6d, 0xf7, 0x18, 0x1b, 0x87, 0x90, 0x52, 0x91, 0xa0, 0x4c, 0x49, + 0xed, 0x2c, 0xfc, 0x03, 0xbc, 0xfc, 0x2c, 0x48, 0x7c, 0x45, 0x4d, 0x09, 0x7e, 0x2b, 0x51, 0x13, + 0x0b, 0x61, 0x28, 0xcb, 0x7c, 0xf5, 0x9d, 0x50, 0x07, 0xad, 0x79, 0x2b, 0xea, 0x24, 0x35, 0xe6, + 0x11, 0x4c, 0x4f, 0x76, 0x5d, 0x28, 0xa9, 0x91, 0xcd, 0xa0, 0x77, 0x32, 0x8f, 0x12, 0x07, 0xf8, + 0x3b, 0xb8, 0x5a, 0xae, 0x77, 0xaa, 0x0a, 0x0d, 0x60, 0x90, 0xa3, 0xd6, 0x62, 0x8b, 0xde, 0x56, + 0x41, 0x1e, 0xc1, 0xd8, 0x19, 0x7d, 0xdc, 0xff, 0x9d, 0xef, 0xe1, 0xc5, 0x17, 0x12, 0x54, 0xea, + 0xcb, 0xa1, 0x9f, 0x60, 0x52, 0x59, 0x2f, 0xc5, 0xb2, 0xb7, 0xd0, 0xd7, 0xd6, 0x1b, 0xb4, 0xad, + 0xe0, 0x91, 0xd9, 0xe0, 0x41, 0x6e, 0x9e, 0xb1, 0xc1, 0x8f, 0x36, 0x8c, 0x9d, 0xf3, 0xe2, 0x5d, + 0x0c, 0xba, 0x99, 0xdc, 0x28, 0x7f, 0x93, 0xfd, 0x36, 0xef, 0xad, 0x4a, 0x5a, 0xa9, 0x52, 0xa6, + 0x41, 0xc7, 0xf2, 0x35, 0x36, 0xda, 0x4e, 0x69, 0x92, 0x22, 0xc7, 0xa0, 0xeb, 0xb4, 0x0a, 0x33, + 0x0e, 0xe3, 0x03, 0x8a, 0x7d, 0x7d, 0xb6, 0x67, 0xf5, 0x33, 0xce, 0x9c, 0x27, 0x45, 0x62, 0x4f, + 0xeb, 0x22, 0xe8, 0xcf, 0x5b, 0x51, 0x37, 0xa9, 0x71, 0xad, 0x95, 0x69, 0x11, 0x0c, 0x1a, 0x5a, + 0x99, 0x16, 0x6c, 0x01, 0x90, 0x49, 0xc2, 0xc3, 0x46, 0xac, 0x51, 0x07, 0xc3, 0x79, 0x27, 0xba, + 0xba, 0x9d, 0x2c, 0x44, 0x91, 0x2d, 0x1e, 0x2a, 0x3a, 0x69, 0x38, 0xf8, 0x3d, 0x8c, 0x6a, 0xc1, + 0x2c, 0x69, 0x07, 0x76, 0xbb, 0xdb, 0x6f, 0x76, 0x0d, 0x23, 0x91, 0xa6, 0x07, 0xd4, 0x1a, 0xab, + 0x77, 0x3e, 0x11, 0xb7, 0xbf, 0xdb, 0x00, 0x8f, 0x78, 0x5c, 0xba, 0xf6, 0xb2, 0x47, 0x18, 0x56, + 0x2d, 0x63, 0x33, 0x7b, 0xeb, 0x93, 0x8e, 0x86, 0x6f, 0x9e, 0xb0, 0xee, 0xe1, 0xf9, 0xec, 0xe7, + 0x9f, 0xbf, 0xbf, 0xda, 0x13, 0x3e, 0x8a, 0x53, 0x41, 0x82, 0x50, 0xd3, 0x5d, 0xeb, 0x86, 0xdd, + 0x43, 0xd7, 0x34, 0x8c, 0x4d, 0xed, 0xa1, 0x46, 0x2b, 0xc3, 0x57, 0x0d, 0xc6, 0x47, 0x4c, 0x6d, + 0x04, 0xf0, 0x5e, 0x8c, 0xeb, 0x9d, 0x32, 0xc7, 0x97, 0xd0, 0x77, 0x5d, 0x62, 0xcc, 0xda, 0xcf, + 0x3a, 0x18, 0xbe, 0x3e, 0xe3, 0x7c, 0x08, 0xb3, 0x21, 0x63, 0x3e, 0x88, 0x5d, 0x97, 0xfc, 0x14, + 0xa6, 0x24, 0x7e, 0x8a, 0x46, 0xb3, 0xfc, 0x14, 0xcd, 0x06, 0x35, 0xa6, 0x30, 0x15, 0xb9, 0x6b, + 0xdd, 0xac, 0xfa, 0xf6, 0x7f, 0xfd, 0xf8, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x79, 0x4a, 0x5f, 0xde, + 0xea, 0x03, 0x00, 0x00, +} diff --git a/examples/mexexample/api/mexexample.pb.gw.go b/examples/mexexample/api/mexexample.pb.gw.go new file mode 100644 index 000000000..a6138cc4b --- /dev/null +++ b/examples/mexexample/api/mexexample.pb.gw.go @@ -0,0 +1,258 @@ +// Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. +// source: api/mexexample.proto + +/* +Package api is a reverse proxy. + +It translates gRPC into RESTful JSON APIs. +*/ +package api + +import ( + "io" + "net/http" + + "github.com/golang/protobuf/proto" + "github.com/grpc-ecosystem/grpc-gateway/runtime" + "github.com/grpc-ecosystem/grpc-gateway/utilities" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/status" +) + +var _ codes.Code +var _ io.Reader +var _ status.Status +var _ = runtime.String +var _ = utilities.NewDoubleArray + +func request_MexExample_DataTest_0(ctx context.Context, marshaler runtime.Marshaler, client MexExampleClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq DataTestRequest + var metadata runtime.ServerMetadata + + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.DataTest(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +func request_MexExample_Echo_0(ctx context.Context, marshaler runtime.Marshaler, client MexExampleClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq EchoRequest + var metadata runtime.ServerMetadata + + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.Echo(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +func request_MexExample_Status_0(ctx context.Context, marshaler runtime.Marshaler, client MexExampleClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq StatusRequest + var metadata runtime.ServerMetadata + + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.Status(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +func request_MexExample_Info_0(ctx context.Context, marshaler runtime.Marshaler, client MexExampleClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq InfoRequest + var metadata runtime.ServerMetadata + + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.Info(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +// RegisterMexExampleHandlerFromEndpoint is same as RegisterMexExampleHandler but +// automatically dials to "endpoint" and closes the connection when "ctx" gets done. +func RegisterMexExampleHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { + conn, err := grpc.Dial(endpoint, opts...) + if err != nil { + return err + } + defer func() { + if err != nil { + if cerr := conn.Close(); cerr != nil { + grpclog.Printf("Failed to close conn to %s: %v", endpoint, cerr) + } + return + } + go func() { + <-ctx.Done() + if cerr := conn.Close(); cerr != nil { + grpclog.Printf("Failed to close conn to %s: %v", endpoint, cerr) + } + }() + }() + + return RegisterMexExampleHandler(ctx, mux, conn) +} + +// RegisterMexExampleHandler registers the http handlers for service MexExample to "mux". +// The handlers forward requests to the grpc endpoint over "conn". +func RegisterMexExampleHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { + return RegisterMexExampleHandlerClient(ctx, mux, NewMexExampleClient(conn)) +} + +// RegisterMexExampleHandler registers the http handlers for service MexExample to "mux". +// The handlers forward requests to the grpc endpoint over the given implementation of "MexExampleClient". +// Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "MexExampleClient" +// doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in +// "MexExampleClient" to call the correct interceptors. +func RegisterMexExampleHandlerClient(ctx context.Context, mux *runtime.ServeMux, client MexExampleClient) error { + + mux.Handle("POST", pattern_MexExample_DataTest_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_MexExample_DataTest_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_MexExample_DataTest_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + mux.Handle("POST", pattern_MexExample_Echo_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_MexExample_Echo_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_MexExample_Echo_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + mux.Handle("POST", pattern_MexExample_Status_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_MexExample_Status_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_MexExample_Status_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + mux.Handle("POST", pattern_MexExample_Info_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_MexExample_Info_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_MexExample_Info_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + return nil +} + +var ( + pattern_MexExample_DataTest_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"datatest"}, "")) + + pattern_MexExample_Echo_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"echo"}, "")) + + pattern_MexExample_Status_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"status"}, "")) + + pattern_MexExample_Info_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"info"}, "")) +) + +var ( + forward_MexExample_DataTest_0 = runtime.ForwardResponseMessage + + forward_MexExample_Echo_0 = runtime.ForwardResponseMessage + + forward_MexExample_Status_0 = runtime.ForwardResponseMessage + + forward_MexExample_Info_0 = runtime.ForwardResponseMessage +) diff --git a/examples/mexexample/api/mexexample.proto b/examples/mexexample/api/mexexample.proto new file mode 100644 index 000000000..0cd81a619 --- /dev/null +++ b/examples/mexexample/api/mexexample.proto @@ -0,0 +1,79 @@ +syntax = "proto3"; +package api; + +import "google/api/annotations.proto"; + +service MexExample { + rpc DataTest(DataTestRequest) returns (DataTestResponse) { + option (google.api.http) = { + post: "/datatest", + body: "*" + }; + } + rpc Echo(EchoRequest) returns (EchoResponse) { + option (google.api.http) = { + post: "/echo", + body: "*" + }; + } + rpc Status(StatusRequest) returns (StatusResponse) { + option (google.api.http) = { + post: "/status", + body: "*" + }; + } + rpc Info(InfoRequest) returns (InfoResponse) { + option (google.api.http) = { + post: "/info", + body: "*" + }; + } +} + +message DataTestRequest { + int64 numbytes = 1; +} + +message DataTestResponse { + string bytes = 1; +} + + +message EchoRequest { + string message = 1; +} + +message EchoResponse { + string message = 1; +} + + +message StatusRequest { + string message = 1; +} + +message StatusResponse { + string message = 1; + string status = 2; +} + +message InfoRequest { + string message = 1; +} + +message InfoResponse { + string message = 1; + string info = 2; + string outbound = 3; + string hostname = 4; + string realoutbound = 5; + uint64 totaltcp = 6; + uint64 totaludp = 7; + repeated Interface interfaces = 8; +} + +message Interface { + string name = 1; + string addresses = 2; +} + diff --git a/examples/mexexample/api/mexexample.swagger.json b/examples/mexexample/api/mexexample.swagger.json new file mode 100644 index 000000000..b652b2134 --- /dev/null +++ b/examples/mexexample/api/mexexample.swagger.json @@ -0,0 +1,230 @@ +{ + "swagger": "2.0", + "info": { + "title": "api/mexexample.proto", + "version": "version not set" + }, + "schemes": [ + "http", + "https" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "paths": { + "/datatest": { + "post": { + "operationId": "DataTest", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/apiDataTestResponse" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/apiDataTestRequest" + } + } + ], + "tags": [ + "MexExample" + ] + } + }, + "/echo": { + "post": { + "operationId": "Echo", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/apiEchoResponse" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/apiEchoRequest" + } + } + ], + "tags": [ + "MexExample" + ] + } + }, + "/info": { + "post": { + "operationId": "Info", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/apiInfoResponse" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/apiInfoRequest" + } + } + ], + "tags": [ + "MexExample" + ] + } + }, + "/status": { + "post": { + "operationId": "Status", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/apiStatusResponse" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/apiStatusRequest" + } + } + ], + "tags": [ + "MexExample" + ] + } + } + }, + "definitions": { + "apiDataTestRequest": { + "type": "object", + "properties": { + "numbytes": { + "type": "string", + "format": "int64" + } + } + }, + "apiDataTestResponse": { + "type": "object", + "properties": { + "bytes": { + "type": "string" + } + } + }, + "apiEchoRequest": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "apiEchoResponse": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "apiInfoRequest": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "apiInfoResponse": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "info": { + "type": "string" + }, + "outbound": { + "type": "string" + }, + "hostname": { + "type": "string" + }, + "realoutbound": { + "type": "string" + }, + "totaltcp": { + "type": "string", + "format": "uint64" + }, + "totaludp": { + "type": "string", + "format": "uint64" + }, + "interfaces": { + "type": "array", + "items": { + "$ref": "#/definitions/apiInterface" + } + } + } + }, + "apiInterface": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "addresses": { + "type": "string" + } + } + }, + "apiStatusRequest": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "apiStatusResponse": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "status": { + "type": "string" + } + } + } + } +} diff --git a/examples/mexexample/docker-build.sh b/examples/mexexample/docker-build.sh new file mode 100755 index 000000000..5dfcd6ca0 --- /dev/null +++ b/examples/mexexample/docker-build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# this has to be in shell script because we have to run at top level +cd ../../.. +pwd +docker build -t mobiledgex/mexexample -f cloud-resource-manager/example/mexexample/Dockerfile . + diff --git a/examples/mexexample/mexexample.go b/examples/mexexample/mexexample.go new file mode 100644 index 000000000..01e9da888 --- /dev/null +++ b/examples/mexexample/mexexample.go @@ -0,0 +1,94 @@ +package main + +import ( + "flag" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "syscall" + + log "github.com/sirupsen/logrus" + + "github.com/mobiledgex/edge-cloud/cloud-resource-manager/example/mexexample/server" +) + +var sigChan chan os.Signal + +func main() { + fmt.Println(os.Args) + debug := flag.Bool("debug", false, "debug") + grpcAddress := flag.String("grpc", ":27272", "GRPC address") + restAddress := flag.String("rest", ":27273", "REST API address") + httpAddress := flag.String("http", ":27274", "HTTP address") + tcpAddress := flag.String("tcp", ":27275", "TCP address") + udpAddress := flag.String("udp", ":27276", "UDP address") + flag.Parse() + if *debug { + log.SetLevel(log.DebugLevel) + } + log.Debugln("starting HTTP Server at", *httpAddress) + http.HandleFunc("/", frontpage) + go func() { + err := http.ListenAndServe(*httpAddress, nil) + if err != nil { + log.Fatal("cannot run HTTP Server", err) + } + }() + log.Debugf("starting REST Server at %s", *restAddress) //really just POST + go func() { + err := server.ListenAndServeREST(*restAddress, *grpcAddress) + if err != nil { + log.Fatalf("cannot run REST server, %v", err) + } + }() + log.Debugf("starting GRPC Server at %s", *grpcAddress) + go func() { + if err := server.ListenAndServeGRPC(*grpcAddress); err != nil { + log.Fatalf("cannot run GRPC server, %v", err) + } + }() + log.Debugf("starting TCP Server at %s", *tcpAddress) + go func() { + if err := server.ListenAndServeTCP(*tcpAddress); err != nil { + log.Fatalf("cannot run TCP server, %v", err) + } + }() + log.Debugf("starting UDP Server at %s", *udpAddress) + go func() { + if err := server.ListenAndServeUDP(*udpAddress); err != nil { + log.Fatalf("cannot run UDP server, %v", err) + } + }() + sigChan = make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + <-sigChan + os.Exit(0) +} + +func frontpage(w http.ResponseWriter, r *http.Request) { + log.Debugln("frontpage") + hn, err := os.Hostname() + if err == nil { + fmt.Fprintf(w, "hostname %s", hn) + } + conn, err := net.Dial("udp", "8.8.8.8:80") + if err == nil { + localAddr := conn.LocalAddr().(*net.UDPAddr) + fmt.Fprintf(w, "outbound ip %v", localAddr.IP) + } + conn.Close() // nolint + fmt.Fprintf(w, "real outbound ip %s", server.GetRealOutboundIP()) + fmt.Fprintf(w, "totaltcp %v", server.TotalTCP) + fmt.Fprintf(w, "totaludp %v", server.TotalUDP) + interfaces, err := net.Interfaces() + if err == nil { + for _, intf := range interfaces { + addrs, err := intf.Addrs() + if err == nil { + fmt.Fprintf(w, "%v %v ", intf, addrs) + } + } + } +} diff --git a/examples/mexexample/mexexample.yaml b/examples/mexexample/mexexample.yaml new file mode 100644 index 000000000..2ac72177c --- /dev/null +++ b/examples/mexexample/mexexample.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: Service +metadata: + name: mexexample-tcp-service + labels: + run: mexexample +spec: + type: LoadBalancer + ports: + - port: 27272 + targetPort: 27272 + protocol: TCP + name: grpc27272 + - port: 27273 + targetPort: 27273 + protocol: TCP + name: rest27273 + - port: 27274 + targetPort: 27274 + protocol: TCP + name: http27274 + - port: 27275 + targetPort: 27275 + protocol: TCP + name: tcp27275 + selector: + run: mexexample +--- +apiVersion: v1 +kind: Service +metadata: + name: mexexample-udp-service + labels: + run: mexexample +spec: + type: LoadBalancer + ports: + - port: 27276 + targetPort: 27276 + protocol: UDP + name: udp27276 + selector: + run: mexexample +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mexexample-deployment +spec: + selector: + matchLabels: + run: mexexample + replicas: 2 + template: + metadata: + labels: + run: mexexample + spec: + volumes: + imagePullSecrets: + - name: mexregistrysecret + containers: + - name: mexexample + image: registry.mobiledgex.net:5000/mobiledgex/mexexample + imagePullPolicy: Always + ports: + - containerPort: 27272 + protocol: TCP + - containerPort: 27273 + protocol: TCP + - containerPort: 27274 + protocol: TCP + - containerPort: 27275 + protocol: TCP + - containerPort: 27276 + protocol: UDP diff --git a/examples/mexexample/server/handlers.go b/examples/mexexample/server/handlers.go new file mode 100644 index 000000000..f59693969 --- /dev/null +++ b/examples/mexexample/server/handlers.go @@ -0,0 +1,95 @@ +package server + +import ( + "bytes" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + + "github.com/mobiledgex/edge-cloud/cloud-resource-manager/example/mexexample/api" + "golang.org/x/net/context" +) + +type Server struct{} + +func (srv *Server) DataTest(ctx context.Context, req *api.DataTestRequest) (res *api.DataTestResponse, err error) { + b := []byte("Z") + response := string(bytes.Repeat(b, int(req.Numbytes))) + res = &api.DataTestResponse{ + Bytes: response, + } + return res, nil +} + +func (srv *Server) Echo(ctx context.Context, req *api.EchoRequest) (res *api.EchoResponse, err error) { + res = &api.EchoResponse{ + Message: req.Message, + } + return res, nil +} + +func (srv *Server) Status(ctx context.Context, req *api.StatusRequest) (res *api.StatusResponse, err error) { + res = &api.StatusResponse{ + Message: req.Message, + Status: "OK", + } + return res, nil +} + +func (srv *Server) Info(ctx context.Context, req *api.InfoRequest) (res *api.InfoResponse, err error) { + res = &api.InfoResponse{ + Totaludp: TotalUDP, + Totaltcp: TotalTCP, + Message: req.Message, + Outbound: GetOutboundIP(), + Realoutbound: GetRealOutboundIP(), + Interfaces: []*api.Interface{}, + } + hn, err := os.Hostname() + if err != nil { + res.Message += fmt.Sprintf("error getting hostname, %v ", err) + } else { + res.Hostname = hn + } + interfaces, err := net.Interfaces() + if err != nil { + res.Message += fmt.Sprintf("error getting interfaces, %v ", err) + } + for _, intf := range interfaces { + addrs, err := intf.Addrs() + if err != nil { + res.Message += fmt.Sprintf("error getting addrs for intf %v, %v ", intf, err) + } else { + intfAddrs := fmt.Sprintf("%v", addrs) + res.Interfaces = append(res.Interfaces, &api.Interface{Name: intf.Name, Addresses: intfAddrs}) + } + } + return res, nil +} + +func GetOutboundIP() string { + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + return fmt.Sprintf("error getting outbound ip, %v", err) + } + defer conn.Close() // no lint + localAddr := conn.LocalAddr().(*net.UDPAddr) + return fmt.Sprintf("%v", localAddr.IP) +} + +func GetRealOutboundIP() string { + resp, err := http.Get("http://api.ipify.org") + if err != nil { + fmt.Println("error getting real outbound ip") + return "unavailable" + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("error reading resp body", err) + return "unavailable" + } + return fmt.Sprintf("%s", body) +} diff --git a/examples/mexexample/server/server.go b/examples/mexexample/server/server.go new file mode 100644 index 000000000..a48481571 --- /dev/null +++ b/examples/mexexample/server/server.go @@ -0,0 +1,128 @@ +package server + +import ( + "fmt" + "net" + "net/http" + "strings" + + "github.com/grpc-ecosystem/grpc-gateway/runtime" + "golang.org/x/net/context" + "google.golang.org/grpc" + + "github.com/mobiledgex/edge-cloud/cloud-resource-manager/example/mexexample/api" +) + +func ListenAndServeGRPC(address string) error { + opts := []grpc.ServerOption{} + grpcServer := grpc.NewServer(opts...) + s := Server{} + api.RegisterMexExampleServer(grpcServer, &s) + + lis, err := net.Listen("tcp", address) + if err != nil { + return fmt.Errorf("failed to listen on %s : %v", address, err) + } + + return grpcServer.Serve(lis) +} + +func headerMatcher(headerName string) (string, bool) { + return strings.ToLower(headerName), true +} + +func ListenAndServeREST(restAddress, grpcAddress string) error { + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + mux := runtime.NewServeMux(runtime.WithIncomingHeaderMatcher(headerMatcher)) + + opts := []grpc.DialOption{grpc.WithInsecure()} + + err := api.RegisterMexExampleHandlerFromEndpoint( + ctx, + mux, + grpcAddress, + opts, + ) + if err != nil { + return fmt.Errorf("could not register REST service: %s", err) + } + + return http.ListenAndServe(restAddress, mux) +} + +func ListenAndServeTCP(tcpAddress string) error { + la, err := net.Listen("tcp", tcpAddress) + if err != nil { + return err + } + defer la.Close() + for { + cl, err := la.Accept() + if err != nil { + return err + } + go handleTCP(cl) + } +} + +var TotalTCP = uint64(0) + +func handleTCP(cl net.Conn) { + defer cl.Close() + myip := GetRealOutboundIP() + buffer := make([]byte, 1024) + for { + nr, err := cl.Read(buffer) + if err != nil { + fmt.Println("error reading from tcp socket", err) + return + } + TotalTCP += uint64(nr) + dat := fmt.Sprintf("%s:%v:", myip, TotalTCP) + _, err = cl.Write([]byte(dat)) + if err != nil { + fmt.Println("error writing to tcp socket", err) + return + } + _, err = cl.Write(buffer) + if err != nil { + fmt.Println("error writing to tcp socket", err) + return + } + } +} + +var TotalUDP = uint64(0) + +func ListenAndServeUDP(udpAddress string) error { + uc, err := net.ListenPacket("udp", udpAddress) + if err != nil { + return err + } + defer uc.Close() + fmt.Println("reading udp", udpAddress) + myip := GetRealOutboundIP() + buffer := make([]byte, 1024) + for { + nr, addr, err := uc.ReadFrom(buffer) + if err != nil { + fmt.Println("udp read error", err) + return err + } + TotalUDP += uint64(nr) + dat := fmt.Sprintf("%s:%v:", myip, TotalUDP) + _, err = uc.WriteTo([]byte(dat), addr) + if err != nil { + fmt.Println("udp write error", err) + return err + } + _, err = uc.WriteTo(buffer, addr) + if err != nil { + fmt.Println("udp write error", err) + return err + } + } +} diff --git a/k8s-prov/install-k8s-base.sh b/k8s-prov/install-k8s-base.sh index 981fb25b1..f74297ca7 100755 --- a/k8s-prov/install-k8s-base.sh +++ b/k8s-prov/install-k8s-base.sh @@ -1,32 +1,75 @@ #!/bin/sh # must run as root # on both master and nodes -swapoff -a -apt-get update && apt-get install -y apt-transport-https curl unzip -apt-get install \ - apt-transport-https \ - ca-certificates \ - curl \ - software-properties-common -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -add-apt-repository \ +set -x +sudo swapoff -a +sudo apt-get update && apt-get install -y apt-transport-https curl unzip python +sudo apt-get install apt-transport-https ca-certificates curl software-properties-common +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +#curl -fsSL https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/docker-gpg | apt-key add - +sudo add-apt-repository \ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) \ stable" -apt-get update && apt-get install -y docker-ce -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - -cat </etc/apt/sources.list.d/kubernetes.list -deb http://apt.kubernetes.io/ kubernetes-xenial main -EOF -apt-get update && apt-get install -y kubelet kubeadm kubectl -sed -i "s/cgroup-driver=systemd/cgroup-driver=cgroupfs/g" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -systemctl daemon-reload -systemctl restart kubelet -wget --quiet https://dl.google.com/go/go1.10.2.linux-amd64.tar.gz -tar xf go1.10.2.linux-amd64.tar.gz -export PATH=`pwd`/go/bin:$PATH -export GOPATH=/usr/local -go get github.com/kubernetes-incubator/cri-tools/cmd/crictl +sudo apt-get update && apt-get install -y docker-ce +sudo which docker +if [ $? -ne 0 ]; then + echo docker install failed + exit 1 +fi +sudo curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +##curl -s https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/google-apt-key.gpg | apt-key add - +echo deb http://apt.kubernetes.io/ kubernetes-xenial main | sudo tee /etc/apt/sources.list.d/kubernetes.list +#apt-get update && apt-get install -y kubelet=1.12.4-00 kubeadm=1.12.4-00 kubectl=1.12.4-00 +# list of kubernetes versions are available at +# curl -s https://packages.cloud.google.com/apt/dists/kubernetes-xenial/main/binary-amd64/Packages | grep Version | awk '{print $2}' + +#apt-get update && apt-get install -y kubelet=1.11.2-00 kubeadm =1.11.2-00kubectl=1.11.2-00 +sudo apt-get update && apt-get install -y kubelet kubeadm kubectl +#curl -o /usr/bin/kubectl -s -LO https://storage.googleapis.com/kubernetes-release/release/v1.12.1/bin/linux/amd64/kubectl +#curl -o /usr/bin/kubeadm -s -LO https://storage.googleapis.com/kubernetes-release/release/v1.12.1/bin/linux/amd64/kubeadm +#curl -o /usr/bin/kubelet -s -LO https://storage.googleapis.com/kubernetes-release/release/v1.12.1/bin/linux/amd64/kubelet +sudo chmod a+rx /usr/bin/kubeadm /usr/bin/kubelet /usr/bin/kubectl +sudo which kubectl +if [ $? -ne 0 ]; then + echo kubectl not installed + exit 1 +fi +sudo which kubeadm +if [ $? -ne 0 ]; then + echo kubeadm not installed + exit 1 +fi +sudo which kubelet +if [ $? -ne 0 ]; then + echo kubelet not installed + exit 1 +fi +#curl -s -LO https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/kubelet.config.yaml +## v1.12.1 is looking for this config.yaml +#cp kubelet.config.yaml /var/lib/kubelet/config.yaml +sudo sed -i "s/cgroup-driver=systemd/cgroup-driver=cgroupfs/g" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +sudo systemctl daemon-reload +sudo systemctl restart kubelet +#wget --quiet https://dl.google.com/go/go1.10.2.linux-amd64.tar.gz +#tar xf go1.10.2.linux-amd64.tar.gz +#export PATH=`pwd`/go/bin:$PATH +#export GOPATH=/usr/local +#which go +#if [ $? -ne 0 ]; then +# echo go not installed +# exit 1 +#fi +#go get github.com/kubernetes-incubator/cri-tools/cmd/crictl +sudo curl https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/crictl -o /usr/local/bin/crictl +sudo chmod +x /usr/local/bin/crictl +sudo which crictl +if [ $? -ne 0 ]; then + echo crictl not installed + exit 1 +fi +sudo kubeadm config images pull +echo install-k8s-base.sh ok #mkdir -p /var/lib/consul #mkdir -p /usr/share/consul #mkdir -p /etc/consul/conf.d diff --git a/k8s-prov/install-k8s-master.sh b/k8s-prov/install-k8s-master.sh index 95a056ef5..c52d12a95 100755 --- a/k8s-prov/install-k8s-master.sh +++ b/k8s-prov/install-k8s-master.sh @@ -1,6 +1,7 @@ #!/bin/sh # must run as root # on the master +set -x if [ $# -lt 3 ]; then echo "Insufficient arguments" echo "Need interface-name master-ip my-ip" @@ -12,15 +13,57 @@ MYIP=$3 echo "Interface $INTF" echo "Master IP $MASTERIP" echo "My IP Address: $MYIP" -sudo apt install -y python +which python +if [ $? -ne 0 ]; then + echo python not installed + exit 1 +fi +which kubeadm +if [ $? -ne 0 ]; then + echo missing kubeadm + exit 1 +fi #nohup consul agent -server -bootstrap-expect=1 -data-dir=/tmp/consul -node=`hostname` -bind=$MYIP -syslog -config-dir=/etc/consul/conf.d & -kubeadm init --apiserver-advertise-address=$MYIP --pod-network-cidr=10.244.0.0/16 --ignore-preflight-errors=all +#kubeadm init --apiserver-advertise-address=$MYIP --pod-network-cidr=10.244.0.0/16 --ignore-preflight-errors=all +kubeadm init --apiserver-advertise-address=$MYIP --pod-network-cidr=192.168.0.0/16 --ignore-preflight-errors=all +if [ $? -ne 0 ]; then + echo kubeadm exited with error + exit 1 +fi #export KUBECONFIG=/etc/kubernetes/admin.conf -mkdir -p $HOME/.kube -sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config -sudo chown $(id -u):$(id -g) $HOME/.kube/config -kubectl apply -f https://raw.githubusercontent.com/projectcalico/canal/master/k8s-install/1.7/rbac.yaml -kubectl apply -f https://raw.githubusercontent.com/projectcalico/canal/master/k8s-install/1.7/canal.yaml +which kubectl +if [ $? -ne 0 ]; then + echo missing kubectl + exit 1 +fi +for d in /home/ubuntu /root; do + mkdir -p $d/.kube + cp /etc/kubernetes/admin.conf $d/.kube/config +done +chown ubuntu:ubuntu /home/ubuntu/.kube/config +export KUBECONFIG=/etc/kubernetes/admin.conf +kubectl version +while [ $? -ne 0 ] ; do + echo kubectl version failed + sleep 7 + kubectl version +done +#kubectl apply -f https://raw.githubusercontent.com/projectcalico/canal/master/k8s-install/1.7/rbac.yaml +#if [ $? -ne 0 ]; then +# echo kubectl exited with error installing rbac +# exit 1 +#fi +#kubectl apply -f https://raw.githubusercontent.com/projectcalico/canal/master/k8s-install/1.7/canal.yaml +# use fixed version. the original will fail validation +#kubectl apply -f https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/canal.yaml +#curl https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml -O +#POD_CIDR="10.244.0.0/16" sed -i -e "s?192.168.0.0/16?$POD_CIDR?g" calico.yaml +#kubectl apply -f calico.yaml +#if [ $? -ne 0 ]; then +# # echo kubectl exited with error installing canal +# echo kubectl exited with error installing canal +# exit 1 +#fi # the pod network plugin has to be done for coredns to come up kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" kubectl get pods --all-namespaces @@ -31,17 +74,16 @@ while [ $? -eq 0 ] ; do kubectl get nodes | grep NotReady done kubectl get nodes -kubeadm token create --print-join-command > /tmp/k8s-join-cmd +if [ $? -ne 0 ]; then + echo kubectl exited with error doing get nodes + exit 1 +fi +kubeadm token create --print-join-command | tee /tmp/k8s-join-cmd cat /tmp/k8s-join-cmd -mkdir -p $HOME/.kube -sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config -sudo chown $(id -u):$(id -g) $HOME/.kube/config -mkdir -p /root/.kube -sudo cp -i /etc/kubernetes/admin.conf /root/.kube/config -sudo chown root:root /root/.kube/config -cd /tmp -echo running simple http server at :8000 -python -m SimpleHTTPServer +#cd /tmp +#echo running simple http server at :8000 +#python -m SimpleHTTPServer #should not get here -echo done http server +#echo error returned from simple http server #consul kv put join-cmd "`cat /tmp/k8s-join-cmd`" +echo master ready diff --git a/k8s-prov/install-k8s-node.sh b/k8s-prov/install-k8s-node.sh index b0c297f46..802113291 100755 --- a/k8s-prov/install-k8s-node.sh +++ b/k8s-prov/install-k8s-node.sh @@ -1,7 +1,7 @@ #!/bin/sh # must be run as root # on all nodes - +set -x if [ $# -lt 3 ]; then echo "Insufficient arguments" echo "Need interface-name master-ip my-ip" @@ -32,18 +32,20 @@ echo "My IP Address: $MYIP" # sleep 7 # JOIN=`consul kv get join-cmd` #done -echo wait... +echo installing k8s node, wait... sleep 60 cd /tmp -wget http://$MASTERIP:8000/k8s-join-cmd +echo waiting for join-cmd +#wget http://$MASTERIP:8000/k8s-join-cmd +scp -i /etc/mobiledgex/id_rsa_mex $MASTERIP:/tmp/k8s-join-cmd . while [ $? -ne 0 ]; do - echo waiting for join-cmd sleep 7 - wget http://$MASTERIP:8000/k8s-join-cmd + #wget http://$MASTERIP:8000/k8s-join-cmd + sudo scp -i /etc/mobiledgex/id_rsa_mex ubuntu@$MASTERIP:/tmp/k8s-join-cmd . done -JOIN=`cat k8s-join-cmd` echo got join cmd +JOIN=`cat /tmp/k8s-join-cmd` cat k8s-join-cmd echo running $JOIN --ignore-preflight-errors=all $JOIN --ignore-preflight-errors=all -echo done running join +echo finished running join diff --git a/mexctl/.gitignore b/mexctl/.gitignore new file mode 100644 index 000000000..35f440a23 --- /dev/null +++ b/mexctl/.gitignore @@ -0,0 +1,46 @@ +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el diff --git a/mexctl/main.go b/mexctl/main.go new file mode 100644 index 000000000..f505d6b9d --- /dev/null +++ b/mexctl/main.go @@ -0,0 +1,146 @@ +package main + +import ( + "flag" + "fmt" + "os" + "reflect" + + sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud-infra/mexos" + "github.com/mobiledgex/edge-cloud/log" +) + +var clusterOps = map[string]func(*mexos.Manifest) error{ + "create": mexos.MEXClusterCreateManifest, + "remove": mexos.MEXClusterRemoveManifest, +} + +var platformOps = map[string]func(*mexos.Manifest) error{ + "create": mexos.MEXPlatformInitManifest, + "remove": mexos.MEXPlatformCleanManifest, +} + +var applicationOps = map[string]func(*mexos.Manifest) error{ + "create": mexos.MEXAppCreateAppManifest, + "remove": mexos.MEXAppDeleteAppManifest, +} + +var openstackOps = map[string]func(*mexos.Manifest) error{} + +var categories = map[string]map[string]func(*mexos.Manifest) error{ + "cluster": clusterOps, + "platform": platformOps, + "application": applicationOps, + "openstack": openstackOps, +} + +var mainflag = flag.NewFlagSet(os.Args[0], flag.ExitOnError) + +func printUsage() { + originalUsage() + fmt.Println("mex -manifest myvals.yaml {platform|cluster|application} {create|remove}") + fmt.Println("mex -manifest myvals.yaml openstack ...") +} + +var originalUsage func() + +func main() { + var err error + help := mainflag.Bool("help", false, "help") + debugLevels := mainflag.String("d", "", fmt.Sprintf("comma separated list of %v", log.DebugLevelStrings)) + base := mainflag.String("base", ".", "base containing templates, directory path or URI") + manifest := mainflag.String("manifest", "", "manifest") + originalUsage = mainflag.Usage + mainflag.Usage = printUsage + if err = mainflag.Parse(os.Args[1:]); err != nil { + log.InfoLog("parse error", "error", err) + os.Exit(1) + } + if *help { + printUsage() + os.Exit(0) + } + log.SetDebugLevelStrs(*debugLevels) + //XXX TODO make log to a remote server / aggregator + args := mainflag.Args() + if len(args) < 2 { + printUsage() + fmt.Println("insufficient args") + os.Exit(1) + } + _, ok := categories[args[0]] + if !ok { + printUsage() + fmt.Println("valid categories are", "categories", reflect.ValueOf(categories).MapKeys()) + os.Exit(1) + } + if *manifest == "" { + printUsage() + fmt.Println("missing manifest") + os.Exit(1) + } + if len(args) < 2 { + printUsage() + fmt.Println("insufficient args") + os.Exit(1) + } + log.DebugLog(log.DebugLevelMexos, "getting mf from manifest", "file", *manifest, "base", *base) + mf := &mexos.Manifest{Base: *base} + if err := mexos.GetVaultEnv(mf, *manifest); err != nil { + log.InfoLog("cannot get mf", "uri", *manifest, "error", err) + os.Exit(1) + } + kind := args[0] + if err := mexos.FillManifestValues(mf, kind, *base); err != nil { + log.InfoLog("cannot fill manifest", "error", err, "kind", kind, "base", *base) + os.Exit(1) + } + if err := mexos.CheckManifest(mf); err != nil { + log.InfoLog("incorrect manifest", "error", err) + os.Exit(1) + } + if _, err := mexos.NewRootLBManifest(mf); err != nil { + log.InfoLog("can't get new rootLB", "error", err) + os.Exit(1) + } + if err := mexos.MEXInit(mf); err != nil { + log.InfoLog("cannot init mex", "error", err) + os.Exit(1) + } + ops := args[1:] + log.DebugLog(log.DebugLevelMexos, "call", "kind", kind, "ops", ops) + err = callOps(mf, kind, ops...) + if err != nil { + log.InfoLog("ops failure", "kind", kind, "ops", ops, "error", err) + os.Exit(1) + } + os.Exit(0) +} + +func callOps(mf *mexos.Manifest, kind string, ops ...string) error { + if kind == "openstack" { + vs := make([]interface{}, len(ops)) + for i, v := range ops { + vs[i] = v + } + out, err := sh.Command("openstack", vs...).Output() + if err != nil { + return fmt.Errorf("error, openstack %v, %v", ops, err) + } + fmt.Println(string(out)) + return nil + } + if _, ok := categories[kind]; !ok { + return fmt.Errorf("invalid category %s", kind) + } + op := ops[0] + if _, ok := categories[kind][op]; !ok { + return fmt.Errorf("invalid category op %s", op) + } + err := categories[kind][op](mf) + if err != nil { + return err + } + return nil +} diff --git a/mexctl/mexctl-all.sh b/mexctl/mexctl-all.sh new file mode 100755 index 000000000..91afac790 --- /dev/null +++ b/mexctl/mexctl-all.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +if [ $# -lt 2 ]; then + echo "need stack and create/remove" + exit 1 +fi + +case "$2" in + create) + for i in platform cluster application; do + go run main.go -d mexos -stack $1 $i $2 + done + ;; + remove) + for i in application cluster platform ; do + go run main.go -d mexos -stack $1 $i $2 + done + ;; + *) + echo invalid operation, must be create or remove + exit 1 + ;; +esac +exit 0 + diff --git a/mexos/.gitignore b/mexos/.gitignore new file mode 100644 index 000000000..35f440a23 --- /dev/null +++ b/mexos/.gitignore @@ -0,0 +1,46 @@ +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el diff --git a/mexos/agent.go b/mexos/agent.go new file mode 100644 index 000000000..0bdfa9414 --- /dev/null +++ b/mexos/agent.go @@ -0,0 +1,222 @@ +package mexos + +import ( + "fmt" + "strings" + + valid "github.com/asaskevich/govalidator" + "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" + "github.com/mobiledgex/edge-cloud/log" +) + +//RunMEXAgentManifest runs the MEX agent on the RootLB. It first registers FQDN to cloudflare domain registry if not already registered. +// It then obtains certficiates from Letsencrypt, if not done yet. Then it runs the docker instance of MEX agent +// on the RootLB. It can be told to manually pull image from docker repository. This allows upgrading with new image. +// It uses MEX private docker repository. If an instance is running already, we don't start another one. +func RunMEXAgentManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "run mex agent") + fqdn := mf.Spec.RootLB + //fqdn is that of the machine/kvm-instance running the agent + if !valid.IsDNSName(fqdn) { + return fmt.Errorf("fqdn %s is not valid", fqdn) + } + if err := setPlatConfManifest(mf); err != nil { + return fmt.Errorf("can't set plat conf, %v", err) + } + sd, err := GetServerDetails(mf, fqdn) + if err == nil { + if sd.Name == fqdn { + log.DebugLog(log.DebugLevelMexos, "server with same name as rootLB exists", "fqdn", fqdn) + rootLB, err := getRootLB(fqdn) + if err != nil { + return fmt.Errorf("cannot find rootlb %s", fqdn) + } + //return RunMEXOSAgentContainer(mf, rootLB) + return RunMEXOSAgentService(mf, rootLB) + } + } + log.DebugLog(log.DebugLevelMexos, "about to create mex agent", "fqdn", fqdn) + rootLB, err := getRootLB(fqdn) + if err != nil { + return fmt.Errorf("cannot find rootlb %s", fqdn) + } + if rootLB == nil { + return fmt.Errorf("cannot run mex agent manifest, rootLB is null") + } + if mf.Spec.ExternalNetwork == "" { + return fmt.Errorf("missing external network") + } + if mf.Spec.Agent.Image == "" { + return fmt.Errorf("missing agent image") + } + if mf.Metadata.Name == "" { + return fmt.Errorf("missing name") + } + log.DebugLog(log.DebugLevelMexos, "record platform config") + err = EnableRootLB(mf, rootLB) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "can't enable agent", "name", rootLB.Name) + return fmt.Errorf("Failed to enable root LB %v", err) + } + err = WaitForRootLB(mf, rootLB) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "timeout waiting for agent to run", "name", rootLB.Name) + return fmt.Errorf("Error waiting for rootLB %v", err) + } + if err := SetupSSHUser(mf, rootLB, sshUser); err != nil { + return err + } + if err = ActivateFQDNA(mf, rootLB, rootLB.Name); err != nil { + return err + } + log.DebugLog(log.DebugLevelMexos, "FQDN A record activated", "name", rootLB.Name) + err = AcquireCertificates(mf, rootLB, rootLB.Name) //fqdn name may be different than rootLB.Name + if err != nil { + return fmt.Errorf("can't acquire certificate for %s, %v", rootLB.Name, err) + } + log.DebugLog(log.DebugLevelMexos, "acquired certificates from letsencrypt", "name", rootLB.Name) + //return RunMEXOSAgentContainer(mf, rootLB) + return RunMEXOSAgentService(mf, rootLB) +} + +func RunMEXOSAgentService(mf *Manifest, rootLB *MEXRootLB) error { + //TODO check if agent is running before restarting again. + log.DebugLog(log.DebugLevelMexos, "will run new mexosagent service") + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return err + } + for _, act := range []string{"stop", "disable"} { + out, err := client.Output("sudo systemctl " + act + " mexosagent.service") + if err != nil { + log.InfoLog("warning: cannot "+act+" mexosagent.service", "out", out, "err", err) + } + } + log.DebugLog(log.DebugLevelMexos, "copying new mexosagent service") + for _, dest := range []struct{ path, name string }{ + {"/usr/local/bin", "mexosagent"}, + {"/lib/systemd/system", "mexosagent.service"}, + } { + cmd := fmt.Sprintf("sudo scp -o %s -o %s -i id_rsa_mex mobiledgex@%s:files-repo/mobiledgex/%s %s", sshOpts[0], sshOpts[1], mf.Values.Registry.Name, dest.name, dest.path) + out, err := client.Output(cmd) + if err != nil { + log.InfoLog("error: cannot download from registry", "fn", dest.name, "path", dest.path, "error", err, "out", out) + return err + } + out, err = client.Output(fmt.Sprintf("sudo chmod a+rx %s/%s", dest.path, dest.name)) + if err != nil { + log.InfoLog("error: cannot chmod", "error", err, "fn", dest.name, "path", dest.path) + return err + } + } + log.DebugLog(log.DebugLevelMexos, "starting mexosagent.service") + for _, act := range []string{"enable", "start"} { + out, err := client.Output("sudo systemctl " + act + " mexosagent.service") + if err != nil { + log.InfoLog("warning: cannot "+act+" mexosagent.service", "out", out, "err", err) + } + } + log.DebugLog(log.DebugLevelMexos, "started mexosagent.service") + return nil +} + +func RunMEXOSAgentContainer(mf *Manifest, rootLB *MEXRootLB) error { + if mexEnv(mf, "MEX_DOCKER_REG_PASS") == "" { + return fmt.Errorf("empty docker registry pass env var") + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return err + } + //XXX rewrite this with --format {{.Names}} + cmd := fmt.Sprintf("docker ps --filter ancestor=%s --format {{.Names}}", mf.Spec.Agent.Image) + out, err := client.Output(cmd) + if err == nil && strings.Contains(out, rootLB.Name) { + //agent docker instance exists + //XXX check better + log.DebugLog(log.DebugLevelMexos, "agent docker instance already running") + return nil + } + cmd = fmt.Sprintf("echo %s > .docker-pass", mexEnv(mf, "MEX_DOCKER_REG_PASS")) + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("can't store docker pass, %s, %v", out, err) + } + log.DebugLog(log.DebugLevelMexos, "seeded docker registry password") + dockerinstanceName := fmt.Sprintf("%s-%s", mf.Metadata.Name, rootLB.Name) + if mf.Spec.DockerRegistry == "" { + log.DebugLog(log.DebugLevelMexos, "warning, empty docker registry spec, using default.") + mf.Spec.DockerRegistry = mf.Values.Registry.Docker + } + cmd = fmt.Sprintf("cat .docker-pass| docker login -u mobiledgex --password-stdin %s", mf.Spec.DockerRegistry) + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("error docker login at %s, %s, %s, %v", rootLB.Name, cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "docker login ok") + cmd = fmt.Sprintf("docker pull %s", mf.Spec.Agent.Image) //probably redundant + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("error pulling docker image at %s, %s, %s, %v", rootLB.Name, cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "pulled agent image ok") + cmd = fmt.Sprintf("docker run -d --rm --name %s --net=host -v `pwd`:/var/www/.cache -v /etc/ssl/certs:/etc/ssl/certs %s -debug", dockerinstanceName, mf.Spec.Agent.Image) + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("error running dockerized agent on RootLB %s, %s, %s, %v", rootLB.Name, cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "now running dockerized mexosagent") + return nil +} + +//UpdateMEXAgentManifest upgrades the mex agent +func UpdateMEXAgentManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "update mex agent") + err := RemoveMEXAgentManifest(mf) + if err != nil { + return err + } + // Force pulling a potentially newer docker image + return RunMEXAgentManifest(mf) +} + +//RemoveMEXAgentManifest deletes mex agent docker instance +func RemoveMEXAgentManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "deleting mex agent") + //XXX we are deleting server kvm!!! + err := DeleteServer(mf, mf.Spec.RootLB) + force := strings.Contains(mf.Spec.Flags, "force") + if err != nil { + if !force { + return err + } + log.DebugLog(log.DebugLevelMexos, "forced to continue, deleting mex agent error", "error", err, "rootLB", mf.Spec.RootLB) + } + log.DebugLog(log.DebugLevelMexos, "removed rootlb", "name", mf.Spec.RootLB) + sip, err := GetServerIPAddr(mf, mf.Values.Network.External, mf.Spec.RootLB) + if err := DeleteSecurityRule(mf, sip); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rule", "error", err, "server ip", sip) + } + if mf.Metadata.DNSZone == "" { + return fmt.Errorf("missing dns zone in manifest, metadata %v", mf.Metadata) + } + if cerr := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); cerr != nil { + return fmt.Errorf("cannot init cloudflare api, %v", cerr) + } + recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + fqdn := mf.Spec.RootLB + if derr != nil { + return fmt.Errorf("can not get dns records for %s, %v", fqdn, derr) + } + for _, rec := range recs { + if rec.Type == "A" && rec.Name == fqdn { + err = cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID) + if err != nil { + return fmt.Errorf("cannot delete dns record id %s Zone %s, %v", rec.ID, mf.Metadata.DNSZone, err) + } + } + } + log.DebugLog(log.DebugLevelMexos, "removed DNS A record", "FQDN", fqdn) + //TODO remove mex-k8s internal nets and router + return nil +} diff --git a/mexos/azure.go b/mexos/azure.go new file mode 100644 index 000000000..c1981a38d --- /dev/null +++ b/mexos/azure.go @@ -0,0 +1,36 @@ +package mexos + +import ( + "fmt" + "time" + + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/azure" + "github.com/mobiledgex/edge-cloud/log" +) + +func azureCreateAKS(mf *Manifest) error { + var err error + if err = azure.CreateResourceGroup(mf.Metadata.ResourceGroup, mf.Metadata.Location); err != nil { + return err + } + if err = azure.CreateAKSCluster(mf.Metadata.ResourceGroup, mf.Metadata.Name); err != nil { + return err + } + //race condition exists where the config file is not ready until just after the cluster create is done + time.Sleep(3 * time.Second) + saveKubeconfig() + if err = azure.GetAKSCredentials(mf.Metadata.ResourceGroup, mf.Metadata.Name); err != nil { + return err + } + kconf, err := GetKconf(mf, false) // XXX + if err != nil { + return fmt.Errorf("cannot get kconf, %v, %v, %v", mf, kconf, err) + } + log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX + //XXX watch out for multiple cluster contexts + if err = copyFile(defaultKubeconfig(), kconf); err != nil { + return fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) + } + log.DebugLog(log.DebugLevelMexos, "created aks", "name", mf.Spec.Key) + return nil +} diff --git a/mexos/basedata.go b/mexos/basedata.go new file mode 100644 index 000000000..f3c343229 --- /dev/null +++ b/mexos/basedata.go @@ -0,0 +1,14 @@ +package mexos + +import ( + "github.com/mobiledgex/edge-cloud/testutil" +) + +//OperatorData contains test data +var OperatorData = testutil.OperatorData + +//CloudletData contains test data +var CloudletData = testutil.CloudletData + +//CollectCloudletData is a dummy function +func CollectCloudletData() {} diff --git a/mexos/cloudflare.go b/mexos/cloudflare.go new file mode 100644 index 000000000..b00be60dc --- /dev/null +++ b/mexos/cloudflare.go @@ -0,0 +1,18 @@ +package mexos + +import ( + "fmt" + + "github.com/mobiledgex/edge-cloud/log" +) + +//CheckCredentialsCF checks for Cloudflare +func CheckCredentialsCF(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "check for cloudflare credentials") + for _, envname := range []string{"MEX_CF_KEY", "MEX_CF_USER"} { + if v := mexEnv(mf, envname); v == "" { + return fmt.Errorf("no env var for %s", envname) + } + } + return nil +} diff --git a/mexos/cluster.go b/mexos/cluster.go new file mode 100644 index 000000000..88a20641f --- /dev/null +++ b/mexos/cluster.go @@ -0,0 +1,384 @@ +package mexos + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/mobiledgex/edge-cloud/log" +) + +//XXX ClusterInst seems to have Nodes which is a number. +// The Nodes should be part of the Cluster flavor. And there should be Max nodes, and current num of nodes. +// Because the whole point of k8s and similar other clusters is the ability to expand. +// Cluster flavor defines what kind of cluster we have available for use. +// A medium cluster flavor may say "I have three nodes, ..." +// Can the Node or Flavor change for the ClusterInst? +// What needs to be done when contents change. +// The old and new values are supposedly to be passed in the future when Cache is updated. +// We have to compare old values and new values and figure out what changed. +// Then act on the changes noticed. +// There is no indication of type of cluster being created. So assume k8s. +// Nor is there any tenant information, so no ability to isolate, identify, account for usage or quota. +// And no network type information or storage type information. +// So if an app needs an external IP, we can't figure out if that is the case. +// Nor is there a way to return the IP address or DNS name. Or even know if it needs a DNS name. +// No ability to open ports, redirect or set up any kind of reverse proxy control. etc. + +//ClusterFlavor contains definitions of cluster flavor +type ClusterFlavor struct { + Kind string + Name string + PlatformFlavor string + Status string + NumNodes int + MaxNodes int + NumMasterNodes int + NetworkSpec string + StorageSpec string + NodeFlavor ClusterNodeFlavor + MasterFlavor ClusterMasterFlavor + Topology string +} + +//NetworkSpec examples: +// TYPE,NAME,CIDR,OPTIONS,EXTRAS +// "priv-subnet,mex-k8s-net-1,10.201.X.0/24,rp-dns-name" +// "external-ip,external-network-shared,1.2.3.4/8,dhcp" +// "external-ip,external-network-shared,1.2.3.4/8" +// "external-dns,external-network-shared,1.2.3.4/8,dns-name" +// "net-custom-type,some-name,8.8.244.33/16,auto-1" + +//StorageSpec examples: +// TYPE,NAME,PARAM,OPTIONS,EXTRAS +// ceph,internal-ceph-cluster,param1:param2:param3,opt1:opt2,extra1:extra2 +// nfs,nfsv4-internal,param1,opt1,extra1 +// gluster,glusterv3-ext,param1,opt1,extra1 +// postgres-cluster,post-v3,param1,opt1,extra1 + +//ClusterNodeFlavor contains details of flavor for the node +type ClusterNodeFlavor struct { + Type string + Name string +} + +//ClusterMasterFlavor contains details of flavor for the master node +type ClusterMasterFlavor struct { + Type string + Name string +} + +//mexCreateClusterKubernetes creates a cluster of nodes. It can take a while, so call from a goroutine. +func mexCreateClusterKubernetes(mf *Manifest) error { + //func mexCreateClusterKubernetes(mf *Manifest) (*string, error) { + log.DebugLog(log.DebugLevelMexos, "create kubernetes cluster", "cluster metadata", mf.Metadata, "spec", mf.Spec) + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("can't create kubernetes cluster, rootLB is null") + } + if mf.Spec.Flavor == "" { + return fmt.Errorf("empty cluster flavor") + } + cf, err := GetClusterFlavor(mf.Spec.Flavor) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "invalid platform flavor, can't create cluster") + return err + } + //TODO more than one networks + if mf.Spec.NetworkScheme == "" { + return fmt.Errorf("empty network spec") + } + if !strings.HasPrefix(mf.Spec.NetworkScheme, "priv-subnet") { + return fmt.Errorf("unsupported netSpec kind %s", mf.Spec.NetworkScheme) + // XXX for now + } + //TODO allow more net types + //TODO validate CIDR, etc. + if mf.Metadata.Tags == "" { + return fmt.Errorf("empty tag") + } + if mf.Metadata.Tenant == "" { + return fmt.Errorf("empty tenant") + } + err = ValidateTenant(mf.Metadata.Tenant) + if err != nil { + return fmt.Errorf("can't validate tenant, %v", err) + } + err = ValidateTags(mf.Metadata.Tags) + if err != nil { + return fmt.Errorf("invalid tag, %v", err) + } + mf.Metadata.Tags += "," + cf.PlatformFlavor + //TODO add whole manifest yaml->json into stringified property of the kvm instance for later + //XXX should check for quota, permissions, access control, etc. here + //guid := xid.New().String() + //kvmname := fmt.Sprintf("%s-1-%s-%s", "mex-k8s-master", mf.Metadata.Name, guid) + kvmname := fmt.Sprintf("%s-1-%s", "mex-k8s-master", mf.Metadata.Name) + sd, err := GetServerDetails(mf, kvmname) + if err == nil { + if sd.Name == kvmname { + log.DebugLog(log.DebugLevelMexos, "k8s master exists", "kvmname", kvmname) + return nil + } + } + log.DebugLog(log.DebugLevelMexos, "proceed to create k8s master kvm", "kvmname", kvmname, "netspec", mf.Spec.NetworkScheme, "tags", mf.Metadata.Tags, "flavor", cf.PlatformFlavor) + err = CreateMEXKVM(mf, kvmname, + "k8s-master", + mf.Spec.NetworkScheme, + mf.Metadata.Tags, + mf.Metadata.Tenant, + 1, + cf.PlatformFlavor, + ) + if err != nil { + return fmt.Errorf("can't create k8s master, %v", err) + } + for i := 1; i <= cf.NumNodes; i++ { + //construct node name + //kvmnodename := fmt.Sprintf("%s-%d-%s-%s", "mex-k8s-node", i, mf.Metadata.Name, guid) + kvmnodename := fmt.Sprintf("%s-%d-%s", "mex-k8s-node", i, mf.Metadata.Name) + err = CreateMEXKVM(mf, kvmnodename, + "k8s-node", + mf.Spec.NetworkScheme, + mf.Metadata.Tags, + mf.Metadata.Tenant, + i, + cf.PlatformFlavor, + ) + if err != nil { + return fmt.Errorf("can't create k8s node, %v", err) + } + } + if mf.Values.Network.External == "" { + return fmt.Errorf("missing external network in platform config") + } + if err = LBAddRoute(mf, rootLB.Name, mf.Values.Network.External, kvmname); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add route on rootlb", "error", err) + //return err + } + if err = SetServerProperty(mf, kvmname, "mex-flavor="+mf.Spec.Flavor); err != nil { + return err + } + ready := false + for i := 0; i < 10; i++ { + ready, err = IsClusterReady(mf, rootLB) + if err != nil { + return err + } + if ready { + log.DebugLog(log.DebugLevelMexos, "kubernetes cluster ready") + break + } + log.DebugLog(log.DebugLevelMexos, "waiting for kubernetes cluster to be ready...") + time.Sleep(30 * time.Second) + } + if !ready { + return fmt.Errorf("cluster not ready (yet)") + } + if err := SeedDockerSecret(mf, rootLB); err != nil { + return err + } + if mf.Metadata.Swarm != "" { + log.DebugLog(log.DebugLevelMexos, "metadata swarm is set, creating docker swarm", "swarm", mf.Metadata.Swarm) + if err := CreateDockerSwarm(mf, rootLB); err != nil { + return err + } + } + if err := CreateDockerRegistrySecret(mf); err != nil { + return err + } + //return &guid, nil + return nil +} + +//mexDeleteClusterKubernetes deletes kubernetes cluster +func mexDeleteClusterKubernetes(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "deleting kubernetes cluster") + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("can't delete kubernetes cluster, rootLB is null") + } + name := mf.Metadata.Name + if name == "" { + log.DebugLog(log.DebugLevelMexos, "error, empty name") + return fmt.Errorf("empty name") + } + srvs, err := ListServers(mf) + if err != nil { + return err + } + log.DebugLog(log.DebugLevelMexos, "looking for server", "name", name, "servers", srvs) + force := strings.Contains(mf.Spec.Flags, "force") + serverDeleted := false + for _, s := range srvs { + if strings.Contains(s.Name, name) { + if !strings.HasPrefix(s.Name, "mex-k8s-") { + continue + } + if strings.Contains(s.Name, "mex-k8s-master") { + err = LBRemoveRoute(mf, rootLB.Name, mf.Values.Network.External, s.Name) + if err != nil { + if !force { + err = fmt.Errorf("failed to remove route for %s, %v", s.Name, err) + log.DebugLog(log.DebugLevelMexos, "failed to remove route", "name", s.Name, "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "forced to continue, failed to remove route ", "name", s.Name, "error", err) + } + } + log.DebugLog(log.DebugLevelMexos, "delete kubernetes server", "name", s.Name) + err = DeleteServer(mf, s.Name) + if err != nil { + if !force { + log.DebugLog(log.DebugLevelMexos, "delete server fail", "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "forced to continue, error while deleting server", "server", s.Name, "error", err) + } + serverDeleted = true + //kconfname := fmt.Sprintf("%s.kubeconfig", s.Name[strings.LastIndex(s.Name, "-")+1:]) + kconfname := GetLocalKconfName(mf) + rerr := os.Remove(kconfname) + if rerr != nil { + log.DebugLog(log.DebugLevelMexos, "error can't remove file", "name", kconfname, "error", rerr) + } + kconfname += "-proxy" + rerr = os.Remove(kconfname) + if rerr != nil { + log.DebugLog(log.DebugLevelMexos, "error can't remove file", "name", kconfname, "error", rerr) + } + } + } + if !serverDeleted { + log.DebugLog(log.DebugLevelMexos, "server not found", "name", name) + return fmt.Errorf("no server with name %s", name) + } + sns, err := ListSubnets(mf, "") + if err != nil { + log.DebugLog(log.DebugLevelMexos, "can't list subnets", "error", err) + return err + } + rn := GetMEXExternalRouter(mf) //XXX for now + for _, s := range sns { + if strings.Contains(s.Name, name) { + rerr := RemoveRouterSubnet(mf, rn, s.Name) + if rerr != nil { + log.DebugLog(log.DebugLevelMexos, "not fatal, continue, can't remove router from subnet", "error", rerr) + //return rerr + } + err = DeleteSubnet(mf, s.Name) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, error while deleting subnet", "error", err) + } + break + } + } + //XXX tell agent to remove the route + //XXX remove kubectl proxy instance + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return err + } + cmd := "sudo ps wwh -C kubectl -o pid,args" + out, err := client.Output(cmd) + if err != nil || out == "" { + return nil // no kubectl running + } + lines := strings.Split(out, "\n") + for _, ln := range lines { + pidnum := parseKCPid(ln, mf.Spec.Key) + if pidnum == 0 { + continue + } + cmd = fmt.Sprintf("sudo kill -9 %d", pidnum) + out, err = client.Output(cmd) + if err != nil { + log.InfoLog("error killing kubectl proxy", "command", cmd, "out", out, "error", err) + } else { + log.DebugLog(log.DebugLevelMexos, "killed kubectl proxy", "line", ln, "cmd", cmd) + } + return nil + } + return nil +} + +//IsClusterReady checks to see if cluster is read, i.e. rootLB is running and active +func IsClusterReady(mf *Manifest, rootLB *MEXRootLB) (bool, error) { + log.DebugLog(log.DebugLevelMexos, "checking if cluster is ready") + if rootLB == nil { + return false, fmt.Errorf("cannot check if cluster is ready, rootLB is null") + } + cf, err := GetClusterFlavor(mf.Spec.Flavor) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "invalid cluster flavor, can't check if cluster is ready") + return false, err + } + name, err := FindClusterWithKey(mf, mf.Spec.Key) + if err != nil { + return false, fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) + } + ipaddr, err := FindNodeIP(mf, name) + if err != nil { + return false, err + } + if mf.Values.Network.External == "" { + return false, fmt.Errorf("is cluster ready, missing external network in platform config") + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return false, fmt.Errorf("can't get ssh client for cluser ready check, %v", err) + } + log.DebugLog(log.DebugLevelMexos, "checking master k8s node for available nodes", "ipaddr", ipaddr) + cmd := fmt.Sprintf("ssh -o %s -o %s -o %s -i id_rsa_mex %s@%s kubectl get nodes -o json", sshOpts[0], sshOpts[1], sshOpts[2], sshUser, ipaddr) + log.DebugLog(log.DebugLevelMexos, "running kubectl get nodes", "cmd", cmd) + out, err := client.Output(cmd) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error checking for kubernetes nodes", "out", out, "err", err) + return false, nil //This is intentional + } + gitems := &genericItems{} + err = json.Unmarshal([]byte(out), gitems) + if err != nil { + return false, fmt.Errorf("failed to json unmarshal kubectl get nodes output, %v, %v", err, out) + } + log.DebugLog(log.DebugLevelMexos, "kubectl reports nodes", "numnodes", len(gitems.Items)) + if len(gitems.Items) < (cf.NumNodes + cf.NumMasterNodes) { + //log.DebugLog(log.DebugLevelMexos, "kubernetes cluster not ready", "log", out) + log.DebugLog(log.DebugLevelMexos, "kubernetes cluster not ready", "len items", len(gitems.Items)) + return false, nil + } + log.DebugLog(log.DebugLevelMexos, "cluster nodes", "numnodes", cf.NumNodes, "nummasters", cf.NumMasterNodes) + //kcpath := MEXDir() + "/" + name[strings.LastIndex(name, "-")+1:] + ".kubeconfig" + if err := CopyKubeConfig(mf, rootLB, name); err != nil { + return false, fmt.Errorf("kubeconfig copy failed, %v", err) + } + log.DebugLog(log.DebugLevelMexos, "cluster ready.") + return true, nil +} + +//FindClusterWithKey finds cluster given a key string +func FindClusterWithKey(mf *Manifest, key string) (string, error) { + log.DebugLog(log.DebugLevelMexos, "find cluster with key", "key", key) + if key == "" { + return "", fmt.Errorf("empty key") + } + srvs, err := ListServers(mf) + if err != nil { + return "", err + } + for _, s := range srvs { + if s.Status == "ACTIVE" && strings.HasSuffix(s.Name, key) && strings.HasPrefix(s.Name, "mex-k8s-master") { + log.DebugLog(log.DebugLevelMexos, "find cluster with key", "key", key, "found", s.Name) + return s.Name, nil + } + } + return "", fmt.Errorf("key %s not found", key) +} diff --git a/mexos/const.go b/mexos/const.go new file mode 100644 index 000000000..8d0b3d1cb --- /dev/null +++ b/mexos/const.go @@ -0,0 +1,20 @@ +package mexos + +const APIversion = "v1" + +const MEXSubnetSeed = 100 +const MEXSubnetLimit = 250 + +const ( + k8smasterRole = "k8s-master" + k8snodeRole = "k8s-node" +) + +//For netspec components +// netType,netName,netCIDR,netOptions +const ( + NetTypeVal = 0 + NetNameVal = 1 + NetCIDRVal = 2 + NetOptVal = 3 +) diff --git a/mexos/dns.go b/mexos/dns.go new file mode 100644 index 000000000..9bf8e3ee6 --- /dev/null +++ b/mexos/dns.go @@ -0,0 +1,263 @@ +package mexos + +import ( + "encoding/json" + "fmt" + "net" + "strings" + "time" + + "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" + "github.com/mobiledgex/edge-cloud/cloudcommon" + "github.com/mobiledgex/edge-cloud/log" +) + +var dnsRegisterRetryDelay time.Duration = 3 * time.Second + +func createAppDNS(mf *Manifest, kconf string) error { + if mf.Metadata.Operator != "gcp" && mf.Metadata.Operator != "azure" { + return fmt.Errorf("error, invalid code path") + } + if err := CheckCredentialsCF(mf); err != nil { + return err + } + if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } + if mf.Spec.URI == "" { + return fmt.Errorf("URI not specified %v", mf) + } + err := validateDomain(mf.Spec.URI) + if err != nil { + return err + } + if mf.Metadata.DNSZone == "" { + return fmt.Errorf("missing DNS zone, metadata %v", mf.Metadata) + } + serviceNames, err := getSvcNames(mf.Metadata.Name, kconf) + if err != nil { + return err + } + if len(serviceNames) < 1 { + return fmt.Errorf("no service names starting with %s", mf.Metadata.Name) + } + recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + if derr != nil { + return fmt.Errorf("error getting dns records for %s, %v", mf.Metadata.DNSZone, err) + } + fqdnBase := uri2fqdn(mf.Spec.URI) + for _, sn := range serviceNames { + externalIP, err := getSvcExternalIP(sn, kconf) + if err != nil { + return err + } + fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) + for _, rec := range recs { + if rec.Type == "A" && rec.Name == fqdn { + if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID); err != nil { + return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) + } + log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) + } + } + if err := cloudflare.CreateDNSRecord(mf.Metadata.DNSZone, fqdn, "A", externalIP, 1, false); err != nil { + return fmt.Errorf("can't create DNS record for %s,%s, %v", fqdn, externalIP, err) + } + //log.DebugLog(log.DebugLevelMexos, "waiting for DNS record to be created on cloudflare...") + //err = WaitforDNSRegistration(fqdn) + //if err != nil { + // return err + //} + log.DebugLog(log.DebugLevelMexos, "registered DNS name, may still need to wait for propagation", "name", fqdn, "externalIP", externalIP) + } + return nil +} + +func deleteAppDNS(mf *Manifest, kconf string) error { + if mf.Metadata.Operator != "gcp" && mf.Metadata.Operator != "azure" { + return fmt.Errorf("error, invalid code path") + } + if err := CheckCredentialsCF(mf); err != nil { + return err + } + if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } + if mf.Spec.URI == "" { + return fmt.Errorf("URI not specified %v", mf) + } + err := validateDomain(mf.Spec.URI) + if err != nil { + return err + } + if mf.Metadata.DNSZone == "" { + return fmt.Errorf("missing DNS zone, metadata %v", mf.Metadata) + } + serviceNames, err := getSvcNames(mf.Metadata.Name, kconf) + if err != nil { + return err + } + if len(serviceNames) < 1 { + return fmt.Errorf("no service names starting with %s", mf.Metadata.Name) + } + recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + if derr != nil { + return fmt.Errorf("cannot get dns records for dns zone %s, error %v", mf.Metadata.DNSZone, err) + } + fqdnBase := uri2fqdn(mf.Spec.URI) + for _, sn := range serviceNames { + fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) + for _, rec := range recs { + if rec.Type == "A" && rec.Name == fqdn { + if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID); err != nil { + return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) + } + } + } + log.DebugLog(log.DebugLevelMexos, "deleted DNS name", "name", fqdn) + } + return nil +} + +func KubeAddDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error { + log.DebugLog(log.DebugLevelMexos, "adding dns records for kubenernets app", "name", mf.Metadata.Name) + rootLBIPaddr, err := GetServerIPAddr(mf, mf.Values.Network.External, rootLB.Name) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot get rootlb IP address", "error", err) + return fmt.Errorf("cannot deploy kubernetes app, cannot get rootlb IP") + } + cmd := fmt.Sprintf("%s kubectl get svc -o json", kp.kubeconfig) + out, err := kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("can not get list of services, %s, %v", out, err) + } + svcs := &svcItems{} + err = json.Unmarshal([]byte(out), svcs) + if err != nil { + return fmt.Errorf("can not unmarshal svc json, %v", err) + } + if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } + recs, err := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + if err != nil { + return fmt.Errorf("error getting dns records for %s, %v", mf.Metadata.DNSZone, err) + } + log.DebugLog(log.DebugLevelMexos, "number of cloudflare dns recs", "dns recs count", len(recs)) + fqdnBase := uri2fqdn(mf.Spec.URI) + log.DebugLog(log.DebugLevelMexos, "kubernetes services", "services", svcs) + processed := 0 + for _, item := range svcs.Items { + if !strings.HasPrefix(item.Metadata.Name, mf.Metadata.Name) { + continue + } + cmd = fmt.Sprintf(`%s kubectl patch svc %s -p '{"spec":{"externalIPs":["%s"]}}'`, kp.kubeconfig, item.Metadata.Name, kp.ipaddr) + out, err = kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error patching for kubernetes service, %s, %s, %v", cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "patched externalIPs on service", "service", item.Metadata.Name, "externalIPs", kp.ipaddr) + fqdn := cloudcommon.ServiceFQDN(item.Metadata.Name, fqdnBase) + for _, rec := range recs { + if rec.Type == "A" && rec.Name == fqdn { + if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID); err != nil { + return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) + } + log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) + } + } + if err := cloudflare.CreateDNSRecord(mf.Metadata.DNSZone, fqdn, "A", rootLBIPaddr, 1, false); err != nil { + return fmt.Errorf("can't create DNS record for %s,%s, %v", fqdn, rootLBIPaddr, err) + } + processed++ + log.DebugLog(log.DebugLevelMexos, "created DNS record", "name", fqdn, "addr", rootLBIPaddr) + } + if processed == 0 { + return fmt.Errorf("cannot patch service, %s not found", mf.Metadata.Name) + } + return nil +} + +func KubeDeleteDNSRecords(rootLB *MEXRootLB, mf *Manifest, kp *kubeParam) error { + //TODO before removing dns records, especially for the purpose of creating + // a dns entry that was there before, to overwrite, we need to check + // if the user really wants to. For example, if the cluster naming was in error, + // it would be bad to overwrite working existing cluster dns. + cmd := fmt.Sprintf("%s kubectl get svc -o json", kp.kubeconfig) + out, err := kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("can not get list of services, %s, %v", out, err) + } + svcs := &svcItems{} + err = json.Unmarshal([]byte(out), svcs) + if err != nil { + return fmt.Errorf("can not unmarshal svc json, %v", err) + } + if cerr := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); cerr != nil { + return fmt.Errorf("cannot init cloudflare api, %v", cerr) + } + recs, derr := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + if derr != nil { + return fmt.Errorf("error getting dns records for %s, %v", mf.Metadata.DNSZone, derr) + } + fqdnBase := uri2fqdn(mf.Spec.URI) + //FIXME use k8s manifest file to delete the whole services and deployments + for _, item := range svcs.Items { + if !strings.HasPrefix(item.Metadata.Name, mf.Metadata.Name) { + continue + } + // cmd := fmt.Sprintf("%s kubectl delete service %s", kp.kubeconfig, item.Metadata.Name) + // out, err := kp.client.Output(cmd) + // if err != nil { + // log.DebugLog(log.DebugLevelMexos, "error deleting kubernetes service", "name", item.Metadata.Name, "cmd", cmd, "out", out, "err", err) + // } else { + // log.DebugLog(log.DebugLevelMexos, "deleted service", "name", item.Metadata.Name) + // } + fqdn := cloudcommon.ServiceFQDN(item.Metadata.Name, fqdnBase) + for _, rec := range recs { + if rec.Type == "A" && rec.Name == fqdn { + if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, rec.ID); err != nil { + return fmt.Errorf("cannot delete existing DNS record %v, %v", rec, err) + } + log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) + } + } + } + cmd = fmt.Sprintf("%s kubectl delete -f %s.yaml", kp.kubeconfig, mf.Metadata.Name) + // cmd = fmt.Sprintf("%s kubectl delete deploy %s", kp.kubeconfig, mf.Metadata.Name+"-deployment") + out, err = kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error deleting kuberknetes app, %s, %s, %s, %v", mf.Metadata.Name, cmd, out, err) + } + + return nil +} + +func LookupDNS(name string) (string, error) { + ips, err := net.LookupIP(name) + if err != nil { + return "", fmt.Errorf("DNS lookup error, %s, %v", name, err) + } + if len(ips) == 0 { + return "", fmt.Errorf("no DNS records, %s", name) + } + for _, ip := range ips { + return ip.String(), nil //XXX return only first one + } + return "", fmt.Errorf("no IP in DNS record for %s", name) +} + +func WaitforDNSRegistration(name string) error { + var ipa string + var err error + + for i := 0; i < 100; i++ { + ipa, err = LookupDNS(name) + if err == nil && ipa != "" { + return nil + } + time.Sleep(dnsRegisterRetryDelay) + } + log.DebugLog(log.DebugLevelMexos, "DNS lookup timed out", "name", name) + return fmt.Errorf("error, timed out while looking up DNS for name %s", name) +} diff --git a/mexos/env.go b/mexos/env.go new file mode 100644 index 000000000..d958e8067 --- /dev/null +++ b/mexos/env.go @@ -0,0 +1,32 @@ +package mexos + +import ( + "fmt" + + "github.com/mobiledgex/edge-cloud/log" +) + +//MEXCheckEnvVars sets up environment vars and checks for credentials required for running +func MEXCheckEnvVars(mf *Manifest) error { + // secrets to be passed via Env var still : MEX_CF_KEY, MEX_CF_USER, MEX_DOCKER_REG_PASS + // TODO: use `secrets` or `vault` + for _, evar := range []string{ + "MEX_CF_KEY", + "MEX_CF_USER", + "MEX_DOCKER_REG_PASS", + } { + if v := mexEnv(mf, evar); v == "" { + return fmt.Errorf("missing env var %s", evar) + } + } + return nil +} + +func mexEnv(mf *Manifest, name string) string { + v, ok := mf.Values.VaultEnvMap[name] + if !ok { + log.DebugLog(log.DebugLevelMexos, "error, env var not exist in vault", "name", name) + return "" + } + return v +} diff --git a/mexos/flavor.go b/mexos/flavor.go new file mode 100644 index 000000000..f6724c2cc --- /dev/null +++ b/mexos/flavor.go @@ -0,0 +1,84 @@ +package mexos + +import ( + "fmt" + + "github.com/mobiledgex/edge-cloud/log" +) + +//AvailableClusterFlavors lists currently available flavors +var AvailableClusterFlavors = []*ClusterFlavor{ + &ClusterFlavor{ + Name: "x1.tiny", + Kind: "mex-cluster-flavor", + PlatformFlavor: "m4.small", + Status: "active", + NumNodes: 2, + NumMasterNodes: 1, + Topology: "type-1", + NetworkSpec: "priv-subnet,mex-k8s-net-1," + defaultPrivateNetRange, + StorageSpec: "default", + NodeFlavor: ClusterNodeFlavor{Name: "k8s-tiny", Type: "k8s-node"}, + MasterFlavor: ClusterMasterFlavor{Name: "k8s-tiny", Type: "k8s-master"}, + }, + &ClusterFlavor{ + Name: "x1.small", + Kind: "mex-cluster-flavor", + PlatformFlavor: "m4.medium", + Status: "active", + NumNodes: 2, + NumMasterNodes: 1, + Topology: "type-1", + NetworkSpec: "priv-subnet,mex-k8s-net-1," + defaultPrivateNetRange, + StorageSpec: "default", + NodeFlavor: ClusterNodeFlavor{Name: "k8s-small", Type: "k8s-node"}, + MasterFlavor: ClusterMasterFlavor{Name: "k8s-small", Type: "k8s-master"}, + }, + &ClusterFlavor{ + Name: "x1.medium", + Kind: "mex-cluster-flavor", + PlatformFlavor: "m4.large", + Status: "active", + NumNodes: 2, + NumMasterNodes: 1, + Topology: "type-1", + NetworkSpec: "priv-subnet,mex-k8s-net-1," + defaultPrivateNetRange, + StorageSpec: "default", + NodeFlavor: ClusterNodeFlavor{Name: "k8s-medium", Type: "k8s-node"}, + MasterFlavor: ClusterMasterFlavor{Name: "k8s-medium", Type: "k8s-master"}, + }, + &ClusterFlavor{ + Name: "x1.large", + Kind: "mex-cluster-flavor", + PlatformFlavor: "m4.xlarge", + Status: "active", + NumNodes: 2, + NumMasterNodes: 1, + Topology: "type-1", + NetworkSpec: "priv-subnet,mex-k8s-net-1," + defaultPrivateNetRange, + StorageSpec: "default", + NodeFlavor: ClusterNodeFlavor{Name: "k8s-large", Type: "k8s-node"}, + MasterFlavor: ClusterMasterFlavor{Name: "k8s-large", Type: "k8s-master"}, + }, +} + +func AddFlavorManifest(mf *Manifest) error { + _, err := GetClusterFlavor(mf.Spec.Flavor) + if err != nil { + return err + } + // Adding flavors in platforms cannot be done dynamically. For example, x1.xlarge cannot be + // implemented in currently DT cloudlets. Controller can learn what flavors available. Not create new ones. + return nil +} + +func GetClusterFlavor(flavor string) (*ClusterFlavor, error) { + log.DebugLog(log.DebugLevelMexos, "get cluster flavor details", "cluster flavor", flavor) + for _, af := range AvailableClusterFlavors { + if af.Name == flavor { + log.DebugLog(log.DebugLevelMexos, "using cluster flavor", "cluster flavor", af) + return af, nil + } + } + return nil, fmt.Errorf("unsupported cluster flavor %s", flavor) +} diff --git a/mexos/fqdn.go b/mexos/fqdn.go new file mode 100644 index 000000000..05ed3ed98 --- /dev/null +++ b/mexos/fqdn.go @@ -0,0 +1,111 @@ +package mexos + +import ( + "fmt" + "strings" + + "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" + "github.com/mobiledgex/edge-cloud/log" +) + +func isDomainName(s string) bool { + l := len(s) + if l == 0 || l > 254 || l == 254 && s[l-1] != '.' { + return false + } + + last := byte('.') + ok := false // Ok once we've seen a letter. + partlen := 0 + for i := 0; i < len(s); i++ { + c := s[i] + switch { + default: + return false + case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_': + ok = true + partlen++ + case '0' <= c && c <= '9': + // fine + partlen++ + case c == '-': + // Byte before dash cannot be dot. + if last == '.' { + return false + } + partlen++ + case c == '.': + // Byte before dot cannot be dot, dash. + if last == '.' || last == '-' { + return false + } + if partlen > 63 || partlen == 0 { + return false + } + partlen = 0 + } + last = c + } + if last == '-' || partlen > 63 { + return false + } + + return ok +} + +func uri2fqdn(uri string) string { + fqdn := strings.Replace(uri, "http://", "", 1) + fqdn = strings.Replace(fqdn, "https://", "", 1) + //XXX assumes no trailing elements + return fqdn +} + +//ActivateFQDNA updates and ensures FQDN is registered properly +func ActivateFQDNA(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { + if rootLB == nil { + return fmt.Errorf("cannot activate certs, rootLB is null") + } + + if mf.Values.Network.External == "" { + return fmt.Errorf("activate fqdn A record, missing external network in manifest") + } + if err := CheckCredentialsCF(mf); err != nil { + return err + } + if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } + log.DebugLog(log.DebugLevelMexos, "getting dns record for zone", "DNSZone", mf.Metadata.DNSZone) + dr, err := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + if err != nil { + return fmt.Errorf("cannot get dns records for %s, %v", fqdn, err) + } + addr, err := GetServerIPAddr(mf, mf.Values.Network.External, fqdn) + for _, d := range dr { + if d.Type == "A" && d.Name == fqdn { + if d.Content == addr { + log.DebugLog(log.DebugLevelMexos, "existing A record", "FQDN", fqdn, "addr", addr) + return nil + } + log.DebugLog(log.DebugLevelMexos, "cloudflare A record has different address, it will be overwritten", "existing", d, "addr", addr) + if err = cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, d.ID); err != nil { + return fmt.Errorf("can't delete DNS record for %s, %v", fqdn, err) + } + break + } + } + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error while talking to cloudflare", "error", err) + return err + } + if err := cloudflare.CreateDNSRecord(mf.Metadata.DNSZone, fqdn, "A", addr, 1, false); err != nil { + return fmt.Errorf("can't create DNS record for %s, %v", fqdn, err) + } + log.DebugLog(log.DebugLevelMexos, "waiting for cloudflare...") + //once successfully inserted the A record will take a bit of time, but not too long due to fast cloudflare anycast + //err = WaitforDNSRegistration(fqdn) + //if err != nil { + // return err + //} + return nil +} diff --git a/mexos/gcloud.go b/mexos/gcloud.go new file mode 100644 index 000000000..b8a6e07af --- /dev/null +++ b/mexos/gcloud.go @@ -0,0 +1,44 @@ +package mexos + +import ( + "fmt" + "time" + + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/gcloud" + "github.com/mobiledgex/edge-cloud/log" +) + +var GCPDefaultProjectID = "still-entity-201400" // XXX + +func gcloudCreateGKE(mf *Manifest) error { + var err error + if mf.Metadata.Project == "" { + log.DebugLog(log.DebugLevelMexos, "warning, empty gcp project ID, using default", "default", GCPDefaultProjectID) + } + if err = gcloud.SetProject(mf.Metadata.Project); err != nil { + return err + } + if err = gcloud.SetZone(mf.Metadata.Zone); err != nil { + return err + } + if err = gcloud.CreateGKECluster(mf.Metadata.Name); err != nil { + return err + } + //race condition exists where the config file is not ready until just after the cluster create is done + time.Sleep(3 * time.Second) + saveKubeconfig() + if err = gcloud.GetGKECredentials(mf.Metadata.Name); err != nil { + return err + } + kconf, err := GetKconf(mf, false) //XXX + if err != nil { + return fmt.Errorf("cannot get kconf, %v, %v, %v", mf, kconf, err) + } + log.DebugLog(log.DebugLevelMexos, "warning, using default config") //XXX + if err = copyFile(defaultKubeconfig(), kconf); err != nil { + return fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) + } + log.DebugLog(log. + DebugLevelMexos, "created gke", "name", mf.Spec.Key) + return nil +} diff --git a/mexos/helm.go b/mexos/helm.go new file mode 100644 index 000000000..7f647cd82 --- /dev/null +++ b/mexos/helm.go @@ -0,0 +1,85 @@ +package mexos + +import ( + "fmt" + + "github.com/mobiledgex/edge-cloud/log" +) + +func DeleteHelmAppManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "delete kubernetes helm app") + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("cannot delete helm app, rootLB is null") + } + if err = ValidateCommon(mf); err != nil { + return err + } + kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) + if err != nil { + return err + } + // remove DNS entries + if err = KubeDeleteDNSRecords(rootLB, mf, kp); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete DNS record", "error", err) + } + // remove Security rules + if err = DeleteProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rules", "error", err) + } + cmd := fmt.Sprintf("%s helm delete %s", kp.kubeconfig, mf.Metadata.Name) + out, err := kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error deleting helm chart, %s, %s, %v", cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "removed helm chart") + return nil +} + +func CreateHelmAppManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "create kubernetes helm app") + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("cannot create helm app, rootLB is null") + } + if err = ValidateCommon(mf); err != nil { + return err + } + kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) + if err != nil { + return err + } + log.DebugLog(log.DebugLevelMexos, "will launch app into cluster", "kubeconfig", kp.kubeconfig, "ipaddr", kp.ipaddr) + + cmd := fmt.Sprintf("%s helm init --wait", kp.kubeconfig) + out, err := kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error initializing tiller for app, %s, %s, %v", cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "helm tiller initialized") + + cmd = fmt.Sprintf("%s helm install %s --name %s", kp.kubeconfig, mf.Spec.Image, mf.Metadata.Name) + out, err = kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error deploying helm chart, %s, %s, %v", cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "applied helm chart") + // Add security rules + if err = AddProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "add spec ports", "ports", mf.Spec.Ports) + // Add DNS Zone + if err = KubeAddDNSRecords(rootLB, mf, kp); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) + return err + } + return nil +} diff --git a/mexos/init.go b/mexos/init.go new file mode 100644 index 000000000..0028e94de --- /dev/null +++ b/mexos/init.go @@ -0,0 +1,14 @@ +package mexos + +import ( + "github.com/mobiledgex/edge-cloud/log" +) + +//MEXInit initializes MEX API +func MEXInit(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "mex init") + if err := MEXCheckEnvVars(mf); err != nil { + return err + } + return nil +} diff --git a/mexos/inst.go b/mexos/inst.go new file mode 100644 index 000000000..03c673b73 --- /dev/null +++ b/mexos/inst.go @@ -0,0 +1,172 @@ +package mexos + +import ( + "fmt" + + "github.com/mobiledgex/edge-cloud/edgeproto" + "github.com/mobiledgex/edge-cloud/log" +) + +//MEXClusterCreateClustInst calls MEXClusterCreate with a manifest created from the template +func MEXClusterCreateClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) error { + //XXX trigger off clusterInst or flavor to pick the right template: mex, aks, gke + mf, err := FillClusterTemplateClustInst(rootLB, clusterInst) + if err != nil { + return err + } + fixValuesInst(mf, rootLB) + return MEXClusterCreateManifest(mf) +} + +//MEXClusterRemoveClustInst calls MEXClusterRemove with a manifest created from the template +func MEXClusterRemoveClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) error { + mf, err := FillClusterTemplateClustInst(rootLB, clusterInst) + if err != nil { + return err + } + fixValuesInst(mf, rootLB) + return MEXClusterRemoveManifest(mf) +} + +func FillClusterTemplateClustInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst) (*Manifest, error) { + log.DebugLog(log.DebugLevelMexos, "fill cluster template manifest cluster inst", "clustinst", clusterInst) + if clusterInst.Key.ClusterKey.Name == "" { + log.DebugLog(log.DebugLevelMexos, "cannot create empty cluster manifest", "clustinst", clusterInst) + return nil, fmt.Errorf("invalid cluster inst %v", clusterInst) + } + if verr := ValidateClusterKind(clusterInst.Key.CloudletKey.OperatorKey.Name); verr != nil { + return nil, verr + } + vp := &rootLB.PlatConf.Values + data := templateFill{ + ResourceKind: "cluster", + Resource: clusterInst.Flavor.Name, + Name: clusterInst.Key.ClusterKey.Name, + Tags: clusterInst.Key.ClusterKey.Name + "-tag", + Tenant: clusterInst.Key.ClusterKey.Name + "-tenant", + Operator: NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name), + Key: clusterInst.Key.ClusterKey.Name, + Kind: vp.Cluster.Kind, //"kubernetes", + ResourceGroup: clusterInst.Key.CloudletKey.Name + "_" + clusterInst.Key.ClusterKey.Name, + Flavor: clusterInst.Flavor.Name, + DNSZone: vp.Network.DNSZone, //"mobiledgex.net", + RootLB: rootLB.Name, + Region: vp.Cluster.Region, //us-west1 + Zone: vp.Cluster.Zone, //us-west1a + Location: vp.Cluster.Location, // us-west + NetworkScheme: vp.Network.Scheme, //"priv-subnet,mex-k8s-net-1,10.101.X.0/24", + Swarm: vp.Cluster.Swarm, + } + + // // if these env variables are not set, fall back to the + // // existing defaults based on deployment type(operator name) + // data.Region = os.Getenv("CLOUDLET_REGION") + // data.Zone = os.Getenv("CLOUDLET_ZONE") + // data.Location = os.Getenv("CLOUDLET_LOCATION") + + // switch clusterInst.Key.CloudletKey.OperatorKey.Name { + // case "gcp": + // if data.Region == "" { + // data.Region = "us-west1" + // } + // if data.Zone == "" { + // data.Zone = "us-west1-a" + // } + // if data.Location == "" { + // data.Location = "us-west" + // } + // data.Project = "still-entity-201400" // XXX + // case "azure": + // if data.Region == "" { + // data.Region = "centralus" + // } + // if data.Zone == "" { + // data.Zone = "centralus" + // } + // if data.Location == "" { + // data.Location = "centralus" + // } + // default: + // if data.Region == "" { + // data.Region = "eu-central-1" + // } + // if data.Zone == "" { + // data.Zone = "eu-central-1c" + // } + // if data.Location == "" { + // data.Location = "buckhorn" + // } + // } + + mf, err := templateUnmarshal(&data, yamlMEXCluster) + if err != nil { + return nil, err + } + fixValuesInst(mf, rootLB) + return mf, nil +} + +func MEXAddFlavorClusterInst(rootLB *MEXRootLB, flavor *edgeproto.ClusterFlavor) error { + log.DebugLog(log.DebugLevelMexos, "adding cluster inst flavor", "flavor", flavor) + + if flavor.Key.Name == "" { + log.DebugLog(log.DebugLevelMexos, "cannot add empty cluster inst flavor", "flavor", flavor) + return fmt.Errorf("will not add empty cluster inst %v", flavor) + } + vp := &rootLB.PlatConf.Values + data := templateFill{ + ResourceKind: "flavor", + Resource: flavor.Key.Name, + Name: flavor.Key.Name, + Tags: flavor.Key.Name + "-tag", + Kind: vp.Cluster.Kind, + Flags: flavor.Key.Name + "-flags", + NumNodes: int(flavor.NumNodes), + NumMasters: int(flavor.NumMasters), + NetworkScheme: vp.Network.Scheme, + MasterFlavor: flavor.MasterFlavor.Name, + NodeFlavor: flavor.NodeFlavor.Name, + StorageSpec: "default", //XXX + Topology: "type-1", //XXX + } + mf, err := templateUnmarshal(&data, yamlMEXFlavor) + if err != nil { + return err + } + fixValuesInst(mf, rootLB) + return MEXAddFlavor(mf) +} + +//MEXAppCreateAppInst creates app inst with templated manifest +func MEXAppCreateAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, app *edgeproto.App) error { + log.DebugLog(log.DebugLevelMexos, "mex create app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) + mf, err := fillAppTemplate(rootLB, appInst, app, clusterInst) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "fillAppTemplate error", "error", err) + return err + } + fixValuesInst(mf, rootLB) + return MEXAppCreateAppManifest(mf) +} + +//MEXAppDeleteAppInst deletes app with templated manifest +func MEXAppDeleteAppInst(rootLB *MEXRootLB, clusterInst *edgeproto.ClusterInst, appInst *edgeproto.AppInst, app *edgeproto.App) error { + log.DebugLog(log.DebugLevelMexos, "mex delete app inst", "rootlb", rootLB.Name, "clusterinst", clusterInst, "appinst", appInst) + mf, err := fillAppTemplate(rootLB, appInst, app, clusterInst) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "fillAppTemplate error", "error", err) + return err + } + fixValuesInst(mf, rootLB) + return MEXAppDeleteAppManifest(mf) +} + +func fixValuesInst(mf *Manifest, rootLB *MEXRootLB) error { + if mf.Values.Kind == "" { + mf.Values = rootLB.PlatConf.Values + } + if mf.Values.Kind == "" { + log.DebugLog(log.DebugLevelMexos, "warning, missing mf values") + } + return nil +} diff --git a/mexos/ip.go b/mexos/ip.go new file mode 100644 index 000000000..ad43b9651 --- /dev/null +++ b/mexos/ip.go @@ -0,0 +1,104 @@ +package mexos + +import ( + "fmt" + "strings" + + "github.com/mobiledgex/edge-cloud/log" +) + +var defaultPrivateNetRange = "10.101.X.0/24" + +//GetInternalIP returns IP of the server +func GetInternalIP(mf *Manifest, name string) (string, error) { + sd, err := GetServerDetails(mf, name) + if err != nil { + return "", err + } + its := strings.Split(sd.Addresses, "=") + if len(its) != 2 { + return "", fmt.Errorf("GetInternalIP: can't parse server detail addresses, %v, %v", sd, err) + } + return its[1], nil +} + +//GetInternalCIDR returns CIDR of server +func GetInternalCIDR(mf *Manifest, name string) (string, error) { + addr, err := GetInternalIP(mf, name) + if err != nil { + return "", err + } + cidr := addr + "/24" // XXX we use this convention of /24 in k8s priv-net + return cidr, nil +} + +func GetAllowedClientCIDR() string { + //XXX TODO get real list of allowed clients from remote database or template configuration + return "0.0.0.0/0" +} + +//XXX allow creating more than one LB + +//GetServerIPAddr gets the server IP +func GetServerIPAddr(mf *Manifest, networkName, serverName string) (string, error) { + //TODO: mexosagent cache + log.DebugLog(log.DebugLevelMexos, "get server ip addr", "networkname", networkName, "servername", serverName) + //sd, err := GetServerDetails(rootLB) + sd, err := GetServerDetails(mf, serverName) + if err != nil { + return "", err + } + its := strings.Split(sd.Addresses, "=") + if len(its) != 2 { + its = strings.Split(sd.Addresses, ";") + foundaddr := "" + if len(its) > 1 { + for _, it := range its { + sits := strings.Split(it, "=") + if len(sits) == 2 { + if strings.Contains(sits[0], "mex-k8s-net") { + continue + } + if strings.TrimSpace(sits[0]) == networkName { // XXX + foundaddr = sits[1] + break + } + } + } + } + if foundaddr != "" { + log.DebugLog(log.DebugLevelMexos, "retrieved server ipaddr", "ipaddr", foundaddr, "netname", networkName, "servername", serverName) + return foundaddr, nil + } + return "", fmt.Errorf("GetServerIPAddr: can't parse server detail addresses, %v, %v", sd, err) + } + if its[0] != networkName { + return "", fmt.Errorf("invalid network name in server detail address, %s", sd.Addresses) + } + addr := its[1] + log.DebugLog(log.DebugLevelMexos, "got server ip addr", "ipaddr", addr, "netname", networkName, "servername", serverName) + return addr, nil +} + +//FindNodeIP finds IP for the given node +func FindNodeIP(mf *Manifest, name string) (string, error) { + log.DebugLog(log.DebugLevelMexos, "find node ip", "name", name) + if name == "" { + return "", fmt.Errorf("empty name") + } + srvs, err := ListServers(mf) + if err != nil { + return "", err + } + for _, s := range srvs { + if s.Status == "ACTIVE" && s.Name == name { + ipaddr, err := GetInternalIP(mf, s.Name) + if err != nil { + return "", fmt.Errorf("can't get IP for %s, %v", s.Name, err) + } + log.DebugLog(log.DebugLevelMexos, "found node ip", "name", name, "ipaddr", ipaddr) + return ipaddr, nil + } + } + return "", fmt.Errorf("node %s, ip not found", name) +} diff --git a/mexos/items.go b/mexos/items.go new file mode 100644 index 000000000..52284e3f7 --- /dev/null +++ b/mexos/items.go @@ -0,0 +1,9 @@ +package mexos + +type genericItem struct { + Item interface{} `json:"items"` +} + +type genericItems struct { + Items []genericItem `json:"items"` +} diff --git a/mexos/kubeconfig.go b/mexos/kubeconfig.go new file mode 100644 index 000000000..b67cbe678 --- /dev/null +++ b/mexos/kubeconfig.go @@ -0,0 +1,173 @@ +package mexos + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/ghodss/yaml" + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/azure" + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/gcloud" + "github.com/mobiledgex/edge-cloud/log" +) + +func GetLocalKconfName(mf *Manifest) string { + kconf := fmt.Sprintf("%s/%s", MEXDir(), GetKconfName(mf)) + if mf.Metadata.Operator != "gcp" && + mf.Metadata.Operator != "azure" { + return kconf + "-proxy" + } + return kconf +} + +func GetKconfName(mf *Manifest) string { + return fmt.Sprintf("%s.%s.%s.kubeconfig", + mf.Values.Cluster.Name, + mf.Values.Operator.Name, + mf.Values.Network.DNSZone) +} + +func GetKconf(mf *Manifest, createIfMissing bool) (string, error) { + name := GetLocalKconfName(mf) + log.DebugLog(log.DebugLevelMexos, "get kubeconfig name", "name", name) + if createIfMissing { // XXX + log.DebugLog(log.DebugLevelMexos, "warning, creating missing kubeconfig", "name", name) + if _, err := os.Stat(name); os.IsNotExist(err) { + // if kubeconfig does not exist, optionally create it. It is possible it was + // created on a different container or we had a restart of the container + log.DebugLog(log.DebugLevelMexos, "creating missing kconf file", "name", name) + switch mf.Metadata.Operator { + case "gcp": + if err = gcloud.GetGKECredentials(mf.Metadata.Name); err != nil { + return "", fmt.Errorf("unable to get GKE credentials %v", err) + } + if err = copyFile(defaultKubeconfig(), name); err != nil { + return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) + } + case "azure": + if err = azure.GetAKSCredentials(mf.Metadata.ResourceGroup, mf.Metadata.Name); err != nil { + return "", fmt.Errorf("unable to get AKS credentials %v", err) + } + if err = copyFile(defaultKubeconfig(), name); err != nil { + return "", fmt.Errorf("can't copy %s, %v", defaultKubeconfig(), err) + } + default: + log.DebugLog(log.DebugLevelMexos, "warning, not creating missing kubeconfig for operator", "operator", mf.Metadata.Operator) + } + } + } + return name, nil +} + +type clusterDetailKc struct { + CertificateAuthorityData string `json:"certificate-authority-data"` + Server string `json:"server"` +} + +type clusterKc struct { + Name string `json:"name"` + Cluster clusterDetailKc `json:"cluster"` +} + +type clusterKcContextDetail struct { + Cluster string `json:"cluster"` + User string `json:"user"` +} + +type clusterKcContext struct { + Name string `json:"name"` + Context clusterKcContextDetail `json:"context"` +} + +type clusterKcUserDetail struct { + ClientCertificateData string `json:"client-certificate-data"` + ClientKeyData string `json:"client-key-data"` +} + +type clusterKcUser struct { + Name string `json:"name"` + User clusterKcUserDetail `json:"user"` +} + +type clusterKubeconfig struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + CurrentContext string `json:"current-context"` + Users []clusterKcUser `json:"users"` + Clusters []clusterKc `json:"clusters"` + Contexts []clusterKcContext `json:"contexts"` + //XXX Missing preferences +} + +//CopyKubeConfig copies over kubeconfig from the cluster +func CopyKubeConfig(mf *Manifest, rootLB *MEXRootLB, name string) error { + log.DebugLog(log.DebugLevelMexos, "copying kubeconfig", "name", name) + if rootLB == nil { + return fmt.Errorf("cannot copy kubeconfig, rootLB is null") + } + ipaddr, err := FindNodeIP(mf, name) + if err != nil { + return err + } + if mf.Values.Network.External == "" { + return fmt.Errorf("copy kube config, missing external network in platform config") + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return fmt.Errorf("can't get ssh client for copying kubeconfig, %v", err) + } + //kconfname := fmt.Sprintf("%s.kubeconfig", name[strings.LastIndex(name, "-")+1:]) + kconfname := GetKconfName(mf) + log.DebugLog(log.DebugLevelMexos, "attempt to get kubeconfig from k8s master", "name", name, "ipaddr", ipaddr, "dest", kconfname) + cmd := fmt.Sprintf("scp -o %s -o %s -i id_rsa_mex %s@%s:.kube/config %s", sshOpts[0], sshOpts[1], sshUser, ipaddr, kconfname) + out, err := client.Output(cmd) + if err != nil { + return fmt.Errorf("can't copy kubeconfig from %s, %s, %v", name, out, err) + } + cmd = fmt.Sprintf("cat %s", kconfname) + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("can't cat %s, %s, %v", kconfname, out, err) + } + port, serr := StartKubectlProxy(mf, rootLB, kconfname) + if serr != nil { + return serr + } + return ProcessKubeconfig(mf, rootLB, name, port, []byte(out)) +} + +//ProcessKubeconfig validates kubeconfig and saves it and creates a copy for proxy access +func ProcessKubeconfig(mf *Manifest, rootLB *MEXRootLB, name string, port int, dat []byte) error { + log.DebugLog(log.DebugLevelMexos, "process kubeconfig file", "name", name) + if rootLB == nil { + return fmt.Errorf("cannot process kubeconfig, rootLB is null") + } + kc := &clusterKubeconfig{} + err := yaml.Unmarshal(dat, kc) + if err != nil { + return fmt.Errorf("can't unmarshal kubeconfig %s, %v", name, err) + } + if len(kc.Clusters) < 1 { + return fmt.Errorf("insufficient clusters info in kubeconfig %s", name) + } + //kconfname := fmt.Sprintf("%s.kubeconfig", name[strings.LastIndex(name, "-")+1:]) + kconfname := GetLocalKconfName(mf) + log.DebugLog(log.DebugLevelMexos, "writing local kubeconfig file", "name", kconfname) + err = ioutil.WriteFile(kconfname, dat, 0666) + if err != nil { + return fmt.Errorf("can't write kubeconfig name %s filename %s,%v", name, kconfname, err) + } + log.DebugLog(log.DebugLevelMexos, "wrote kubeconfig", "file", kconfname) + kc.Clusters[0].Cluster.Server = fmt.Sprintf("http://%s:%d", rootLB.Name, port) + dat, err = yaml.Marshal(kc) + if err != nil { + return fmt.Errorf("can't marshal kubeconfig proxy edit %s, %v", name, err) + } + kconfname = kconfname + "-proxy" + err = ioutil.WriteFile(kconfname, dat, 0666) + if err != nil { + return fmt.Errorf("can't write kubeconfig proxy %s, %v", kconfname, err) + } + log.DebugLog(log.DebugLevelMexos, "kubeconfig-proxy file saved", "file", kconfname) + return nil +} diff --git a/mexos/kubectl.go b/mexos/kubectl.go new file mode 100644 index 000000000..b50f77f5f --- /dev/null +++ b/mexos/kubectl.go @@ -0,0 +1,218 @@ +package mexos + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud-infra/openstack-tenant/agent/cloudflare" + "github.com/mobiledgex/edge-cloud/cloudcommon" + "github.com/mobiledgex/edge-cloud/log" +) + +func CreateDockerRegistrySecret(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "creating docker registry secret in kubernetes") + kconf, err := GetKconf(mf, false) + if err != nil { + return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err) + } + out, err := sh.Command("kubectl", "create", "secret", "docker-registry", "mexregistrysecret", "--docker-server="+mf.Values.Registry.Docker, "--docker-username=mobiledgex", "--docker-password="+mexEnv(mf, "MEX_DOCKER_REG_PASS"), "--docker-email=docker@mobiledgex.com", "--kubeconfig="+kconf).CombinedOutput() + if err != nil { + if !strings.Contains(string(out), "AlreadyExists") { + return fmt.Errorf("can't add docker registry secret, %s, %v", out, err) + } else { + log.DebugLog(log.DebugLevelMexos, "warning, docker registry secret already exists.") + } + } + log.DebugLog(log.DebugLevelMexos, "ok, created mexregistrysecret") + return nil +} + +func runKubectlCreateApp(mf *Manifest, kubeManifest string) error { + log.DebugLog(log.DebugLevelMexos, "run kubectl create app", "kubeManifest", kubeManifest) + if err := CreateDockerRegistrySecret(mf); err != nil { + return err + } + kfile := mf.Metadata.Name + ".yaml" + if err := writeKubeManifest(kubeManifest, kfile); err != nil { + return err + } + defer os.Remove(kfile) + kconf, err := GetKconf(mf, false) + if err != nil { + return fmt.Errorf("error creating app due to kconf missing, %v, %v", mf, err) + } + out, err := sh.Command("kubectl", "create", "-f", kfile, "--kubeconfig="+kconf).Output() + if err != nil { + return fmt.Errorf("error creating app, %s, %v, %v", out, err, mf) + } + err = createAppDNS(mf, kconf) + if err != nil { + return fmt.Errorf("error creating dns entry for app, %v, %v", err, mf) + } + return nil +} + +func getSvcExternalIP(name string, kconf string) (string, error) { + log.DebugLog(log.DebugLevelMexos, "get service external IP", "name", name) + externalIP := "" + var out []byte + var err error + //wait for Load Balancer to assign external IP address. It takes a variable amount of time. + for i := 0; i < 100; i++ { + out, err = sh.Command("kubectl", "get", "svc", "--kubeconfig="+kconf, "-o", "json").Output() + if err != nil { + return "", fmt.Errorf("error getting svc %s, %s, %v", name, out, err) + } + svcs := &svcItems{} + err = json.Unmarshal(out, svcs) + if err != nil { + return "", fmt.Errorf("error unmarshalling svc json, %v", err) + } + log.DebugLog(log.DebugLevelMexos, "getting externalIP, examine list of services", "name", name, "svcs", svcs) + for _, item := range svcs.Items { + log.DebugLog(log.DebugLevelMexos, "svc item", "item", item, "name", name) + if item.Metadata.Name != name { + continue + } + for _, ingress := range item.Status.LoadBalancer.Ingresses { + if ingress.IP != "" { + externalIP = ingress.IP + log.DebugLog(log.DebugLevelMexos, "got externaIP for app", "externalIP", externalIP) + return externalIP, nil + } + } + } + time.Sleep(3 * time.Second) + } + if externalIP == "" { + return "", fmt.Errorf("timed out trying to get externalIP") + } + return externalIP, nil +} + +func getSvcNames(name string, kconf string) ([]string, error) { + out, err := sh.Command("kubectl", "get", "svc", "--kubeconfig="+kconf, "-o", "json").Output() + if err != nil { + return nil, fmt.Errorf("error getting svc %s, %s, %v", name, out, err) + } + svcs := &svcItems{} + err = json.Unmarshal(out, svcs) + if err != nil { + return nil, fmt.Errorf("error unmarshalling svc json, %v", err) + } + var serviceNames []string + for _, item := range svcs.Items { + if strings.HasPrefix(item.Metadata.Name, name) { + serviceNames = append(serviceNames, item.Metadata.Name) + } + } + log.DebugLog(log.DebugLevelMexos, "service names", "names", serviceNames) + return serviceNames, nil +} + +func runKubectlDeleteApp(mf *Manifest, kubeManifest string) error { + if err := CheckCredentialsCF(mf); err != nil { + return err + } + if err := cloudflare.InitAPI(mexEnv(mf, "MEX_CF_USER"), mexEnv(mf, "MEX_CF_KEY")); err != nil { + return fmt.Errorf("cannot init cloudflare api, %v", err) + } + kconf, err := GetKconf(mf, false) + if err != nil { + return fmt.Errorf("error deleting app due to kconf missing, %v, %v", mf, err) + } + kfile := mf.Metadata.Name + ".yaml" + err = writeKubeManifest(kubeManifest, kfile) + if err != nil { + return err + } + defer os.Remove(kfile) + serviceNames, err := getSvcNames(mf.Metadata.Name, kconf) + if err != nil { + return err + } + if len(serviceNames) < 1 { + return fmt.Errorf("no service names starting with %s", mf.Metadata.Name) + } + out, err := sh.Command("kubectl", "delete", "-f", kfile, "--kubeconfig="+kconf).CombinedOutput() + if err != nil { + return fmt.Errorf("error deleting app, %s, %v, %v", out, mf, err) + } + if mf.Metadata.DNSZone == "" { + return fmt.Errorf("missing dns zone, metadata %v", mf.Metadata) + } + fqdnBase := uri2fqdn(mf.Spec.URI) + dr, err := cloudflare.GetDNSRecords(mf.Metadata.DNSZone) + if err != nil { + return fmt.Errorf("cannot get dns records for %s, %v", mf.Metadata.DNSZone, err) + } + for _, sn := range serviceNames { + fqdn := cloudcommon.ServiceFQDN(sn, fqdnBase) + for _, d := range dr { + if d.Type == "A" && d.Name == fqdn { + if err := cloudflare.DeleteDNSRecord(mf.Metadata.DNSZone, d.ID); err != nil { + return fmt.Errorf("cannot delete DNS record, %v", d) + } + log.DebugLog(log.DebugLevelMexos, "deleted DNS record", "name", fqdn) + } + } + } + return nil +} + +func saveKubeconfig() { + kc := defaultKubeconfig() + if err := os.Rename(kc, kc+".save"); err != nil { + log.DebugLog(log.DebugLevelMexos, "can't rename", "name", kc, "error", err) + } +} + +func parseKCPort(ln string) int { + if !strings.Contains(ln, "kubectl") { + return 0 + } + if !strings.Contains(ln, "--port") { + return 0 + } + var a, b, c, port string + n, serr := fmt.Sscanf(ln, "%s %s %s %s", &a, &b, &c, &port) + if serr != nil { + return 0 + } + if n != 4 { + return 0 + } + portnum, aerr := strconv.Atoi(port) + if aerr != nil { + return 0 + } + return portnum +} + +func parseKCPid(ln string, key string) int { + ln = strings.TrimSpace(ln) + if !strings.Contains(ln, "kubectl") { + return 0 + } + if !strings.HasSuffix(ln, key) { + return 0 + } + var pid string + n, serr := fmt.Sscanf(ln, "%s", &pid) + if serr != nil { + return 0 + } + if n != 1 { + return 0 + } + pidnum, aerr := strconv.Atoi(pid) + if aerr != nil { + return 0 + } + return pidnum +} diff --git a/mexos/kubeproxy.go b/mexos/kubeproxy.go new file mode 100644 index 000000000..b87e9f839 --- /dev/null +++ b/mexos/kubeproxy.go @@ -0,0 +1,73 @@ +package mexos + +import ( + "fmt" + "strings" + "time" + + "github.com/mobiledgex/edge-cloud/log" +) + +//StartKubectlProxy starts kubectl proxy on the rootLB to handle kubectl commands remotely. +// To be called after copying over the kubeconfig file from cluster to rootLB. +func StartKubectlProxy(mf *Manifest, rootLB *MEXRootLB, kubeconfig string) (int, error) { + log.DebugLog(log.DebugLevelMexos, "start kubectl proxy", "kubeconfig", kubeconfig) + if rootLB == nil { + return 0, fmt.Errorf("cannot kubectl proxy, rootLB is null") + } + if mf.Values.Network.External == "" { + return 0, fmt.Errorf("start kubectl proxy, missing external network in platform config") + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return 0, err + } + maxPort := 8000 + cmd := "sudo ps wwh -C kubectl -o args" + out, err := client.Output(cmd) + if err == nil && out != "" { + lines := strings.Split(out, "\n") + for _, ln := range lines { + portnum := parseKCPort(ln) + if portnum > maxPort { + maxPort = portnum + } + } + } + maxPort++ + log.DebugLog(log.DebugLevelMexos, "port for kubectl proxy", "maxport", maxPort) + cmd = fmt.Sprintf("kubectl proxy --port %d --accept-hosts='.*' --address='0.0.0.0' --kubeconfig=%s ", maxPort, kubeconfig) + //Use .Start() because we don't want to hang + cl1, cl2, err := client.Start(cmd) + if err != nil { + return 0, fmt.Errorf("error running kubectl proxy, %s, %v", cmd, err) + } + cl1.Close() //nolint + cl2.Close() //nolint + err = AddSecurityRuleCIDR(mf, GetAllowedClientCIDR(), "tcp", GetMEXSecurityRule(mf), maxPort) + log.DebugLog(log.DebugLevelMexos, "adding external ingress security rule for kubeproxy", "port", maxPort) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, error while adding external ingress security rule for kubeproxy", "error", err, "port", maxPort) + } + + cmd = "sudo ps wwh -C kubectl -o args" + for i := 0; i < 5; i++ { + //verify + out, outerr := client.Output(cmd) + if outerr == nil { + if out == "" { + continue + } + lines := strings.Split(out, "\n") + for _, ln := range lines { + if parseKCPort(ln) == maxPort { + log.DebugLog(log.DebugLevelMexos, "kubectl confirmed running with port", "port", maxPort) + } + } + return maxPort, nil + } + log.DebugLog(log.DebugLevelMexos, "waiting for kubectl proxy...") + time.Sleep(3 * time.Second) + } + return 0, fmt.Errorf("timeout error verifying kubectl proxy") +} diff --git a/mexos/kubernetes.go b/mexos/kubernetes.go new file mode 100644 index 000000000..836d25e1b --- /dev/null +++ b/mexos/kubernetes.go @@ -0,0 +1,326 @@ +package mexos + +import ( + "encoding/json" + "fmt" + + "github.com/mobiledgex/edge-cloud/log" + ssh "github.com/nanobox-io/golang-ssh" +) + +//CreateKubernetesAppManifest instantiates a new kubernetes deployment +func CreateKubernetesAppManifest(mf *Manifest, kubeManifest string) error { + log.DebugLog(log.DebugLevelMexos, "create kubernetes app") + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("cannot create kubernetes app manifest, rootLB is null") + } + if err = ValidateCommon(mf); err != nil { + return err + } + if mf.Spec.URI == "" { //XXX TODO register to the DNS registry for public IP app,controller needs to tell us which kind of app + return fmt.Errorf("empty app URI") + } + kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) + if err != nil { + return err + } + log.DebugLog(log.DebugLevelMexos, "will launch app into cluster", "kubeconfig", kp.kubeconfig, "ipaddr", kp.ipaddr) + var cmd string + if mexEnv(mf, "MEX_DOCKER_REG_PASS") == "" { + return fmt.Errorf("empty docker registry password environment variable") + } + if err := CreateDockerRegistrySecret(mf); err != nil { + return err + } + cmd = fmt.Sprintf("cat <<'EOF'> %s.yaml \n%s\nEOF", mf.Metadata.Name, kubeManifest) + out, err := kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error writing KubeManifest, %s, %s, %v", cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "wrote kubernetes manifest file") + cmd = fmt.Sprintf("%s kubectl create -f %s.yaml", kp.kubeconfig, mf.Metadata.Name) + out, err = kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error deploying kubernetes app, %s, %v", out, err) + } + log.DebugLog(log.DebugLevelMexos, "applied kubernetes manifest") + // Add security rules + if err = AddProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "ok, added spec ports", "ports", mf.Spec.Ports) + // Add DNS Zone + if err = KubeAddDNSRecords(rootLB, mf, kp); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add DNS entries", "error", err) + return err + } + return nil +} + +type kubeParam struct { + kubeconfig string + client ssh.Client + ipaddr string +} + +//ValidateKubernetesParameters checks the kubernetes parameters and kubeconfig settings +func ValidateKubernetesParameters(mf *Manifest, rootLB *MEXRootLB, clustName string) (*kubeParam, error) { + log.DebugLog(log.DebugLevelMexos, "validate kubernetes parameters rootLB", "cluster", clustName) + if rootLB == nil { + return nil, fmt.Errorf("cannot validate kubernetes parameters, rootLB is null") + } + if rootLB.PlatConf == nil { + return nil, fmt.Errorf("validate kubernetes parameters, missing platform config") + } + if mf.Values.Network.External == "" { + return nil, fmt.Errorf("validate kubernetes parameters, missing external network in platform config") + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return nil, err + } + name, err := FindClusterWithKey(mf, clustName) + if err != nil { + return nil, fmt.Errorf("can't find cluster with key %s, %v", clustName, err) + } + ipaddr, err := FindNodeIP(mf, name) + if err != nil { + return nil, err + } + //kubeconfig := fmt.Sprintf("KUBECONFIG=%s.kubeconfig", name[strings.LastIndex(name, "-")+1:]) + kubeconfig := fmt.Sprintf("KUBECONFIG=%s", GetKconfName(mf)) + return &kubeParam{kubeconfig, client, ipaddr}, nil +} + +//KubernetesApplyManifest does `apply` on the manifest yaml +func KubernetesApplyManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "apply kubernetes manifest") + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("cannot apply kubernetes manifest, rootLB is null") + } + if mf.Metadata.Name == "" { + return fmt.Errorf("missing name") + } + kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) + if err != nil { + return err + } + kubeManifest := mf.Config.ConfigDetail.Manifest + cmd := fmt.Sprintf("cat <<'EOF'> %s \n%s\nEOF", mf.Metadata.Name, kubeManifest) + out, err := kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error writing deployment, %s, %s, %v", cmd, out, err) + } + log.DebugLog(log.DebugLevelMexos, "wrote deployment file") + cmd = fmt.Sprintf("%s kubectl apply -f %s", kp.kubeconfig, mf.Metadata.Name) + out, err = kp.client.Output(cmd) + if err != nil { + return fmt.Errorf("error applying kubernetes manifest, %s, %s, %v", cmd, out, err) + } + return nil +} + +//CreateKubernetesNamespaceManifest creates a new namespace in kubernetes +func CreateKubernetesNamespaceManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "create kubernetes namespace") + err := KubernetesApplyManifest(mf) + if err != nil { + return fmt.Errorf("error applying kubernetes namespace manifest, %v", err) + } + return nil +} + +//TODO DeleteKubernetesNamespace + +//TODO allow configmap creation from files + +//SetKubernetesConfigmapValues sets a key-value in kubernetes configmap +// func SetKubernetesConfigmapValues(rootLBName string, clustername string, configname string, keyvalues ...string) error { +// log.DebugLog(log.DebugLevelMexos, "set configmap values", "rootlbname", rootLBName, "clustername", clustername, "configname", configname) +// rootLB, err := getRootLB(rootLBName) +// if err != nil { +// return err +// } +// if rootLB == nil { +// return fmt.Errorf("cannot set kubeconfig map values, rootLB is null") +// } +// kp, err := ValidateKubernetesParameters(mf, rootLB, clustername) +// if err != nil { +// return err +// } +// //TODO support namespace +// cmd := fmt.Sprintf("%s kubectl create configmap %s ", kp.kubeconfig, configname) +// for _, kv := range keyvalues { +// items := strings.Split(kv, "=") +// if len(items) != 2 { +// return fmt.Errorf("malformed key=value pair, %s", kv) +// } +// cmd = cmd + " --from-literal=" + kv +// } +// out, err := kp.client.Output(cmd) +// if err != nil { +// return fmt.Errorf("error setting key/values to kubernetes configmap, %s, %s, %v", cmd, out, err) +// } +// return nil +// } + +//TODO +//func GetKubernetesConfigmapValues(rootLB, clustername, configname string) (map[string]string, error) { +//} + +//GetKubernetesConfigmapYAML returns yaml reprentation of the key-values +// func GetKubernetesConfigmapYAML(rootLBName string, clustername, configname string) (string, error) { +// log.DebugLog(log.DebugLevelMexos, "get kubernetes configmap", "rootlbname", rootLBName, "clustername", clustername, "configname", configname) +// rootLB, err := getRootLB(rootLBName) +// if err != nil { +// return "", err +// } +// if rootLB == nil { +// return "", fmt.Errorf("cannot get kubeconfigmap yaml, rootLB is null") +// } +// kp, err := ValidateKubernetesParameters(mf, rootLB, clustername) +// if err != nil { +// return "", err +// } +// //TODO support namespace +// cmd := fmt.Sprintf("%s kubectl get configmap %s -o yaml", kp.kubeconfig, configname) +// out, err := kp.client.Output(cmd) +// if err != nil { +// return "", fmt.Errorf("error getting configmap yaml, %s, %s, %v", cmd, out, err) +// } +// return out, nil +// } + +func DeleteKubernetesAppManifest(mf *Manifest, kubeManifest string) error { + log.DebugLog(log.DebugLevelMexos, "delete kubernetes app") + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("cannot remove kubernetes app manifest, rootLB is null") + } + if mf.Spec.URI == "" { //XXX TODO register to the DNS registry for public IP app,controller needs to tell us which kind of app + return fmt.Errorf("empty app URI") + } + //TODO: support other URI: file://, nfs://, ftp://, git://, or embedded as base64 string + //if !strings.Contains(mf.Spec.Flavor, "kubernetes") { + // return fmt.Errorf("unsupported kubernetes flavor %s", mf.Spec.Flavor) + //} + if err = ValidateCommon(mf); err != nil { + return err + } + kp, err := ValidateKubernetesParameters(mf, rootLB, mf.Spec.Key) + if err != nil { + return err + } + // Clean up security rules and nginx proxy + if err = DeleteProxySecurityRules(rootLB, mf, kp.ipaddr); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot clean up security rules", "name", mf.Metadata.Name, "rootlb", rootLB.Name, "error", err) + } + // Clean up DNS entries + if err = KubeDeleteDNSRecords(rootLB, mf, kp); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot clean up DNS entries", "name", mf.Metadata.Name, "rootlb", rootLB.Name, "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "deleted deployment", "name", mf.Metadata.Name) + return nil +} + +type KubernetesNode struct { + Name string + Role string + Addr string +} + +type KNodeMetadata struct { + Labels map[string]string `json:"labels"` + //TODO annotations, resourceVersion, creationTimestamp, name, selfLink, uid +} + +type KNodeAddr struct { + Address string `json:"address"` + Type string `json:"type"` +} + +type KNodeStatus struct { + Addresses []KNodeAddr `json:"addresses"` + //TODO allocatable, capacity,conditions,daemonEndpoints,images,nodeInfo, +} + +type KAPINode struct { + ApiVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata KNodeMetadata `json:"metadata"` + //TODO spec + Status KNodeStatus `json:"status"` +} + +type KAPINodes struct { + ApiVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Items []KAPINode `json:"items"` + //TODO metadata +} + +func GetKubernetesNodes(mf *Manifest, rootLB *MEXRootLB) ([]KubernetesNode, error) { + log.DebugLog(log.DebugLevelMexos, "getting kubernetes nodes") + name, err := FindClusterWithKey(mf, mf.Spec.Key) + if err != nil { + return nil, fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) + } + ipaddr, err := FindNodeIP(mf, name) + if err != nil { + return nil, err + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return nil, fmt.Errorf("can't get ssh client for getting kubernetes nodes, %v", err) + } + cmd := fmt.Sprintf("ssh -o %s -o %s -o %s -i id_rsa_mex %s@%s kubectl get nodes -o json", sshOpts[0], sshOpts[1], sshOpts[2], sshUser, ipaddr) + log.DebugLog(log.DebugLevelMexos, "running kubectl get nodes", "cmd", cmd) + out, err := client.Output(cmd) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error checking for kubernetes nodes", "out", out, "err", err) + return nil, fmt.Errorf("error doing kubectl get nodes, %v", err) + } + knodes := KAPINodes{} + err = json.Unmarshal([]byte(out), &knodes) + if err != nil { + return nil, fmt.Errorf("error unmarshalling kubectl get nodes result, %v, %s", err, out) + } + if knodes.Kind != "List" { + return nil, fmt.Errorf("error, kubectl get nodes result is not a list") + } + kl := make([]KubernetesNode, 0) + for _, n := range knodes.Items { + if n.Kind != "Node" { + continue + } + kn := KubernetesNode{} + for _, a := range n.Status.Addresses { + if a.Type == "InternalIP" { + kn.Addr = a.Address + } + if a.Type == "Hostname" { + kn.Name = a.Address + } + if _, ok := n.Metadata.Labels["node-role.kubernetes.io/master"]; ok { + kn.Role = "master" + } else { + kn.Role = "worker" + } + } + kl = append(kl, kn) + } + return kl, nil +} diff --git a/openstack-prov/oscliapi/mex.go b/mexos/kvm.go similarity index 67% rename from openstack-prov/oscliapi/mex.go rename to mexos/kvm.go index 14abec736..86ccf6b1b 100644 --- a/openstack-prov/oscliapi/mex.go +++ b/mexos/kvm.go @@ -1,56 +1,128 @@ -package oscli +package mexos import ( "fmt" "net" "os" "strings" + "time" + sh "github.com/codeskyblue/go-sh" "github.com/mobiledgex/edge-cloud/log" ) -// These are platform specific custom vars -var eMEXImageName = os.Getenv("MEX_OS_IMAGE") // "mobiledgex-16.04-2" -var eMEXUserData = os.Getenv("MEX_USERDATA") // "/home/bob/userdata.txt" -var eMEXDir = os.Getenv("MEX_DIR") -var eMEXSubnetSeed = 100 // XXX -var eMEXSubnetLimit = 250 // XXX -var defaultImage = "mobiledgex-16.04-2" -var defaultFlavor = "x1.medium" -var defaultKVMFlavor = "m4.large" - -const ( - k8smasterRole = "k8s-master" - k8snodeRole = "k8s-node" -) - -//For netspec components -// netType,netName,netCIDR,netOptions -const ( - NetTypeVal = 0 - NetNameVal = 1 - NetCIDRVal = 2 - NetOptVal = 3 -) +//CreateQCOW2AppManifest creates qcow2 app +func CreateQCOW2AppManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "create qcow2 vm based app") + //TODO: support other URI: file://, nfs://, ftp://, git://, or embedded as base64 string + if !strings.HasPrefix(mf.Spec.Image, "http://") && + !strings.HasPrefix(mf.Spec.Image, "https://") { + return fmt.Errorf("unsupported qcow2 image spec %s", mf.Spec.Image) + } + if !strings.Contains(mf.Spec.Flavor, "qcow2") { + return fmt.Errorf("unsupported qcow2 flavor %s", mf.Spec.Flavor) + } + if err := ValidateCommon(mf); err != nil { + return err + } -//NetSpecInfo has basic layout for netspec -type NetSpecInfo struct { - Kind, Name, CIDR, Options string - Extra []string + savedQcowName := mf.Metadata.Name + ".qcow2" // XXX somewhere safe instead + alreadyExist := false + images, err := ListImages(mf) + if err != nil { + return fmt.Errorf("cannot list openstack images, %v", err) + } + for _, img := range images { + if img.Name == mf.Metadata.Name && img.Status == "active" { + log.DebugLog(log.DebugLevelMexos, "warning, glance has image already", "name", mf.Metadata.Name) + if !strings.Contains(mf.Spec.Flags, "force") { + alreadyExist = true + } else { + log.DebugLog(log.DebugLevelMexos, "forced to download image again. delete existing glance image") + if ierr := DeleteImage(mf, mf.Metadata.Name); ierr != nil { + return fmt.Errorf("error deleting glance image %s, %v", mf.Metadata.Name, ierr) + } + } + } + } + if !alreadyExist { + log.DebugLog(log.DebugLevelMexos, "getting qcow2 image", "image", mf.Spec.Image, "name", savedQcowName) + out, cerr := sh.Command("curl", "-s", "-o", savedQcowName, mf.Spec.Image).Output() + if cerr != nil { + return fmt.Errorf("error retrieving qcow image, %s, %s, %v", savedQcowName, out, cerr) + } + finfo, serr := os.Stat(savedQcowName) + if serr != nil { + if os.IsNotExist(serr) { + return fmt.Errorf("downloaded qcow2 file %s does not exist, %v", savedQcowName, serr) + } + return fmt.Errorf("error looking for downloaded qcow2 file %v", serr) + } + if finfo.Size() < 1000 { //too small + return fmt.Errorf("invalid downloaded qcow2 file %s", savedQcowName) + } + log.DebugLog(log.DebugLevelMexos, "qcow2 image being created", "image", mf.Spec.Image, "name", savedQcowName) + err = CreateImage(mf, mf.Metadata.Name, savedQcowName) + if err != nil { + return fmt.Errorf("cannot create openstack glance image instance from %s, %v", savedQcowName, err) + } + log.DebugLog(log.DebugLevelMexos, "saved qcow image to glance", "name", mf.Metadata.Name) + found := false + for i := 0; i < 10; i++ { + images, ierr := ListImages(mf) + if ierr != nil { + return fmt.Errorf("error while getting list of qcow2 glance images, %v", ierr) + } + for _, img := range images { + if img.Name == mf.Metadata.Name && img.Status == "active" { + found = true + break + } + } + if found { + break + } + log.DebugLog(log.DebugLevelMexos, "waiting for the image to become active", "name", mf.Metadata.Name) + time.Sleep(2 * time.Second) + } + if !found { + return fmt.Errorf("timed out waiting for glance to activate the qcow2 image %s", mf.Metadata.Name) + } + } + log.DebugLog(log.DebugLevelMexos, "qcow image is active in glance", "name", mf.Metadata.Name) + if !strings.HasPrefix(mf.Spec.NetworkScheme, "external-ip,") { //XXX for now + return fmt.Errorf("invalid network scheme for qcow2 kvm app, %s", mf.Spec.NetworkScheme) + } + items := strings.Split(mf.Spec.NetworkScheme, ",") + if len(items) < 2 { + return fmt.Errorf("can't find external network name in %s", mf.Spec.NetworkScheme) + } + extNetwork := items[1] + opts := &OSServerOpt{ + Name: mf.Metadata.Name, + Image: mf.Metadata.Name, + Flavor: mf.Spec.ImageFlavor, + NetIDs: []string{extNetwork}, + } + //TODO properties + //TODO userdata + log.DebugLog(log.DebugLevelMexos, "calling create openstack kvm server", "opts", opts) + err = CreateServer(mf, opts) + if err != nil { + return fmt.Errorf("can't create openstack kvm server instance %v, %v", opts, err) + } + log.DebugLog(log.DebugLevelMexos, "created openstack kvm server", "opts", opts) + return nil } -func init() { - if eMEXImageName == "" { - eMEXImageName = defaultImage +func DeleteQCOW2AppManifest(mf *Manifest) error { + if mf.Metadata.Name == "" { + return fmt.Errorf("missing name, no openstack kvm to delete") } - hm := os.Getenv("HOME") - if eMEXDir == "" { - eMEXDir = hm + "/.mobiledgex" + if err := DeleteServer(mf, mf.Metadata.Name); err != nil { + return fmt.Errorf("cannot delete openstack kvm %s, %v", mf.Metadata.Name, err) } - if eMEXUserData == "" { - eMEXUserData = eMEXDir + "/userdata.txt" - } - log.DebugLog(log.DebugLevelMexos, "mex init environment", "MEX_LARGE_IMAGE", eMEXImageName, "MEX_USERDATA", eMEXUserData, "MEX_DIR", eMEXDir) + return nil } //CreateFlavorMEXVM creates basic KVM for mobiledgex applications @@ -59,11 +131,11 @@ func init() { // Roles can be any string but special ones are k8s-master and k8s-node. // To avoid running bootstrap setup for creating kubernets cluster, set skipk8s to true. // For more detailed information please read `mobiledgex-init.sh` -func CreateFlavorMEXVM(name, image, flavor, netID, userdata, role, edgeproxy, skipk8s, k8smaster, privatenet, privaterouter, tags, tenant string) error { +func CreateFlavorMEXVM(mf *Manifest, name, image, flavor, netID, userdata, role, edgeproxy, skipk8s, k8smaster, privatenet, privaterouter, tags, tenant string) error { if name == "" { return fmt.Errorf("name required") } - sd, err := GetServerDetails(name) + sd, err := GetServerDetails(mf, name) if err == nil { log.DebugLog(log.DebugLevelMexos, "warning, server already exists", "name", sd.Name, "server detail", sd) return nil @@ -73,15 +145,15 @@ func CreateFlavorMEXVM(name, image, flavor, netID, userdata, role, edgeproxy, sk } if image == "" { - image = eMEXImageName + image = GetMEXImageName(mf) } if flavor == "" { return fmt.Errorf("Missing platform flavor") } if userdata == "" { - userdata = eMEXUserData + userdata = GetMEXUserData(mf) } - opts := &ServerOpt{ + opts := &OSServerOpt{ Name: name, Image: image, Flavor: flavor, @@ -106,9 +178,15 @@ func CreateFlavorMEXVM(name, image, flavor, netID, userdata, role, edgeproxy, sk props = append(props, "privaterouter="+privaterouter) props = append(props, "tags="+tags) props = append(props, "tenant="+tenant) + if mf.Values.Network.HolePunch != "" { + props = append(props, "holepunch="+mf.Values.Network.HolePunch) + } + if mf.Values.Registry.Update != "" { + props = append(props, "update="+mf.Values.Registry.Update) + } opts.Properties = props log.DebugLog(log.DebugLevelMexos, "create flavor MEX KVM", "flavor", flavor, "server opts", opts) - err = CreateServer(opts) + err = CreateServer(mf, opts) if err != nil { log.DebugLog(log.DebugLevelMexos, "error creating flavor MEX KVM", "server opts", opts) return fmt.Errorf("can't create server, opts %v, %v", opts, err) @@ -116,23 +194,14 @@ func CreateFlavorMEXVM(name, image, flavor, netID, userdata, role, edgeproxy, sk return nil } -func getNameTag(name string) string { - if strings.Index(name, "-") > 0 { - ix := strings.LastIndex(name, "-") - if len(name) > (ix + 1) { - return name[ix+1:] - } - } - return name -} - //CreateMEXKVM is easier way to create a MEX app capable KVM // role can be k8s-master, k8s-node, or something else -func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlavor string) error { - log.DebugLog(log.DebugLevelMexos, "createMEXKVM", "name", name, "role", role, "netSpec", netSpec, +func CreateMEXKVM(mf *Manifest, name, role, netSpec, tags, tenant string, id int, platformFlavor string) error { + log.DebugLog(log.DebugLevelMexos, "createMEXKVM", + "name", name, "role", role, "netSpec", netSpec, "tags", tags, "tenant", tenant, "id", id) - mexRouter := GetMEXExternalRouter() - netID := GetMEXExternalNetwork() //do we really want to default to ext? + mexRouter := GetMEXExternalRouter(mf) + netID := GetMEXExternalNetwork(mf) //do we really want to default to ext? skipk8s := "yes" var masterIP, privRouterIP, privNet, edgeProxy string var err error @@ -168,13 +237,14 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav if ni.CIDR == "" { return fmt.Errorf("missing CIDR spec in %v", ni) } - if ni.Name != GetMEXNetwork() { //XXX for now - return fmt.Errorf("netspec net name %s not equal to default MEX net %s", ni.Name, GetMEXNetwork()) + if ni.Name != GetMEXNetwork(mf) { //XXX for now + return fmt.Errorf("netspec net name %s not equal to default MEX net %s", ni.Name, GetMEXNetwork(mf)) } //XXX openstack bug - subnet does not take tags but description field can be used to tag stuff // Use tag as part of name - sn := ni.Name + "-subnet-" + getNameTag(name) - snd, snderr := GetSubnetDetail(sn) + sn := ni.Name + "-subnet-" + mf.Values.Cluster.Name + log.DebugLog(log.DebugLevelMexos, "using subnet name", "subnet", sn) + snd, snderr := GetSubnetDetail(mf, sn) if snderr != nil { if role == k8snodeRole { return fmt.Errorf("subnet %s does not exist, %v", sn, snderr) @@ -212,7 +282,7 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav if octno != 2 { return fmt.Errorf("net CIDR, we want octno to be 2 for now") } - sns, snserr := ListSubnets(ni.Name) + sns, snserr := ListSubnets(mf, ni.Name) if snserr != nil { return fmt.Errorf("can't get list of subnets for %s, %v", ni.Name, snserr) } @@ -257,12 +327,12 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav // should not happen return fmt.Errorf("subnet %s not found", sn) } - id = id + eMEXSubnetSeed + id = id + MEXSubnetSeed log.DebugLog(log.DebugLevelMexos, "node id", "id", id) // worker nodes start at 100+id. // there may be many masters... allow for upto 100! //leave some space at end - if id > eMEXSubnetLimit { + if id > MEXSubnetLimit { log.DebugLog(log.DebugLevelMexos, "error k8s-node is is too big", "id", id) return fmt.Errorf("k8s-node id is too big") } @@ -278,18 +348,18 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav ni.CIDR = fmt.Sprintf("%d.%d.%d.%d/%s", v4a[0], v4a[1], v4a[2], id, sits[1]) // we can have up to 150 nodes of workers per subnet. // change these values as needed. - sl, err := ListSubnets(ni.Name) + sl, err := ListSubnets(mf, ni.Name) if err != nil { return fmt.Errorf("can't get a list of subnets, %v", err) } for _, sn := range sl { - sd, err := GetSubnetDetail(sn.ID) + sd, err := GetSubnetDetail(mf, sn.ID) if err != nil { return fmt.Errorf("can't get subnet detail, %s, %v", sn.Name, err) } if sd.CIDR == ni.CIDR { log.DebugLog(log.DebugLevelMexos, "subnet exists with the same CIDR, find another range", "CIDR", sd.CIDR) - cidr, err := getNewSubnetRange(id, v4a, sits, sl) + cidr, err := getNewSubnetRange(mf, id, v4a, sits, sl) if err != nil { return fmt.Errorf("failed to get a new subnet range, %v", err) } @@ -311,12 +381,12 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav if role == k8smasterRole { if snd == nil { log.DebugLog(log.DebugLevelMexos, "k8s master, no existing subnet, creating subnet", "name", sn) - err = CreateSubnet(ni.CIDR, GetMEXNetwork(), edgeProxy, sn, false) + err = CreateSubnet(mf, ni.CIDR, GetMEXNetwork(mf), edgeProxy, sn, false) if err != nil { return err } //TODO: consider adding tags to subnet - err = AddRouterSubnet(mexRouter, sn) + err = AddRouterSubnet(mf, mexRouter, sn) if err != nil { return fmt.Errorf("cannot add router %s to subnet %s, %v", mexRouter, sn, err) } @@ -330,7 +400,7 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav //master node num is 1 //so, master node will always have .2 //XXX master always at X.X.X.2 - netID = GetMEXNetwork() + ",v4-fixed-ip=" + ipaddr.String() + netID = GetMEXNetwork(mf) + ",v4-fixed-ip=" + ipaddr.String() masteripaddr := net.IPv4(v4[0], v4[1], v4[2], byte(2)) masterIP = masteripaddr.String() log.DebugLog(log.DebugLevelMexos, "k8s master ip addr", "netID", netID, "ipaddr", ipaddr, "masterip", masterIP) @@ -338,18 +408,18 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav // NOT k8s case // for now just agent stuff. log.DebugLog(log.DebugLevelMexos, "create mex kvm, plain kvm, not kubernetes case") - edgeProxy, err = GetExternalGateway(netID) + edgeProxy, err = GetExternalGateway(mf, netID) if err != nil { return fmt.Errorf("can't get external gateway for %s, %v", netID, err) } log.DebugLog(log.DebugLevelMexos, "external gateway", "external gateway, edgeproxy", edgeProxy) - rd, rderr := GetRouterDetail(mexRouter) + rd, rderr := GetRouterDetail(mf, mexRouter) if rderr != nil { return fmt.Errorf("can't get router detail for %s, %v", mexRouter, rderr) } log.DebugLog(log.DebugLevelMexos, "router detail", "detail", rd) - reg, regerr := GetRouterDetailExternalGateway(rd) + reg, regerr := GetRouterDetailExternalGateway(mf, rd) if regerr != nil { //return fmt.Errorf("can't get router detail external gateway, %v", regerr) log.DebugLog(log.DebugLevelMexos, "can't get router detail, not fatal") @@ -382,11 +452,11 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav if err != nil { return fmt.Errorf("cannot get flavor from tags '%s'", tags) } - err = CreateFlavorMEXVM(name, - eMEXImageName, + err = CreateFlavorMEXVM(mf, name, + GetMEXImageName(mf), platformFlavor, netID, // either external-net or internal-net,v4-fixed-ip=X.X.X.X - eMEXUserData, + GetMEXUserData(mf), role, // k8s-master,k8s-node,something else edgeProxy, skipk8s, // if yes, skip @@ -404,13 +474,13 @@ func CreateMEXKVM(name, role, netSpec, tags, tenant string, id int, platformFlav return nil } -func getNewSubnetRange(id int, v4a []byte, sits []string, sl []Subnet) (*string, error) { +func getNewSubnetRange(mf *Manifest, id int, v4a []byte, sits []string, sl []OSSubnet) (*string, error) { var cidr string - for newID := id + 1; newID < eMEXSubnetLimit; newID++ { + for newID := id + 1; newID < MEXSubnetLimit; newID++ { cidr = fmt.Sprintf("%d.%d.%d.%d/%s", v4a[0], v4a[1], v4a[2], newID, sits[1]) found := false for _, snn := range sl { - sdd, err := GetSubnetDetail(snn.ID) + sdd, err := GetSubnetDetail(mf, snn.ID) if err != nil { return nil, fmt.Errorf("cannot get subnet detail %s, %v", snn.Name, err) } @@ -428,27 +498,29 @@ func getNewSubnetRange(id int, v4a []byte, sits []string, sl []Subnet) (*string, //DestroyMEXKVM deletes the MEX KVM instance. If server instance is k8s-master, // first remove router from subnet which was created for it. Then remove subnet before // deleting server KVM instance. -func DestroyMEXKVM(name, role string) error { +func DestroyMEXKVM(mf *Manifest, name, role string) error { + //TODO send shutdown command to running VM. Left undone so we insteadi optionally + // send ssh shutdown manually before deleting the KVM instance via API or mexctl. log.DebugLog(log.DebugLevelMexos, "delete mex kvm server", "name", name, "role", role) - err := DeleteServer(name) + err := DeleteServer(mf, name) if err != nil { return fmt.Errorf("can't delete %s, %v", name, err) } if role == k8smasterRole { - sn := "subnet-" + getNameTag(name) - rn := GetMEXExternalRouter() + sn := "subnet-" + mf.Values.Cluster.Name + rn := GetMEXExternalRouter(mf) log.DebugLog(log.DebugLevelMexos, "removing router from subnet", "router", rn, "subnet", sn) - err := RemoveRouterSubnet(rn, sn) + err := RemoveRouterSubnet(mf, rn, sn) if err != nil { - return fmt.Errorf("can't remove router %s from subnet %s, %v", rn, sn, err) + return fmt.Errorf("can't remove subnet %s from router %s, %v", sn, rn, err) } //XXX This may not work until all nodes are removed, since // IP addresses are allocated out of this subnet // All nodes should be deleted first. log.DebugLog(log.DebugLevelMexos, "deleting subnet", "name", sn) - err = DeleteSubnet(sn) + err = DeleteSubnet(mf, sn) if err != nil { return fmt.Errorf("can't delete subnet %s, %v", sn, err) } @@ -462,9 +534,10 @@ func ParseNetSpec(netSpec string) (*NetSpecInfo, error) { if netSpec == "" { return nil, fmt.Errorf("empty netspec") } + log.DebugLog(log.DebugLevelMexos, "parsing netspec", "netspec", netSpec) items := strings.Split(netSpec, ",") if len(items) < 3 { - return nil, fmt.Errorf("malformed net spec, insufficient items") + return nil, fmt.Errorf("malformed net spec, insufficient items %v", items) } ni.Kind = items[NetTypeVal] ni.Name = items[NetNameVal] @@ -475,6 +548,7 @@ func ParseNetSpec(netSpec string) (*NetSpecInfo, error) { if len(items) > 5 { ni.Extra = items[NetOptVal+1:] } + log.DebugLog(log.DebugLevelMexos, "netspec info", "ni", ni, "items", items) switch items[NetTypeVal] { case "priv-subnet": return ni, nil diff --git a/mexos/lb.go b/mexos/lb.go new file mode 100644 index 000000000..19fb60188 --- /dev/null +++ b/mexos/lb.go @@ -0,0 +1,94 @@ +package mexos + +import ( + "fmt" + "net" + "strings" + + "github.com/mobiledgex/edge-cloud/log" +) + +//LBAddRoute adds a route to LB +func LBAddRoute(mf *Manifest, rootLBName, extNet, name string) error { + if rootLBName == "" { + return fmt.Errorf("empty rootLB") + } + if name == "" { + return fmt.Errorf("empty name") + } + if extNet == "" { + return fmt.Errorf("empty external network") + } + ap, err := LBGetRoute(mf, rootLBName, name) + if err != nil { + return err + } + if len(ap) != 2 { + return fmt.Errorf("expected 2 addresses, got %d", len(ap)) + } + cmd := fmt.Sprintf("sudo ip route add %s via %s dev ens3", ap[0], ap[1]) + client, err := GetSSHClient(mf, rootLBName, extNet, sshUser) + if err != nil { + return err + } + out, err := client.Output(cmd) + if err != nil { + if strings.Contains(out, "RTNETLINK") && strings.Contains(out, " exists") { + log.DebugLog(log.DebugLevelMexos, "warning, can't add existing route to rootLB", "cmd", cmd, "out", out, "error", err) + return nil + } + return fmt.Errorf("can't add route to rootlb, %s, %s, %v", cmd, out, err) + } + return nil +} + +//LBRemoveRoute removes route for LB +func LBRemoveRoute(mf *Manifest, rootLB, extNet, name string) error { + ap, err := LBGetRoute(mf, rootLB, name) + if err != nil { + return err + } + if len(ap) != 2 { + return fmt.Errorf("expected 2 addresses, got %d", len(ap)) + } + cmd := fmt.Sprintf("sudo ip route delete %s via %s dev ens3", ap[0], ap[1]) + client, err := GetSSHClient(mf, rootLB, extNet, sshUser) + if err != nil { + return err + } + out, err := client.Output(cmd) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "can't delete route at rootLB", "cmd", cmd, "out", out, "error", err) + //not a fatal error + return nil + } + return nil +} + +//LBGetRoute returns route of LB +func LBGetRoute(mf *Manifest, rootLB, name string) ([]string, error) { + cidr, err := GetInternalCIDR(mf, name) + if err != nil { + return nil, err + } + _, ipn, err := net.ParseCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("can't parse %s, %v", cidr, err) + } + v4 := ipn.IP.To4() + dn := fmt.Sprintf("%d.%d.%d.0/24", v4[0], v4[1], v4[2]) + rn := GetMEXExternalRouter(mf) + rd, err := GetRouterDetail(mf, rn) + if err != nil { + return nil, fmt.Errorf("can't get router detail for %s, %v", rn, err) + } + reg, err := GetRouterDetailExternalGateway(mf, rd) + if err != nil { + return nil, fmt.Errorf("can't get router detail external gateway, %v", err) + } + if len(reg.ExternalFixedIPs) < 1 { + return nil, fmt.Errorf("can't get external fixed ips list from router detail external gateway") + } + fip := reg.ExternalFixedIPs[0] + return []string{dn, fip.IPAddress}, nil +} diff --git a/mexos/letsencrypt.go b/mexos/letsencrypt.go new file mode 100644 index 000000000..a640b6920 --- /dev/null +++ b/mexos/letsencrypt.go @@ -0,0 +1,135 @@ +package mexos + +import ( + "fmt" + "io/ioutil" + "strings" + "time" + + sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud/log" +) + +//TODO acme.sh seems to leave out _acme domains. We need to clean up periodically + +func checkPEMFile(fn string) error { + content, err := ioutil.ReadFile(fn) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "can't read downloaded pem file", "name", fn, "error", err) + return fmt.Errorf("can't read downloaded pem file %s, %v", fn, err) + } + contstr := string(content) + if strings.Contains(contstr, "404 Not Found") { + log.DebugLog(log.DebugLevelMexos, "404 not found in pem file", "name", fn) + return fmt.Errorf("registry does not have the pem file %s", fn) + } + if !strings.HasPrefix(contstr, "-----BEGIN") { + log.DebugLog(log.DebugLevelMexos, "does not look like pem file", "name", fn) + return fmt.Errorf("does not look like pem file %s", fn) + } + return nil +} + +//AcquireCertificates obtains certficates from Letsencrypt over ACME. It should be used carefully. The API calls have quota. +func AcquireCertificates(mf *Manifest, rootLB *MEXRootLB, fqdn string) error { + log.DebugLog(log.DebugLevelMexos, "acquiring certificates for FQDN", "FQDN", fqdn) + if rootLB == nil { + return fmt.Errorf("cannot acquire certs, rootLB is null") + } + if mf.Values.Network.External == "" { + return fmt.Errorf("acquire certificate, missing external network in manifest") + } + if cerr := CheckCredentialsCF(mf); cerr != nil { + return cerr + } + kf := PrivateSSHKey() + srcfile := fmt.Sprintf("mobiledgex@%s:files-repo/certs/%s/fullchain.cer", mf.Values.Registry.Name, fqdn) + dkey := fmt.Sprintf("%s/%s.key", fqdn, fqdn) + certfile := "cert.pem" + keyfile := "key.pem" + log.DebugLog(log.DebugLevelMexos, "trying to get cached cert files") + out, err := sh.Command("scp", "-o", sshOpts[0], "-o", sshOpts[1], "-i", kf, srcfile, certfile).Output() + if err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, failed to get cached cert file", "src", srcfile, "cert", certfile, "error", err, "out", out) + } else if checkPEMFile(certfile) == nil { + srcfile = fmt.Sprintf("mobiledgex@%s:files-repo/certs/%s", mf.Values.Registry.Name, dkey) + out, err = sh.Command("scp", "-o", sshOpts[0], "-o", sshOpts[1], "-i", kf, srcfile, keyfile).Output() + if err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, failed to get cached key file", "src", srcfile, "cert", certfile, "error", err, "out", out) + } else if checkPEMFile(keyfile) == nil { + //because Letsencrypt complains if we get certs repeated for the same fqdn + log.DebugLog(log.DebugLevelMexos, "got cached certs from registry", "FQDN", fqdn) + addr, ierr := GetServerIPAddr(mf, mf.Values.Network.External, fqdn) //XXX should just use fqdn but paranoid + if ierr != nil { + log.DebugLog(log.DebugLevelMexos, "failed to get server ip addr", "FQDN", fqdn, "error", ierr) + return ierr + } + client, err := GetSSHClient(mf, fqdn, mf.Values.Network.External, sshUser) + if err != nil { + return fmt.Errorf("can't get ssh client for cert, %v", err) + } + for _, fn := range []string{certfile, keyfile} { + out, oerr := sh.Command("scp", "-o", sshOpts[0], "-o", sshOpts[1], "-i", kf, fn, sshUser+"@"+addr+":").CombinedOutput() + if oerr != nil { + return fmt.Errorf("cannot copy %s to %s, %v, %v", fn, addr, oerr, string(out)) + } + log.DebugLog(log.DebugLevelMexos, "copied", "fn", fn, "addr", addr) + if out, err := client.Output(fmt.Sprintf("sudo cp %s /root", fn)); err != nil { + return fmt.Errorf("cannot copy %s to /root, %v, %s", fn, err, out) + } + } + log.DebugLog(log.DebugLevelMexos, "using cached cert and key", "FQDN", fqdn) + return nil + } + } + log.DebugLog(log.DebugLevelMexos, "did not get cached cert and key files, will try to acquire new cert") + client, err := GetSSHClient(mf, fqdn, mf.Values.Network.External, sshUser) + if err != nil { + return fmt.Errorf("can't get ssh client for acme.sh, %v", err) + } + fullchain := fqdn + "/fullchain.cer" + cmd := fmt.Sprintf("ls -a %s", fullchain) + _, err = client.Output(cmd) + if err == nil { + return nil + } + cmd = fmt.Sprintf("docker run --rm -e CF_Key=%s -e CF_Email=%s -v `pwd`:/acme.sh --net=host neilpang/acme.sh --issue -d %s --dns dns_cf", mexEnv(mf, "MEX_CF_KEY"), mexEnv(mf, "MEX_CF_USER"), fqdn) + res, err := client.Output(cmd) + if err != nil { + return fmt.Errorf("error running acme.sh docker, %s, %v", res, err) + } + cmd = fmt.Sprintf("ls -a %s", fullchain) + success := false + for i := 0; i < 10; i++ { + _, err = client.Output(cmd) + if err == nil { + success = true + break + } + log.DebugLog(log.DebugLevelMexos, "waiting for letsencrypt...") + time.Sleep(30 * time.Second) // ACME takes minimum 200 seconds + } + if !success { + return fmt.Errorf("timeout waiting for ACME") + } + for _, d := range []struct{ src, dest string }{ + {fullchain, certfile}, + {dkey, keyfile}, + } { + cmd = fmt.Sprintf("cp %s %s", d.src, d.dest) + res, err := client.Output(cmd) + if err != nil { + return fmt.Errorf("fail to copy %s to %s on %s, %v, %v", d.src, d.dest, fqdn, err, res) + } + if out, err := client.Output(fmt.Sprintf("sudo cp %s /root", d.dest)); err != nil { + return fmt.Errorf("cannot copy %s to /root, %v, %s", d.dest, err, out) + } + } + cmd = fmt.Sprintf("scp -o %s -o %s -i id_rsa_mex -r %s mobiledgex@%s:files-repo/certs", sshOpts[0], sshOpts[1], fqdn, mf.Values.Registry.Name) // XXX + res, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("failed to upload certs for %s, %v, %v", fqdn, err, res) + } + log.DebugLog(log.DebugLevelMexos, "saved acquired cert and key", "FQDN", fqdn) + return nil +} diff --git a/mexos/manifest.go b/mexos/manifest.go new file mode 100644 index 000000000..259fa5e83 --- /dev/null +++ b/mexos/manifest.go @@ -0,0 +1,328 @@ +package mexos + +import ( + "bytes" + "fmt" + "strings" + "text/template" + + "github.com/ghodss/yaml" + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/azure" + "github.com/mobiledgex/edge-cloud-infra/k8s-prov/gcloud" + "github.com/mobiledgex/edge-cloud/cloudcommon" + "github.com/mobiledgex/edge-cloud/log" +) + +//MEXClusterCreateManifest creates a cluster +func MEXClusterCreateManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "creating cluster via manifest") + switch mf.Metadata.Operator { + case "gcp": + return gcloudCreateGKE(mf) + case "azure": + return azureCreateAKS(mf) + default: + //guid, err := mexCreateClusterKubernetes(mf) + err := mexCreateClusterKubernetes(mf) + if err != nil { + return fmt.Errorf("can't create cluster, %v", err) + } + //log.DebugLog(log.DebugLevelMexos, "new guid", "guid", *guid) + log.DebugLog(log.DebugLevelMexos, "created kubernetes cluster") + return nil + } +} + +//MEXAddFlavor adds flavor using manifest +func MEXAddFlavor(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "add mex flavor") + //TODO use full manifest and validate against platform data + return AddFlavorManifest(mf) +} + +// TODO DeleteFlavor -- but almost never done + +// TODO lookup guid using name + +//MEXClusterRemoveManifest removes a cluster +func MEXClusterRemoveManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "removing cluster") + switch mf.Metadata.Operator { + case "gcp": + return gcloud.DeleteGKECluster(mf.Metadata.Name) + case "azure": + return azure.DeleteAKSCluster(mf.Metadata.ResourceGroup) + default: + if err := mexDeleteClusterKubernetes(mf); err != nil { + return fmt.Errorf("can't remove cluster, %v", err) + } + return nil + } +} + +//MEXPlatformInitCloudletKey calls MEXPlatformInit with templated manifest +func MEXPlatformInitCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) error { + ckmf, err := fillPlatformTemplateCloudletKey(rootLB, cloudletKeyStr) + if err != nil { + return err + } + ckmf.Values = rootLB.PlatConf.Values + return MEXPlatformInitManifest(ckmf) +} + +//MEXPlatformCleanCloudletKey calls MEXPlatformClean with templated manifest +func MEXPlatformCleanCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) error { + mf, err := fillPlatformTemplateCloudletKey(rootLB, cloudletKeyStr) + if err != nil { + return err + } + return MEXPlatformCleanManifest(mf) +} + +//MEXPlatformInitManifest initializes platform +func MEXPlatformInitManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "init platform") + err := MEXCheckEnvVars(mf) + if err != nil { + return err + } + switch mf.Metadata.Operator { + case "gcp": + return nil //nothing to do + case "azure": + return nil //nothing to do + default: + if err = MEXCheckEnvVars(mf); err != nil { + return err + } + //TODO validate all mf content against platform data + if err = RunMEXAgentManifest(mf); err != nil { + return err + } + } + return nil +} + +//MEXPlatformCleanManifest cleans up the platform +func MEXPlatformCleanManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "clean platform") + err := MEXCheckEnvVars(mf) + if err != nil { + return err + } + switch mf.Metadata.Operator { + case "gcp": + return nil //nothing to do + case "azure": + return nil + default: + if err = MEXCheckEnvVars(mf); err != nil { + return err + } + if err = RemoveMEXAgentManifest(mf); err != nil { + return err + } + } + return nil +} + +//MEXAppCreateAppManifest creates app instances on the cluster platform +func MEXAppCreateAppManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "create app from manifest") + appDeploymentType := mf.Config.ConfigDetail.Deployment + log.DebugLog(log.DebugLevelMexos, "app deployment", "imageType", mf.Spec.ImageType, "deploymentType", appDeploymentType, "config", mf.Config) + var kubeManifest string + var err error + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + kubeManifest, err = GetKubeManifest(mf) + if err != nil { + return err + } + if !strings.HasPrefix(kubeManifest, "apiVersion: v1") { + log.DebugLog(log.DebugLevelMexos, "bad apiVersion at beginning kubemanifest") + return fmt.Errorf("bad apiversion at beginning of kube manifest") + } + } + //TODO values.application.template + switch mf.Metadata.Operator { + case "gcp": + fallthrough + case "azure": + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + return runKubectlCreateApp(mf, kubeManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { + return fmt.Errorf("not yet supported") + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + return fmt.Errorf("not yet supported") + } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { + return fmt.Errorf("not yet supported") + } + return fmt.Errorf("unknown deployment type %s", appDeploymentType) + default: + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + return CreateKubernetesAppManifest(mf, kubeManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { + return CreateQCOW2AppManifest(mf) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + return CreateHelmAppManifest(mf) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { + return CreateDockerSwarmAppManifest(mf) + } + return fmt.Errorf("unknown deployment type %s", appDeploymentType) + } +} + +//MEXAppDeleteManifest kills app +func MEXAppDeleteAppManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "delete app with manifest") + appDeploymentType := mf.Config.ConfigDetail.Deployment + log.DebugLog(log.DebugLevelMexos, "app delete", "imageType", mf.Spec.ImageType, "deploymentType", appDeploymentType, "config", mf.Config) + kubeManifest, err := GetKubeManifest(mf) + if err != nil { + return err + } + switch mf.Metadata.Operator { + case "gcp": + fallthrough + case "azure": + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + return runKubectlDeleteApp(mf, kubeManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { + return fmt.Errorf("not yet supported") + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + return fmt.Errorf("not yet supported") + } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { + return fmt.Errorf("not yet supported") + } + return fmt.Errorf("unknown image type %s", appDeploymentType) + default: + if appDeploymentType == cloudcommon.AppDeploymentTypeKubernetes { + return DeleteKubernetesAppManifest(mf, kubeManifest) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeKVM { + return DeleteQCOW2AppManifest(mf) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeHelm { + return DeleteHelmAppManifest(mf) + } else if appDeploymentType == cloudcommon.AppDeploymentTypeDockerSwarm { + return DeleteDockerSwarmAppManifest(mf) + } + return fmt.Errorf("unknown image type %s", mf.Spec.ImageType) + } +} + +func GetDefaultRegistryBase(mf *Manifest, base string) string { + mf.Base = base + if mf.Base == "" { + mf.Base = fmt.Sprintf("scp://%s/files-repo/mobiledgex", mf.Values.Registry.Name) + } + log.DebugLog(log.DebugLevelMexos, "default registry base", "base", mf.Base) + return mf.Base +} + +func FillManifestValues(mf *Manifest, kind, base string) error { + if mf.Values.Name == "" { + return fmt.Errorf("no name for mf values") + } + base = GetDefaultRegistryBase(mf, base) + var uri string + switch kind { + case "openstack": + kind = "platform" + fallthrough + case "platform": + fallthrough + case "cluster": + uri = fmt.Sprintf("%s/%s/%s/%s.yaml", mf.Base, kind, mf.Values.Operator.Name, mf.Values.Base) + case "application": + uri = fmt.Sprintf("%s/%s/%s/%s.yaml", mf.Base, kind, mf.Values.Application.Base, mf.Values.Base) + default: + return fmt.Errorf("invalid manifest kind %s", kind) + } + dat, err := GetURIFile(mf, uri) + if err != nil { + return err + } + //log.DebugLog(log.DebugLevelMexos, "got file", "uri", uri, "data", string(dat)) + tmpl, err := template.New(mf.Values.Name).Parse(string(dat)) + if err != nil { + return err + } + var outbuffer bytes.Buffer + //log.DebugLog(log.DebugLevelMexos, "mf values", "values", mf.Values) + err = tmpl.Execute(&outbuffer, &mf.Values) + if err != nil { + return err + } + err = yaml.Unmarshal(outbuffer.Bytes(), mf) + if err != nil { + return err + } + return nil +} + +func GetDefaultSecurityRule(mf *Manifest) string { + return mf.Values.Network.SecurityRule +} + +func GetMEXSecurityRule(mf *Manifest) string { + return mf.Values.Network.SecurityRule +} + +//GetMEXExternalRouter returns default MEX external router name +func GetMEXExternalRouter(mf *Manifest) string { + //TODO validate existence and status + return mf.Values.Network.Router +} + +//GetMEXExternalNetwork returns default MEX external network name +func GetMEXExternalNetwork(mf *Manifest) string { + //TODO validate existence and status + return mf.Values.Network.External +} + +//GetMEXNetwork returns default MEX network, internal and prepped +func GetMEXNetwork(mf *Manifest) string { + //TODO validate existence and status + return mf.Values.Network.Name +} + +func GetMEXImageName(mf *Manifest) string { + return mf.Values.Cluster.OSImage +} + +func GetMEXUserData(mf *Manifest) string { + return MEXDir() + "/userdata.txt" +} + +func GetKubeManifest(mf *Manifest) (string, error) { + var kubeManifest string + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return "", fmt.Errorf("cannot get rootlb, while getting kubemanifest, %v", err) + } + base := rootLB.PlatConf.Base + if base == "" { + log.DebugLog(log.DebugLevelMexos, "base is empty, using default") + base = GetDefaultRegistryBase(mf, base) + } + mani := mf.Config.ConfigDetail.Manifest + //XXX controlling pass full yaml text in parameter of another yaml + log.DebugLog(log.DebugLevelMexos, "getting kubernetes manifest", "base", base, "manifest", mani) + if !strings.Contains(mani, "://") { + fn := fmt.Sprintf("%s/%s", base, mani) + log.DebugLog(log.DebugLevelMexos, "getting manifest file", "uri", fn) + res, err := GetURIFile(mf, fn) + if err != nil { + return "", err + } + kubeManifest = string(res) + } else { + //XXX controller is passing full yaml as a string. + log.DebugLog(log.DebugLevelMexos, "getting deployment from cloudcommon", "base", mf.Base, "manifest", mani) + //XXX again it seems to download yaml but already yaml full string is passed from controller + kubeManifest, err = cloudcommon.GetDeploymentManifest(mf.Config.ConfigDetail.Manifest) + if err != nil { + return "", err + } + } + return kubeManifest, nil +} diff --git a/mexos/misc.go b/mexos/misc.go new file mode 100644 index 000000000..d9478051d --- /dev/null +++ b/mexos/misc.go @@ -0,0 +1,91 @@ +package mexos + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/mobiledgex/edge-cloud/log" + "github.com/mobiledgex/edge-cloud/util" +) + +func PrivateSSHKey() string { + return MEXDir() + "/id_rsa_mex" +} + +func MEXDir() string { + return os.Getenv("HOME") + "/.mobiledgex" +} + +func defaultKubeconfig() string { + return os.Getenv("HOME") + "/.kube/config" +} + +func copyFile(src string, dst string) error { + data, err := ioutil.ReadFile(src) + if err != nil { + return err + } + err = ioutil.WriteFile(dst, data, 0644) + if err != nil { + return err + } + return nil +} + +func writeKubeManifest(kubeManifest string, filename string) error { + outFile, err := os.OpenFile(filename, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("unable to open k8s deployment file %s: %s", filename, err.Error()) + } + _, err = outFile.WriteString(kubeManifest) + if err != nil { + outFile.Close() + os.Remove(filename) + return fmt.Errorf("unable to write k8s deployment file %s: %s", filename, err.Error()) + } + outFile.Sync() + outFile.Close() + return nil +} + +func NormalizeName(name string) string { + return util.K8SSanitize(name) // XXX +} + +func SeedDockerSecret(mf *Manifest, rootLB *MEXRootLB) error { + log.DebugLog(log.DebugLevelMexos, "seed docker secret") + name, err := FindClusterWithKey(mf, mf.Spec.Key) + if err != nil { + return err + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return fmt.Errorf("can't get ssh client for docker swarm, %v", err) + } + masteraddr, err := FindNodeIP(mf, name) + if err != nil { + return err + } + var out string + cmd := fmt.Sprintf("echo %s > .docker-pass", mexEnv(mf, "MEX_DOCKER_REG_PASS")) + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("can't store docker password, %s, %v", out, err) + } + log.DebugLog(log.DebugLevelMexos, "stored docker password") + cmd = fmt.Sprintf("scp -o %s -o %s -i id_rsa_mex .docker-pass %s:", sshOpts[0], sshOpts[1], masteraddr) + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("can't copy docker password to k8s-master, %s, %v", out, err) + } + log.DebugLog(log.DebugLevelMexos, "copied over docker password") + cmd = fmt.Sprintf("ssh -o %s -o %s -i id_rsa_mex %s 'cat .docker-pass| docker login -u mobiledgex --password-stdin %s'", sshOpts[0], sshOpts[1], masteraddr, mf.Values.Registry.Docker) + //TODO allow different docker registry as specified in the manifest + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("can't docker login on k8s-master to %s, %s, %v", mf.Values.Registry.Docker, out, err) + } + log.DebugLog(log.DebugLevelMexos, "docker login ok") + return nil +} diff --git a/mexos/network.go b/mexos/network.go new file mode 100644 index 000000000..88ea3446b --- /dev/null +++ b/mexos/network.go @@ -0,0 +1,99 @@ +package mexos + +import ( + "encoding/json" + "fmt" + "net" + "strings" + + "github.com/mobiledgex/edge-cloud/log" +) + +//GetExternalGateway retrieves Gateway IP from the external network information. It first gets external +// network information. Using that it further gets subnet information. Inside that subnet information +// there should be gateway IP if the network is set up correctly. +// Not to be confused with GetRouterDetailExternalGateway. +func GetExternalGateway(mf *Manifest, extNetName string) (string, error) { + nd, err := GetNetworkDetail(mf, extNetName) + if err != nil { + return "", fmt.Errorf("can't get details for external network %s, %v", extNetName, err) + } + + if nd.Status != "ACTIVE" { + return "", fmt.Errorf("network %s is not active, status %s", extNetName, nd.Status) + } + if nd.AdminStateUp != "UP" { + return "", fmt.Errorf("network %s is not admin-state set to up", extNetName) + } + subnets := strings.Split(nd.Subnets, ",") + //XXX beware of extra spaces + if len(subnets) < 1 { + return "", fmt.Errorf("no subnets for %s", extNetName) + } + //XXX just use first subnet -- may not work in all cases, but there is no tagging done rightly yet + sd, err := GetSubnetDetail(mf, subnets[0]) + if err != nil { + return "", fmt.Errorf("cannot get details for subnet %s, %v", subnets[0], err) + } + //TODO check status of subnet + if sd.GatewayIP == "" { + return "", fmt.Errorf("cannot get external network's gateway IP") + } + log.DebugLog(log.DebugLevelMexos, "get external gatewayIP", "gatewayIP", sd.GatewayIP, "subnet detail", sd) + return sd.GatewayIP, nil +} + +//GetNextSubnetRange will find the CIDR for the next range of subnet that can be created. For example, +// if the subnet detail we get has 10.101.101.0/24 then the next one can be 10.101.102.0/24 +func GetNextSubnetRange(mf *Manifest, subnetName string) (string, error) { + sd, err := GetSubnetDetail(mf, subnetName) + if err != nil { + return "", err + } + if sd.CIDR == "" { + return "", fmt.Errorf("missing CIDR in subnet %s", subnetName) + } + _, ipv4Net, err := net.ParseCIDR(sd.CIDR) + if err != nil { + return "", fmt.Errorf("can't parse CIDR %s, %v", sd.CIDR, err) + } + i := strings.Index(sd.CIDR, "/") + suffix := sd.CIDR[i:] + v4 := ipv4Net.IP.To4() + ipnew := net.IPv4(v4[0], v4[1], v4[2]+1, v4[3]) + log.DebugLog(log.DebugLevelMexos, "get next subnet range", "new ip range", ipnew, "suffix", suffix) + return ipnew.String() + suffix, nil +} + +//GetRouterDetailExternalGateway is different than GetExternalGateway. This function gets +// the gateway interface in the subnet within external network. This is +// accessible from private networks to route packets to the external network. +// The GetExternalGateway gets the gateway for the outside network. This is +// for the packets to be routed out to the external network, i.e. internet. +func GetRouterDetailExternalGateway(mf *Manifest, rd *OSRouterDetail) (*OSExternalGateway, error) { + if rd.ExternalGatewayInfo == "" { + return nil, fmt.Errorf("empty external gateway info") + } + externalGateway := &OSExternalGateway{} + err := json.Unmarshal([]byte(rd.ExternalGatewayInfo), externalGateway) + if err != nil { + return nil, fmt.Errorf("can't get unmarshal external gateway info, %v", err) + } + log.DebugLog(log.DebugLevelMexos, "get router detail external gateway", "external gateway", externalGateway) + return externalGateway, nil +} + +// GetRouterDetailInterfaces gets the list of interfaces on the router. For example, each private +// subnet connected to the router will be listed here with own interface definition. +func GetRouterDetailInterfaces(mf *Manifest, rd *OSRouterDetail) ([]OSRouterInterface, error) { + if rd.InterfacesInfo == "" { + return nil, fmt.Errorf("missing interfaces info in router details") + } + interfaces := []OSRouterInterface{} + err := json.Unmarshal([]byte(rd.InterfacesInfo), &interfaces) + if err != nil { + return nil, fmt.Errorf("can't unmarshal router detail interfaces") + } + log.DebugLog(log.DebugLevelMexos, "get router detail interfaces", "interfaces", interfaces) + return interfaces, nil +} diff --git a/mexos/nginx.go b/mexos/nginx.go new file mode 100644 index 000000000..c5d36d60c --- /dev/null +++ b/mexos/nginx.go @@ -0,0 +1,83 @@ +package mexos + +import ( + "fmt" + "strings" + + "github.com/mobiledgex/edge-cloud/log" + "github.com/parnurzeal/gorequest" +) + +func AddNginxProxy(mf *Manifest, rootLBName, name, ipaddr string, ports []PortDetail) error { + log.DebugLog(log.DebugLevelMexos, "add nginx proxy", "name", name, "ports", ports) + + request := gorequest.New() + npURI := fmt.Sprintf("http://%s:%s/v1/nginx", rootLBName, mf.Values.Agent.Port) + pl, err := FormNginxProxyRequest(ports, ipaddr, name) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot form nginx proxy request") + return err + } + log.DebugLog(log.DebugLevelMexos, "nginx proxy add request post", "request", *pl) + resp, body, errs := request.Post(npURI).Set("Content-Type", "application/json").Send(pl).End() + if errs != nil { + return fmt.Errorf("error, can't request nginx proxy add, %v", errs) + } + if strings.Contains(body, "OK") { + log.DebugLog(log.DebugLevelMexos, "ok, nginx proxy add request post") + return nil + } + log.DebugLog(log.DebugLevelMexos, "warning, error while adding nginx proxy", "resp", resp, "body", body) + return fmt.Errorf("cannot add nginx proxy, resp %v", resp) +} + +func FormNginxProxyRequest(ports []PortDetail, ipaddr string, name string) (*string, error) { + portstrs := []string{} + for _, p := range ports { + switch p.MexProto { + case "LProtoHTTP": + portstrs = append(portstrs, + fmt.Sprintf(`{"mexproto":"%s", "external": "%d", "internal": "%d", "origin":"%s:%d", "path":"/%s"}`, + p.MexProto, p.PublicPort, p.InternalPort, ipaddr, p.InternalPort, p.PublicPath)) + case "LProtoTCP": + portstrs = append(portstrs, + fmt.Sprintf(`{"mexproto":"%s", "external": "%d", "origin": "%s:%d"}`, + p.MexProto, p.PublicPort, ipaddr, p.InternalPort)) + case "LProtoUDP": + portstrs = append(portstrs, + fmt.Sprintf(`{"mexproto":"%s", "external": "%d", "origin": "%s:%d"}`, + p.MexProto, p.PublicPort, ipaddr, p.InternalPort)) + default: + log.DebugLog(log.DebugLevelMexos, "invalid mexproto", "port", p) + } + } + portspec := "" + for i, ps := range portstrs { + if i == 0 { + portspec += ps + } else { + portspec += "," + ps + } + + } + pl := fmt.Sprintf(`{ "message":"add", "name": "%s" , "ports": %s }`, name, "["+portspec+"]") + return &pl, nil +} + +func DeleteNginxProxy(mf *Manifest, rootLBName, name string) error { + log.DebugLog(log.DebugLevelMexos, "delete nginx proxy", "name", name) + request := gorequest.New() + npURI := fmt.Sprintf("http://%s:%s/v1/nginx", rootLBName, mf.Values.Agent.Port) + pl := fmt.Sprintf(`{"message":"delete","name":"%s"}`, name) + log.DebugLog(log.DebugLevelMexos, "nginx proxy delete request post", "request", pl) + resp, body, errs := request.Post(npURI).Set("Content-Type", "application/json").Send(pl).End() + if errs != nil { + return fmt.Errorf("error, can't request nginx proxy delete, %v", errs) + } + if strings.Contains(body, "OK") { + log.DebugLog(log.DebugLevelMexos, "deleted nginx proxy OK") + return nil + } + log.DebugLog(log.DebugLevelMexos, "error while deleting nginx proxy", "resp", resp, "body", body) + return fmt.Errorf("cannot delete nginx proxy, resp %v", resp) +} diff --git a/mexos/openstack.go b/mexos/openstack.go new file mode 100644 index 000000000..1a57f6e92 --- /dev/null +++ b/mexos/openstack.go @@ -0,0 +1,171 @@ +package mexos + +type OSLimit struct { + Name string + Value int +} + +type OSServer struct { + Status, Name, Image, ID, Flavor, Networks string +} + +type OSServerOpt struct { + AvailabilityZone string //XXX not used yet + Name, Image, Flavor string + UserData string + NetIDs []string + Properties []string +} + +type OSServerDetail struct { + TaskState string `json:"OS-EXT-STS:task_state"` + Addresses string `json:"addresses"` + Image string `json:"image"` + VMState string `json:"OS-EXT-STS:vm_state"` + LaunchedAt string `json:"OS-SRV-USG:launched_at"` + Flavor string `json:"flavor"` + ID string `json:"id"` + SecurityGroups string `json:"security_groups"` + VolumesAttached string `json:"volumes_attached"` + UserID string `json:"user_id"` + DiskConfig string `json:"OS-DCF:diskConfig"` + AccessIPv4 string `json:"accessIPv4"` + AccessIPv6 string `json:"accessIPv6"` + Progress int `json:"progress"` + PowerState string `json:"OS-EXT-STS:power_state"` + ProjectID string `json:"project_id"` + ConfigDrive string `json:"config_drive"` + Status string `json:"status"` + Updated string `json:"updated"` + HostID string `json:"hostId"` + TerminatedAt string `json:"OS-SRV-USG:terminated_at"` + KeyName string `json:"key_name"` + AvailabilityZone string `json:"OS-EXT-AZ:availability_zone"` + Name string `json:"name"` + Created string `json:"created"` + Properties string `json:"properties"` +} + +type OSImage struct { + Status, ID, Name string +} + +type OSNetwork struct { + Subnets, ID, Name string +} + +type OSNetworkDetail struct { + ID string `json:"id"` + Name string `json:"name"` + ProviderPhysicalNetwork string `json:"provider:physical_network"` + IPv6AddressScope string `json:"ipv6_address_scope"` + DNSDomain string `json:"dns_domain"` + IsVLANTransparent string `json:"is_vlan_transparent"` + ProviderNetworkType string `json:"provider:network_type"` + External string `json:"router:external"` + AvailabilityZoneHints string `json:"availability_zone_hints"` + AvailabilityZones string `json:"availability_zones"` + Segments string `json:"segments"` + IPv4AddressScope string `json:"ipv4_address_scope"` + ProjectID string `json:"project_id"` + Status string `json:"status"` + Subnets string `json:"subnets"` + Description string `json:"description"` + Tags string `json:"tags"` + UpdatedAt string `json:"updated_at"` + ProviderSegmentationID string `json:"provider:segmentation_id"` + QOSPolicyID string `json:"qos_policy_id"` + AdminStateUp string `json:"admin_state_up"` + CreatedAt string `json:"created_at"` + RevisionNumber int `json:"revision_number"` + MTU int `json:"mtu"` + PortSecurityEnabled bool `json:"port_security_enabled"` + Shared bool `json:"shared"` + IsDefault bool `json:"is_default"` +} + +type OSFlavor struct { + Name, ID string + RAM, Ephemeral, VCPUs, Disk int +} + +type OSSubnet struct { + Name, ID, Network, Subnet string +} + +type OSSubnetDetail struct { + ID string `json:"id"` + Name string `json:"name"` + ServiceTypes string `json:"service_types"` + Description string `json:"description"` + EnableDHCP bool `json:"enable_dhcp"` + SegmentID string `json:"segment_id"` + NetworkID string `json:"network_id"` + CreatedAt string `json:"created_at"` + Tags string `json:"tags"` + DNSNameServers string `json:"dns_nameservers"` + UpdatedAt string `json:"updated_at"` + IPv6RAMode string `json:"ipv6_ra_mode"` + AllocationPools string `json:"allocation_pools"` + GatewayIP string `json:"gateway_ip"` + RevisionNumber int `json:"revision_number"` + IPv6AddressMode string `json:"ipv6_address_mode"` + IPVersion int `json:"ip_version"` + HostRoutes string `json:"host_routes"` + CIDR string `json:"cidr"` + ProjectID string `json:"project_id"` + SubnetPoolID string `json:"subnetpool_id"` +} + +type OSRouter struct { + Name, ID, Status, State, Project string + HA, Distributed bool +} + +type OSRouterDetail struct { + ID string `json:"id"` + Name string `json:"name"` + ExternalGatewayInfo string `json:"external_gateway_info"` + Status string `json:"status"` + AvailabilityZoneHints string `json:"availability_zone_hints"` + AvailabilityZones string `json:"availability_zones"` + Description string `json:"description"` + AdminStateUp string `json:"admin_state_up"` + CreatedAt string `json:"created_at"` + Tags string `json:"tags"` + UpdatedAt string `json:"updated_at"` + InterfacesInfo string `json:"interfaces_info"` + ProjectID string `json:"project_id"` + FlavorID string `json:"flavor_id"` + Routes string `json:"routes"` + Distributed bool `json:"distributed"` + HA bool `json:"ha"` + RevisionNumber int `json:"revision_number"` +} + +type OSExternalGateway struct { + NetworkID string `json:"network_id"` //subnet of external net + EnableSNAT bool `json:"enable_snat"` + ExternalFixedIPs []OSExternalFixedIP `json:"external_fixed_ips"` //gateway between extnet and privnet +} + +type OSExternalFixedIP struct { + SubnetID string `json:"subnet_id"` + IPAddress string `json:"ip_address"` +} + +type OSRouterInterface struct { + SubnetID string `json:"subnet_id"` //attached privnet + IPAddress string `json:"ip_address"` //router for the privnet side on the subnet CIDR, usually X.X.X.1 but should really confirm by reading this + PortID string `json:"port_id"` +} + +type NeutronErrorDetail struct { + Message string `json:"message"` + Type string `json:"type"` + Detail string `json:"detail"` +} + +type NeutronErrorType struct { + NeutronError NeutronErrorDetail +} diff --git a/mexos/oscli.go b/mexos/oscli.go new file mode 100644 index 000000000..99daaf667 --- /dev/null +++ b/mexos/oscli.go @@ -0,0 +1,448 @@ +package mexos + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud/log" +) + +//ListServers returns list of servers, KVM instances, running on the system +func ListServers(mf *Manifest) ([]OSServer, error) { + out, err := sh.Command("openstack", "server", "list", "-f", "json").Output() + if err != nil { + err = fmt.Errorf("cannot get server list, %v", err) + return nil, err + } + var servers []OSServer + err = json.Unmarshal(out, &servers) + if err != nil { + err = fmt.Errorf("cannot unmarshal, %v", err) + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "list servers", "servers", servers) + return servers, nil +} + +//ListImages lists avilable images in glance +func ListImages(mf *Manifest) ([]OSImage, error) { + out, err := sh.Command("openstack", "image", "list", "-f", "json").Output() + if err != nil { + err = fmt.Errorf("cannot get image list, %v", err) + return nil, err + } + var images []OSImage + err = json.Unmarshal(out, &images) + if err != nil { + err = fmt.Errorf("cannot unmarshal, %v", err) + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "list images", "images", images) + return images, nil +} + +//ListNetworks lists networks known to the platform. Some created by the operator, some by users. +func ListNetworks(mf *Manifest) ([]OSNetwork, error) { + out, err := sh.Command("openstack", "network", "list", "-f", "json").Output() + if err != nil { + err = fmt.Errorf("cannot get network list, %v", err) + return nil, err + } + var networks []OSNetwork + err = json.Unmarshal(out, &networks) + if err != nil { + err = fmt.Errorf("cannot unmarshal, %v", err) + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "list networks", "networks", networks) + return networks, nil +} + +//ListFlavors lists flavors known to the platform. These vary. On Buckhorn cloudlet the are m4. prefixed. +func ListFlavors(mf *Manifest) ([]OSFlavor, error) { + out, err := sh.Command("openstack", "flavor", "list", "-f", "json").Output() + if err != nil { + err = fmt.Errorf("cannot get flavor list, %v", err) + return nil, err + } + var flavors []OSFlavor + err = json.Unmarshal(out, &flavors) + if err != nil { + err = fmt.Errorf("cannot unmarshal, %v", err) + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "list flavors", flavors) + return flavors, nil +} + +//CreateServer instantiates a new server instance, which is a KVM instance based on a qcow2 image from glance +func CreateServer(mf *Manifest, opts *OSServerOpt) error { + args := []string{ + "server", "create", + "--config-drive", "true", //XXX always + "--image", opts.Image, "--flavor", opts.Flavor, + } + if opts.UserData != "" { + args = append(args, "--user-data", opts.UserData) + } + for _, p := range opts.Properties { + args = append(args, "--property", p) + // `p` should be like: "key=value" + } + for _, n := range opts.NetIDs { + args = append(args, "--nic", "net-id="+n) + // `n` should be like: "public,v4-fixed-ip=172.24.4.201" + } + args = append(args, opts.Name) + //TODO additional args + iargs := make([]interface{}, len(args)) + for i, v := range args { + iargs[i] = v + } + log.DebugLog(log.DebugLevelMexos, "openstack create server", "opts", opts, "iargs", iargs) + out, err := sh.Command("openstack", iargs...).Output() + if err != nil { + err = fmt.Errorf("cannot create server, %v, '%s'", err, out) + return err + } + return nil +} + +// GetServerDetails returns details of the KVM instance +func GetServerDetails(mf *Manifest, name string) (*OSServerDetail, error) { + active := false + srvDetail := &OSServerDetail{} + for i := 0; i < 10; i++ { + out, err := sh.Command("openstack", "server", "show", "-f", "json", name).Output() + if err != nil { + err = fmt.Errorf("can't show server %s, %s, %v", name, out, err) + return nil, err + } + //fmt.Printf("%s\n", out) + err = json.Unmarshal(out, srvDetail) + if err != nil { + err = fmt.Errorf("cannot unmarshal while getting server detail, %v", err) + return nil, err + } + if srvDetail.Status == "ACTIVE" { + active = true + break + } + log.DebugLog(log.DebugLevelMexos, "wait for server to become ACTIVE", "server detail", srvDetail) + time.Sleep(30 * time.Second) + } + if !active { + return nil, fmt.Errorf("while getting server detail, waited but server %s is too slow getting to active state", name) + } + log.DebugLog(log.DebugLevelMexos, "server detail", "server detail", srvDetail) + return srvDetail, nil +} + +//DeleteServer destroys a KVM instance +// sometimes it is not possible to destroy. Like most things in Openstack, try again. +func DeleteServer(mf *Manifest, id string) error { + log.DebugLog(log.DebugLevelMexos, "deleting server", "id", id) + out, err := sh.Command("openstack", "server", "delete", id).Output() + if err != nil { + err = fmt.Errorf("can't delete server %s, %s, %v", id, out, err) + return err + } + return nil +} + +// CreateNetwork creates a network with a name. +func CreateNetwork(mf *Manifest, name string) error { + log.DebugLog(log.DebugLevelMexos, "creating network", "network", name) + out, err := sh.Command("openstack", "network", "create", name).Output() + if err != nil { + err = fmt.Errorf("can't create network %s, %s, %v", name, out, err) + return err + } + return nil +} + +//DeleteNetwork destroys a named network +// Sometimes it will fail. Openstack will refuse if there are resources attached. +func DeleteNetwork(mf *Manifest, name string) error { + log.DebugLog(log.DebugLevelMexos, "deleting network", "network", name) + out, err := sh.Command("openstack", "network", "delete", name).Output() + if err != nil { + err = fmt.Errorf("can't delete network %s, %s, %v", name, out, err) + return err + } + return nil +} + +//CreateSubnet creates a subnet within a network. A subnet is assigned ranges. Optionally DHCP can be enabled. +func CreateSubnet(mf *Manifest, netRange, networkName, gatewayAddr, subnetName string, dhcpEnable bool) error { + var dhcpFlag string + if dhcpEnable { + dhcpFlag = "--dhcp" + } else { + dhcpFlag = "--no-dhcp" + } + out, err := sh.Command("openstack", "subnet", "create", + "--subnet-range", netRange, // e.g. 10.101.101.0/24 + "--network", networkName, // mex-k8s-net-1 + dhcpFlag, + "--gateway", gatewayAddr, // e.g. 10.101.101.1 + subnetName).CombinedOutput() // e.g. mex-k8s-subnet-1 + if err != nil { + nerr := &NeutronErrorType{} + if ix := strings.Index(string(out), `{"NeutronError":`); ix > 0 { + neutronErr := out[ix:] + if jerr := json.Unmarshal(neutronErr, nerr); jerr != nil { + err = fmt.Errorf("can't create subnet %s, %s, %v, error while parsing neutron error, %v", subnetName, out, err, jerr) + return err + } + if strings.Index(nerr.NeutronError.Message, "overlap") > 0 { + sd, serr := GetSubnetDetail(mf, subnetName) + if serr != nil { + return fmt.Errorf("cannot get subnet detail for %s, while fixing overlap error, %v", subnetName, serr) + } + log.DebugLog(log.DebugLevelMexos, "create subnet, existing subnet detail", "subnet detail", sd) + + //XXX do more validation + + log.DebugLog(log.DebugLevelMexos, "create subnet, reusing existing subnet", "result", out, "error", err) + return nil + } + } + err = fmt.Errorf("can't create subnet %s, %s, %v", subnetName, out, err) + return err + } + return nil +} + +//DeleteSubnet deletes the subnet. If this fails, remove any attached resources, like router, and try again. +func DeleteSubnet(mf *Manifest, subnetName string) error { + log.DebugLog(log.DebugLevelMexos, "deleting subnet", "name", subnetName) + out, err := sh.Command("openstack", "subnet", "delete", subnetName).Output() + if err != nil { + err = fmt.Errorf("can't delete subnet %s, %s, %v", subnetName, out, err) + return err + } + return nil +} + +//CreateRouter creates new router. A router can be attached to network and subnets. +func CreateRouter(mf *Manifest, routerName string) error { + log.DebugLog(log.DebugLevelMexos, "creating router", "name", routerName) + out, err := sh.Command("openstack", "router", "create", routerName).Output() + if err != nil { + err = fmt.Errorf("can't create router %s, %s, %v", routerName, out, err) + return err + } + return nil +} + +//DeleteRouter removes the named router. The router needs to not be in use at the time of deletion. +func DeleteRouter(mf *Manifest, routerName string) error { + log.DebugLog(log.DebugLevelMexos, "deleting router", "name", routerName) + out, err := sh.Command("openstack", "router", "delete", routerName).Output() + if err != nil { + err = fmt.Errorf("can't delete router %s, %s, %v", routerName, out, err) + return err + } + return nil +} + +//SetRouter assigns the router to a particular network. The network needs to be attached to +// a real external network. This is intended only for routing to external network for now. No internal routers. +// Sometimes, oftentimes, it will fail if the network is not external. +func SetRouter(mf *Manifest, routerName, networkName string) error { + log.DebugLog(log.DebugLevelMexos, "setting router to network", "router", routerName, "network", networkName) + out, err := sh.Command("openstack", "router", "set", routerName, "--external-gateway", networkName).Output() + if err != nil { + err = fmt.Errorf("can't set router %s to %s, %s, %v", routerName, networkName, out, err) + return err + } + return nil +} + +//AddRouterSubnet will connect subnet to another network, possibly external, via a router +func AddRouterSubnet(mf *Manifest, routerName, subnetName string) error { + log.DebugLog(log.DebugLevelMexos, "adding router to subnet", "router", routerName, "network", subnetName) + out, err := sh.Command("openstack", "router", "add", "subnet", routerName, subnetName).Output() + if err != nil { + err = fmt.Errorf("can't add router %s to subnet %s, %s, %v", routerName, subnetName, out, err) + return err + } + return nil +} + +//RemoveRouterSubnet is useful to remove the router from the subnet before deletion. Otherwise subnet cannot +// be deleted. +func RemoveRouterSubnet(mf *Manifest, routerName, subnetName string) error { + log.DebugLog(log.DebugLevelMexos, "removing subnet from router", "router", routerName, "subnet", subnetName) + out, err := sh.Command("openstack", "router", "remove", "subnet", routerName, subnetName).Output() + if err != nil { + err = fmt.Errorf("can't remove router %s from subnet %s, %s, %v", routerName, subnetName, out, err) + return err + } + return nil +} + +//ListSubnets returns a list of subnets available +func ListSubnets(mf *Manifest, netName string) ([]OSSubnet, error) { + var err error + var out []byte + + if netName != "" { + out, err = sh.Command("openstack", "subnet", "list", "--network", netName, "-f", "json").Output() + } else { + out, err = sh.Command("openstack", "subnet", "list", "-f", "json").Output() + } + if err != nil { + err = fmt.Errorf("can't get a list of subnets, %v", err) + return nil, err + } + subnets := []OSSubnet{} + err = json.Unmarshal(out, &subnets) + if err != nil { + err = fmt.Errorf("can't unmarshal subnets, %v", err) + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "list subnets", "subnets", subnets) + return subnets, nil +} + +//ListRouters returns a list of routers available +func ListRouters(mf *Manifest) ([]OSRouter, error) { + out, err := sh.Command("openstack", "router", "list", "-f", "json").Output() + if err != nil { + err = fmt.Errorf("can't get a list of routers, %s, %v", out, err) + return nil, err + } + routers := []OSRouter{} + err = json.Unmarshal(out, &routers) + if err != nil { + err = fmt.Errorf("can't unmarshal routers, %v", err) + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "list routers", "routers", routers) + return routers, nil +} + +//GetRouterDetail returns details per router +func GetRouterDetail(mf *Manifest, routerName string) (*OSRouterDetail, error) { + out, err := sh.Command("openstack", "router", "show", "-f", "json", routerName).Output() + if err != nil { + err = fmt.Errorf("can't get router details for %s, %s, %v", routerName, out, err) + return nil, err + } + routerDetail := &OSRouterDetail{} + err = json.Unmarshal(out, routerDetail) + if err != nil { + err = fmt.Errorf("can't unmarshal router detail, %v", err) + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "router detail", "router detail", routerDetail) + return routerDetail, nil +} + +//CreateServerImage snapshots running service into a qcow2 image +func CreateServerImage(mf *Manifest, serverName, imageName string) error { + log.DebugLog(log.DebugLevelMexos, "creating image snapshot from server", "server", serverName, "image", imageName) + out, err := sh.Command("openstack", "server", "image", "create", serverName, "--name", imageName).Output() + if err != nil { + err = fmt.Errorf("can't create image from %s into %s, %s, %v", serverName, imageName, out, err) + return err + } + return nil +} + +//CreateImage puts images into glance +func CreateImage(mf *Manifest, imageName, qcowFile string) error { + log.DebugLog(log.DebugLevelMexos, "creating image in glance", "image", imageName, "qcow", qcowFile) + out, err := sh.Command("openstack", "image", "create", + imageName, + "--disk-format", "qcow2", + "--container-format", "bare", + "--file", qcowFile).Output() + if err != nil { + err = fmt.Errorf("can't create image in glace, %s, %s, %s, %v", imageName, qcowFile, out, err) + return err + } + return nil +} + +//SaveImage takes the image name available in glance, as a result of for example the above create image. +// It will then save that into a local file. The image transfer happens from glance into your own laptop +// or whatever. +// This can take a while, transferring all the data. +func SaveImage(mf *Manifest, saveName, imageName string) error { + log.DebugLog(log.DebugLevelMexos, "saving image", "save name", saveName, "image name", imageName) + out, err := sh.Command("openstack", "image", "save", "--file", saveName, imageName).Output() + if err != nil { + err = fmt.Errorf("can't save image from %s to file %s, %s, %v", imageName, saveName, out, err) + return err + } + return nil +} + +//DeleteImage deletes the named image from glance. Sometimes backing store is still busy and +// will refuse to honor the request. Like most things in Openstack, wait for a while and try +// again. +func DeleteImage(mf *Manifest, imageName string) error { + log.DebugLog(log.DebugLevelMexos, "deleting image", "name", imageName) + out, err := sh.Command("openstack", "image", "delete", imageName).Output() + if err != nil { + err = fmt.Errorf("can't delete image %s, %s, %v", imageName, out, err) + return err + } + return nil +} + +//GetSubnetDetail returns details for the subnet. This is useful when getting router/gateway +// IP for a given subnet. The gateway info is used for creating a server. +// Also useful in general, like other `detail` functions, to get the ID map for the name of subnet. +func GetSubnetDetail(mf *Manifest, subnetName string) (*OSSubnetDetail, error) { + out, err := sh.Command("openstack", "subnet", "show", "-f", "json", subnetName).Output() + if err != nil { + err = fmt.Errorf("can't get subnet details for %s, %s, %v", subnetName, out, err) + return nil, err + } + subnetDetail := &OSSubnetDetail{} + err = json.Unmarshal(out, subnetDetail) + if err != nil { + return nil, fmt.Errorf("can't unmarshal subnet detail, %v", err) + } + log.DebugLog(log.DebugLevelMexos, "get subnet detail", "subnet detail", subnetDetail) + return subnetDetail, nil +} + +//GetNetworkDetail returns details about a network. It is used, for example, by GetExternalGateway. +func GetNetworkDetail(mf *Manifest, networkName string) (*OSNetworkDetail, error) { + out, err := sh.Command("openstack", "network", "show", "-f", "json", networkName).Output() + if err != nil { + err = fmt.Errorf("can't get details for network %s, %s, %v", networkName, out, err) + return nil, err + } + networkDetail := &OSNetworkDetail{} + err = json.Unmarshal(out, networkDetail) + if err != nil { + return nil, fmt.Errorf("can't unmarshal network detail, %v", err) + } + log.DebugLog(log.DebugLevelMexos, "get network detail", "network detail", networkDetail) + return networkDetail, nil +} + +//SetServerProperty sets properties for the server +func SetServerProperty(mf *Manifest, name, property string) error { + if name == "" { + return fmt.Errorf("empty name") + } + if property == "" { + return fmt.Errorf("empty property") + } + out, err := sh.Command("openstack", "server", "set", "--property", property, name).Output() + if err != nil { + return fmt.Errorf("can't set property %s on server %s, %s, %v", property, name, out, err) + } + log.DebugLog(log.DebugLevelMexos, "set server property", "name", name, "property", property) + return nil +} diff --git a/mexos/platform.go b/mexos/platform.go new file mode 100644 index 000000000..8c9f62b75 --- /dev/null +++ b/mexos/platform.go @@ -0,0 +1,27 @@ +package mexos + +import ( + "fmt" + + "github.com/mobiledgex/edge-cloud/log" +) + +func setPlatConfManifest(mf *Manifest) error { + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("cannot set platform config manifest, rootLB is null") + } + setPlatConf(rootLB, mf) + return nil +} + +func setPlatConf(rootLB *MEXRootLB, mf *Manifest) { + log.DebugLog(log.DebugLevelMexos, "rootlb platconf set") + if rootLB == nil { + log.DebugLog(log.DebugLevelMexos, "cannot set platconf, rootLB is null") + } + rootLB.PlatConf = mf +} diff --git a/mexos/ports.go b/mexos/ports.go new file mode 100644 index 000000000..3d83d393c --- /dev/null +++ b/mexos/ports.go @@ -0,0 +1,36 @@ +package mexos + +import ( + "fmt" + "strings" + + dme "github.com/mobiledgex/edge-cloud/d-match-engine/dme-proto" + "github.com/mobiledgex/edge-cloud/edgeproto" +) + +func addPorts(mf *Manifest, appInst *edgeproto.AppInst) error { + for ii, _ := range appInst.MappedPorts { + port := &appInst.MappedPorts[ii] + if mf.Spec.Ports == nil { + mf.Spec.Ports = make([]PortDetail, 0) + } + mexproto, ok := dme.LProto_name[int32(port.Proto)] + if !ok { + return fmt.Errorf("invalid LProto %d", port.Proto) + } + proto := "UDP" + if port.Proto != dme.LProto_LProtoUDP { + proto = "TCP" + } + p := PortDetail{ + Name: fmt.Sprintf("%s%d", strings.ToLower(mexproto), port.InternalPort), + MexProto: mexproto, + Proto: proto, + InternalPort: int(port.InternalPort), + PublicPort: int(port.PublicPort), + PublicPath: port.PublicPath, + } + mf.Spec.Ports = append(mf.Spec.Ports, p) + } + return nil +} diff --git a/mexos/revproxy.go b/mexos/revproxy.go new file mode 100644 index 000000000..e0ff51a81 --- /dev/null +++ b/mexos/revproxy.go @@ -0,0 +1,41 @@ +package mexos + +import ( + "fmt" + "strings" + + "github.com/mobiledgex/edge-cloud/log" + "github.com/parnurzeal/gorequest" +) + +//AddPathReverseProxy adds a new route to origin on the reverse proxy +func AddPathReverseProxy(mf *Manifest, rootLBName, path, origin string) []error { + log.DebugLog(log.DebugLevelMexos, "add path to reverse proxy", "rootlbname", rootLBName, "path", path, "origin", origin) + if path == "" { + return []error{fmt.Errorf("empty path")} + } + if origin == "" { + return []error{fmt.Errorf("empty origin")} + } + request := gorequest.New() + maURI := fmt.Sprintf("http://%s:%s/v1/proxy", rootLBName, mf.Values.Agent.Port) + // The L7 reverse proxy terminates TLS at the RootLB and uses path routing to get to the service at a IP:port + pl := fmt.Sprintf(`{ "message": "add", "proxies": [ { "path": "/%s/*catchall", "origin": "%s" } ] }`, path, origin) + resp, body, errs := request.Post(maURI).Set("Content-Type", "application/json").Send(pl).End() + if errs != nil { + return errs + } + if strings.Contains(body, "OK") { + log.DebugLog(log.DebugLevelMexos, "added path to revproxy") + return nil + } + errs = append(errs, fmt.Errorf("resp %v, body %s", resp, body)) + return errs +} + +//DeletePathReverseProxy Deletes a new route to origin on the reverse proxy +func DeletePathReverseProxy(rootLBName, path, origin string) []error { + log.DebugLog(log.DebugLevelMexos, "delete path reverse proxy", "path", path, "origin", origin) + //TODO + return nil +} diff --git a/mexos/rootlb.go b/mexos/rootlb.go new file mode 100644 index 000000000..8f8ec2514 --- /dev/null +++ b/mexos/rootlb.go @@ -0,0 +1,190 @@ +package mexos + +import ( + "fmt" + "strings" + "time" + + "github.com/mobiledgex/edge-cloud/log" +) + +//MEXRootLB has rootLB data +type MEXRootLB struct { + Name string + PlatConf *Manifest +} + +var MEXRootLBMap = make(map[string]*MEXRootLB) + +//NewRootLB gets a new rootLB instance +func NewRootLB(rootLBName string) (*MEXRootLB, error) { + log.DebugLog(log.DebugLevelMexos, "getting new rootLB", "rootLBName", rootLBName) + if _, ok := MEXRootLBMap[rootLBName]; ok { + return nil, fmt.Errorf("rootlb %s already exists", rootLBName) + } + newRootLB := &MEXRootLB{Name: rootLBName} + MEXRootLBMap[rootLBName] = newRootLB + return newRootLB, nil +} + +//NewRootLBManifest creates rootLB instance and sets Platform Config with manifest +func NewRootLBManifest(mf *Manifest) (*MEXRootLB, error) { + log.DebugLog(log.DebugLevelMexos, "getting new rootLB with manifest", "name", mf.Spec.RootLB) + rootLB, err := NewRootLB(mf.Spec.RootLB) + if err != nil { + return nil, err + } + setPlatConf(rootLB, mf) + if rootLB == nil { + log.DebugLog(log.DebugLevelMexos, "error, newrootlbmanifest, rootLB is null") + } + return rootLB, nil +} + +//DeleteRootLB to be called by code that called NewRootLB +func DeleteRootLB(rootLBName string) { + delete(MEXRootLBMap, rootLBName) //no mutex because caller should be serializing New/Delete in a control loop +} + +func getRootLB(name string) (*MEXRootLB, error) { + rootLB, ok := MEXRootLBMap[name] + if !ok { + return nil, fmt.Errorf("can't find rootlb %s", name) + } + if rootLB == nil { + log.DebugLog(log.DebugLevelMexos, "getrootlb, rootLB is null") + } + return rootLB, nil +} + +var rootLBPorts = []int{ + 18889, //mexosagent HTTP server + 18888, //mexosagent GRPC server + 443, //mexosagent reverse proxy HTTPS + 8001, //kubectl proxy + 6443, //kubernetes control + 8000, //mex k8s join token server +} + +//TODO more than one kubectl proxy, one per hosted cluster + +//EnableRootLB creates a seed presence node in cloudlet that also becomes first Agent node. +// It also sets up first basic network router and subnet, ready for running first MEX agent. +func EnableRootLB(mf *Manifest, rootLB *MEXRootLB) error { + log.DebugLog(log.DebugLevelMexos, "enable rootlb", "name", rootLB.Name) + if rootLB == nil { + return fmt.Errorf("cannot enable rootLB, rootLB is null") + } + if mf.Values.Network.External == "" { + return fmt.Errorf("enable rootlb, missing external network in manifest") + } + err := PrepNetwork(mf) + if err != nil { + return err + } + sl, err := ListServers(mf) + if err != nil { + return err + } + if mf.Metadata.DNSZone == "" { + return fmt.Errorf("missing dns zone in manifest, metadata %v", mf.Metadata) + } + found := 0 + for _, s := range sl { + if s.Name == rootLB.Name { + log.DebugLog(log.DebugLevelMexos, "found existing rootlb", "server", s) + found++ + } + } + if found == 0 { + log.DebugLog(log.DebugLevelMexos, "not found existing server", "name", rootLB.Name) + netspec := fmt.Sprintf("external-ip,%s", mf.Values.Network.External) + if strings.Contains(mf.Spec.Options, "dhcp") { + netspec = netspec + ",dhcp" + } + cf, err := GetClusterFlavor(mf.Spec.Flavor) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "invalid platform flavor, can't create rootLB") + return fmt.Errorf("cannot create rootLB invalid platform flavor %v", err) + } + log.DebugLog(log.DebugLevelMexos, "creating agent node kvm", "netspec", netspec) + err = CreateMEXKVM(mf, rootLB.Name, + "mex-agent-node", //important, don't change + netspec, + mf.Metadata.Tags, + mf.Metadata.Tenant, + 1, + cf.PlatformFlavor, + ) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error while creating mex kvm", "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "created kvm instance", "name", rootLB.Name) + + //rootLBIPaddr, ierr := GetServerIPAddr(mf, mf.Values.Network.External, rootLB.Name) + // if ierr != nil { + // log.DebugLog(log.DebugLevelMexos, "cannot get rootlb IP address", "error", ierr) + // return fmt.Errorf("created rootlb but cannot get rootlb IP") + // } + ruleName := GetMEXSecurityRule(mf) + //privateNetCIDR := strings.Replace(defaultPrivateNetRange, "X", "0", 1) + allowedClientCIDR := GetAllowedClientCIDR() + for _, p := range rootLBPorts { + // for _, cidr := range []string{rootLBIPaddr + "/32", privateNetCIDR, allowedClientCIDR} { + // go func(cidr string) { + // err := AddSecurityRuleCIDR(mf, cidr, "tcp", ruleName, p) + // if err != nil { + // log.DebugLog(log.DebugLevelMexos, "warning, error while adding security rule", "error", err, "cidr", cidr, "rulename", ruleName, "port", p) + // } + // }(cidr) + // } + if err := AddSecurityRuleCIDR(mf, allowedClientCIDR, "tcp", ruleName, p); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot add security rule", "error", err, "cidr", allowedClientCIDR, "port", p, "rule", ruleName) + } + } + //TODO: removal of security rules. Needs to be done for general resource per VM object. + // Add annotation to the running VM. When VM is removed, go through annotations + // and undo the resource allocations, like security rules, etc. + } else { + log.DebugLog(log.DebugLevelMexos, "re-using existing kvm instance", "name", rootLB.Name) + } + log.DebugLog(log.DebugLevelMexos, "done enabling rootlb", "name", rootLB.Name) + return nil +} + +//WaitForRootLB waits for the RootLB instance to be up and copies of SSH credentials for internal networks. +// Idempotent, but don't call all the time. +func WaitForRootLB(mf *Manifest, rootLB *MEXRootLB) error { + log.DebugLog(log.DebugLevelMexos, "wait for rootlb", "name", rootLB.Name) + if rootLB == nil { + return fmt.Errorf("cannot wait for lb, rootLB is null") + } + + if mf.Values.Network.External == "" { + return fmt.Errorf("waiting for lb, missing external network in manifest") + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return err + } + running := false + for i := 0; i < 10; i++ { + log.DebugLog(log.DebugLevelMexos, "waiting for rootlb...") + _, err := client.Output("sudo grep 'all done' /var/log/mobiledgex.log") //XXX beware of use of word done + if err == nil { + log.DebugLog(log.DebugLevelMexos, "rootlb is running", "name", rootLB.Name) + running = true + //if err := CopySSHCredential(mf, rootLB.Name, mf.Values.Network.External, "root"); err != nil { + // return fmt.Errorf("can't copy ssh credential to RootLB, %v", err) + //} + break + } + time.Sleep(30 * time.Second) + } + if !running { + return fmt.Errorf("while creating cluster, timeout waiting for RootLB") + } + log.DebugLog(log.DebugLevelMexos, "done waiting for rootlb", "name", rootLB.Name) + return nil +} diff --git a/mexos/securityrule.go b/mexos/securityrule.go new file mode 100644 index 000000000..17247cb48 --- /dev/null +++ b/mexos/securityrule.go @@ -0,0 +1,102 @@ +package mexos + +import ( + "encoding/json" + "fmt" + "strings" + + sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud/log" +) + +// TODO service to periodically clean up the leftover rules + +func AddProxySecurityRules(rootLB *MEXRootLB, mf *Manifest, masteraddr string) error { + // rootLBIPaddr, err := GetServerIPAddr(mf, mf.Values.Network.External, rootLB.Name) + // if err != nil { + // log.DebugLog(log.DebugLevelMexos, "cannot get rootlb IP address", "error", err) + // return fmt.Errorf("cannot deploy kubernetes app, cannot get rootlb IP") + // } + sr := GetMEXSecurityRule(mf) + allowedClientCIDR := GetAllowedClientCIDR() + for _, port := range mf.Spec.Ports { + for _, sec := range []struct { + addr string + port int + }{ + {allowedClientCIDR, port.PublicPort}, + {allowedClientCIDR, port.InternalPort}, + } { + // go func(addr string, port int, proto string) { + // err := AddSecurityRuleCIDR(mf, addr, strings.ToLower(proto), sr, port) + // if err != nil { + // log.DebugLog(log.DebugLevelMexos, "warning, error while adding security rule", "cidr", addr, "securityrule", sr, "port", port, "proto", proto) + // } + // }(sec.addr, sec.port, port.Proto) + if err := AddSecurityRuleCIDR(mf, sec.addr, strings.ToLower(port.Proto), sr, sec.port); err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, error while adding security rule", "addr", sec.addr, "port", sec.port, "proto", port.Proto) + } + } + } + if len(mf.Spec.Ports) > 0 { + if err := AddNginxProxy(mf, rootLB.Name, mf.Metadata.Name, masteraddr, mf.Spec.Ports); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot add nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports) + return err + } + } + log.DebugLog(log.DebugLevelMexos, "added nginx proxy", "name", mf.Metadata.Name, "ports", mf.Spec.Ports) + return nil +} + +func DeleteProxySecurityRules(rootLB *MEXRootLB, mf *Manifest, ipaddr string) error { + log.DebugLog(log.DebugLevelMexos, "delete spec ports", "ports", mf.Spec.Ports) + err := DeleteNginxProxy(mf, rootLB.Name, mf.Metadata.Name) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot delete nginx proxy", "name", mf.Metadata.Name, "rootlb", rootLB.Name, "error", err) + } + if err := DeleteSecurityRule(mf, ipaddr); err != nil { + return err + } + // TODO - implement the clean up of security rules + return nil +} + +func AddSecurityRuleCIDR(mf *Manifest, cidr string, proto string, name string, port int) error { + portStr := fmt.Sprintf("%d", port) + out, err := sh.Command("openstack", "security", "group", "rule", "create", "--remote-ip", cidr, "--proto", proto, "--dst-port", portStr, "--ingress", name).Output() + if err != nil { + return fmt.Errorf("can't add security group rule for port %d to %s,%s,%v", port, name, out, err) + } + return nil +} + +type SecurityRule struct { + IPRange string `json:"IP Range"` + PortRange string `json:"Port Range"` + SGID string `json:"Security Group"` + ID string `json:"ID"` + Proto string `json:"IP Protocol"` +} + +func DeleteSecurityRule(mf *Manifest, sip string) error { + sr := []SecurityRule{} + dat, err := sh.Command("openstack", "security", "group", "rule", "list", "-f", "json").Output() + if err != nil { + return fmt.Errorf("cannot get list of security group rules, %v", err) + } + if err := json.Unmarshal(dat, &sr); err != nil { + return fmt.Errorf("cannot unmarshal security group rule list, %v", err) + } + for _, s := range sr { + if strings.HasSuffix(s.IPRange, "/32") { + adr := strings.Replace(s.IPRange, "/32", "", -1) + if adr == sip { + _, err := sh.Command("openstack", "security", "group", "rule", "delete", s.ID).Output() + if err != nil { + log.DebugLog(log.DebugLevelMexos, "warning, cannot delete security rule", "id", s.ID, "error", err) + } + } + } + } + return nil +} diff --git a/mexos/ssh.go b/mexos/ssh.go new file mode 100644 index 000000000..db4716b5a --- /dev/null +++ b/mexos/ssh.go @@ -0,0 +1,79 @@ +package mexos + +import ( + "fmt" + + sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud/log" + ssh "github.com/nanobox-io/golang-ssh" +) + +var sshOpts = []string{"StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null", "LogLevel=ERROR"} +var sshUser = "ubuntu" + +//CopySSHCredential copies over the ssh credential for mex to LB +func CopySSHCredential(mf *Manifest, serverName, networkName, userName string) error { + //TODO multiple keys to be copied and added to authorized_keys if needed + log.DebugLog(log.DebugLevelMexos, "copying ssh credentials", "server", serverName, "network", networkName, "user", userName) + addr, err := GetServerIPAddr(mf, networkName, serverName) + if err != nil { + return err + } + kf := PrivateSSHKey() + out, err := sh.Command("scp", "-o", sshOpts[0], "-o", sshOpts[1], "-i", kf, kf, sshUser+"@"+addr+":").Output() + if err != nil { + return fmt.Errorf("can't copy %s to %s, %s, %v", kf, addr, out, err) + } + return nil +} + +//GetSSHClient returns ssh client handle for the server +func GetSSHClient(mf *Manifest, serverName, networkName, userName string) (ssh.Client, error) { + auth := ssh.Auth{Keys: []string{PrivateSSHKey()}} + addr, err := GetServerIPAddr(mf, networkName, serverName) + if err != nil { + return nil, err + } + client, err := ssh.NewNativeClient(userName, addr, "SSH-2.0-mobiledgex-ssh-client-1.0", 22, &auth, nil) + if err != nil { + return nil, fmt.Errorf("cannot get ssh client for server %s on network %s, %v", serverName, networkName, err) + } + //log.DebugLog(log.DebugLevelMexos, "got ssh client", "addr", addr, "key", auth) + return client, nil +} + +func GetSSHClientIP(mf *Manifest, ipaddr, userName string) (ssh.Client, error) { + auth := ssh.Auth{Keys: []string{PrivateSSHKey()}} + client, err := ssh.NewNativeClient(userName, ipaddr, "SSH-2.0-mobiledgex-ssh-client-1.0", 22, &auth, nil) + if err != nil { + return nil, fmt.Errorf("cannot get ssh client for ipaddr %s, %v", ipaddr, err) + } + return client, nil +} + +func SetupSSHUser(mf *Manifest, rootLB *MEXRootLB, user string) error { + log.DebugLog(log.DebugLevelMexos, "setting up ssh user", "user", user) + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, user) + if err != nil { + return err + } + // XXX cloud-init creates non root user but it does not populate all the needed files. + // packer will create images with correct things for root .ssh. It cannot provision + // them for the `ubuntu` user. It may not yet exist until cloud-init runs. So we fix it here. + for _, cmd := range []string{ + fmt.Sprintf("sudo cp /root/.ssh/config /home/%s/.ssh/", user), + fmt.Sprintf("sudo chown %s:%s /home/%s/.ssh/config", user, user, user), + fmt.Sprintf("sudo chmod 600 /home/%s/.ssh/config", user), + fmt.Sprintf("sudo cp /root/id_rsa_mex /home/%s/", user), + fmt.Sprintf("sudo chown %s:%s /home/%s/id_rsa_mex", user, user, user), + fmt.Sprintf("sudo chmod 600 /home/%s/id_rsa_mex", user), + } { + out, err := client.Output(cmd) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error setting up ssh user", + "user", user, "error", err, "out", out) + return err + } + } + return nil +} diff --git a/mexos/stats.go b/mexos/stats.go new file mode 100644 index 000000000..d01c4b0d5 --- /dev/null +++ b/mexos/stats.go @@ -0,0 +1,28 @@ +package mexos + +import ( + "encoding/json" + "fmt" + + sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud/log" +) + +//GetLimits is used to retrieve tenant level platform stats +func GetLimits(mf *Manifest) ([]OSLimit, error) { + log.DebugLog(log.DebugLevelMexos, "GetLimits", "mf value name", mf.Values.Name) + //err := sh.Command("openstack", "limits", "show", "--absolute", "-f", "json", sh.Dir("/tmp")).WriteStdout("os-out.txt") + out, err := sh.Command("openstack", "limits", "show", "--absolute", "-f", "json", sh.Dir("/tmp")).Output() + if err != nil { + err = fmt.Errorf("cannot get limits from openstack, %v", err) + return nil, err + } + var limits []OSLimit + err = json.Unmarshal(out, &limits) + if err != nil { + err = fmt.Errorf("cannot unmarshal, %v", err) + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "get limits", "limits", limits) + return limits, nil +} diff --git a/mexos/svc.go b/mexos/svc.go new file mode 100644 index 000000000..9c66bee14 --- /dev/null +++ b/mexos/svc.go @@ -0,0 +1,49 @@ +package mexos + +type ksaPort struct { + Name string `json:"name"` + Protocol string `json:"protocol"` + Port int `json:"port"` + NodePort int `json:"nodePort"` + TargetPort int `json:"targetPort"` +} + +type ksaSpec struct { + Ports []ksaPort `json:"ports"` +} + +type kubernetesServiceAbbrev struct { + Spec ksaSpec `json:"spec"` +} + +type ingressItem struct { + IP string `json:"ip"` +} + +type loadBalancerItem struct { + Ingresses []ingressItem `json:"ingress"` +} + +type statusItem struct { + LoadBalancer loadBalancerItem `json:"loadBalancer"` +} + +type metadataItem struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + CreationTimestamp string `json:"creationTimestamp"` + ResourceVersion string `json:"resourceVersion"` + UID string `json:"uid"` +} + +type svcItem struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata metadataItem `json:"metadata"` + Spec interface{} `json:"spec"` + Status statusItem `json:"status"` +} + +type svcItems struct { + Items []svcItem `json:"items"` +} diff --git a/mexos/swarm.go b/mexos/swarm.go new file mode 100644 index 000000000..64eb36603 --- /dev/null +++ b/mexos/swarm.go @@ -0,0 +1,246 @@ +package mexos + +import ( + "fmt" + "strconv" + "strings" + + "github.com/ghodss/yaml" + "github.com/mobiledgex/edge-cloud/log" +) + +func CreateDockerSwarm(mf *Manifest, rootLB *MEXRootLB) error { + //TODO independent swarm cluster without k8s + log.DebugLog(log.DebugLevelMexos, "creating docker swarm", "name", mf.Metadata.Swarm) + name, err := FindClusterWithKey(mf, mf.Spec.Key) + if err != nil { + return err + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return fmt.Errorf("can't get ssh client for docker swarm, %v", err) + } + masteraddr, err := FindNodeIP(mf, name) + if err != nil { + return err + } + cmd := fmt.Sprintf("ssh -o %s -o %s -o %s -i id_rsa_mex %s@%s docker swarm init --advertise-addr %s", sshOpts[0], sshOpts[1], sshOpts[2], sshUser, masteraddr, masteraddr) + log.DebugLog(log.DebugLevelMexos, "running docker swarm init", "cmd", cmd) + out, err := client.Output(cmd) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error, docker swarm init", "out", out, "err", err) + return fmt.Errorf("cannot docker swarm init, %v, %s", err, out) + } + cmd = fmt.Sprintf("ssh -o %s -o %s -o %s -i id_rsa_mex %s@%s docker swarm join-token worker -q", sshOpts[0], sshOpts[1], sshOpts[2], sshUser, masteraddr) + out, err = client.Output(cmd) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error, docker swarm join-token", "out", out, "err", err) + return fmt.Errorf("cannot docker swarm join-token worker, %v, %s", err, out) + } + token := strings.TrimSpace(out) + if len(token) < 1 { + return fmt.Errorf("docker join token too short") + } + knodes, err := GetKubernetesNodes(mf, rootLB) + if err != nil { + return err + } + nodesJoined := 0 + for _, n := range knodes { + if n.Role == "master" { + continue + } + if n.Addr == "" { + errmsg := fmt.Sprintf("missing address for kubernetes node, %v", n) + log.DebugLog(log.DebugLevelMexos, errmsg) + return fmt.Errorf(errmsg) + } + log.DebugLog(log.DebugLevelMexos, "docker worker node join", "node", n) + cmd = fmt.Sprintf("ssh -o %s -o %s -o %s -i id_rsa_mex %s@%s docker swarm join --token %s %s:2377", sshOpts[0], sshOpts[1], sshOpts[2], sshUser, n.Addr, token, masteraddr) + out, err = client.Output(cmd) + if err != nil { + errmsg := fmt.Sprintf("cannot docker swarm join, %v, %s, cmd %s", err, out, cmd) + log.DebugLog(log.DebugLevelMexos, errmsg) + return fmt.Errorf(errmsg) + } + nodesJoined++ + } + log.DebugLog(log.DebugLevelMexos, "ok, docker swarm nodes joined", "num worker nodes", nodesJoined) + return nil +} + +//TODO make it support full docker-compose file spec. + +type DockerService struct { + Image string `json:"image"` + Build string `json:"build"` + Ports []string `json:"ports"` +} + +type DockerCompose struct { + Version string `json:"version"` + Services map[string]DockerService `json:"services"` +} + +func CreateDockerSwarmAppManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "create docker-swarm app") + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("docker swarm app manifest, rootLB is null") + } + name, err := FindClusterWithKey(mf, mf.Spec.Key) + if err != nil { + return fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) + } + masteraddr, err := FindNodeIP(mf, name) + if err != nil { + return err + } + var cmd string + if mexEnv(mf, "MEX_DOCKER_REG_PASS") == "" { + return fmt.Errorf("empty docker registry password environment variable") + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return err + } + base := rootLB.PlatConf.Base + if base == "" { + log.DebugLog(log.DebugLevelMexos, "base is empty, using default") + base = GetDefaultRegistryBase(mf, base) + } + mani := mf.Config.ConfigDetail.Manifest + fn := fmt.Sprintf("%s/%s", base, mani) + res, err := GetURIFile(mf, fn) + if err != nil { + return fmt.Errorf("error getting docker compose manifest, %v", err) + } + dc := &DockerCompose{} + if err := yaml.Unmarshal(res, dc); err != nil { + return fmt.Errorf("cannot unmarshal docker compose file, %v", err) + } + dcfn := fmt.Sprintf("docker-compose-%s.yaml", mf.Metadata.Name) + log.DebugLog(log.DebugLevelMexos, "writing docker-compose file", "fn", dcfn) + cmd = fmt.Sprintf("cat <<'EOF'> %s \n%s\nEOF", dcfn, string(res)) + out, err := client.Output(cmd) + if err != nil { + return fmt.Errorf("error writing docker-compose yaml, %s, %v", out, err) + } + cmd = fmt.Sprintf("scp -i id_rsa_mex %s %s:", dcfn, masteraddr) + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("error copying docker-compose yaml, %s, %v", out, err) + } + log.DebugLog(log.DebugLevelMexos, "deploying docker stack", "name", mf.Metadata.Name) + cmd = fmt.Sprintf("ssh -i id_rsa_mex %s docker stack deploy --compose-file %s %s", masteraddr, dcfn, mf.Metadata.Name) + out, err = client.Output(cmd) + if err != nil { + return fmt.Errorf("error deploying docker-swarm app, %s, %s, %v", cmd, out, err) + } + if err := DockerComposePorts(mf, dc); err != nil { + return err + } + log.DebugLog(log.DebugLevelMexos, "adding proxy and security rules", "name", mf.Metadata.Name) + if err = AddProxySecurityRules(rootLB, mf, masteraddr); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "ok, docker stack deployed", "name", mf.Metadata.Name, "fn", dcfn, "ports", mf.Spec.Ports) + // TODO add custom DNS entries per app service endpoints + return nil +} + +func DockerComposePorts(mf *Manifest, dc *DockerCompose) error { + //TODO more complete syntax support for docker swarm ports + // https://docs.docker.com/compose/compose-file/#ports + mf.Spec.Ports = make([]PortDetail, 0) + for k, svc := range dc.Services { + if svc.Ports != nil { + for _, pp := range svc.Ports { + ps := strings.Split(pp, ":") + if len(ps) != 2 { + return fmt.Errorf("malformed port pair in docker swarm svc ports, %s", pp) + } + intp, err := strconv.Atoi(ps[0]) + if err != nil { + return fmt.Errorf("cannot convert internalport, %s", pp) + } + pubp, err := strconv.Atoi(ps[1]) + if err != nil { + return fmt.Errorf("cannot convert publicport, %s", pp) + } + pd := PortDetail{ + Name: fmt.Sprintf("%s%d", k, pubp), + MexProto: "LProtoTCP", //XXX + Proto: "TCP", //XXX + InternalPort: intp, + PublicPort: pubp, + } + mf.Spec.Ports = append(mf.Spec.Ports, pd) + } + } + } + return nil +} + +func DeleteDockerSwarmAppManifest(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "delete docker-swarm app") + rootLB, err := getRootLB(mf.Spec.RootLB) + if err != nil { + return err + } + if rootLB == nil { + return fmt.Errorf("docker swarm app manifest, rootLB is null") + } + name, err := FindClusterWithKey(mf, mf.Spec.Key) + if err != nil { + return fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err) + } + masteraddr, err := FindNodeIP(mf, name) + if err != nil { + return err + } + var cmd string + if mexEnv(mf, "MEX_DOCKER_REG_PASS") == "" { + return fmt.Errorf("empty docker registry password environment variable") + } + client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser) + if err != nil { + return err + } + base := rootLB.PlatConf.Base + if base == "" { + log.DebugLog(log.DebugLevelMexos, "base is empty, using default") + base = GetDefaultRegistryBase(mf, base) + } + mani := mf.Config.ConfigDetail.Manifest + fn := fmt.Sprintf("%s/%s", base, mani) + res, err := GetURIFile(mf, fn) + if err != nil { + return fmt.Errorf("error getting docker compose manifest, %v", err) + } + dc := &DockerCompose{} + if err := yaml.Unmarshal(res, dc); err != nil { + return fmt.Errorf("cannot unmarshal docker compose file, %v", err) + } + log.DebugLog(log.DebugLevelMexos, "removing docker stack", "name", mf.Metadata.Name) + cmd = fmt.Sprintf("ssh -i id_rsa_mex %s docker stack rm %s", masteraddr, mf.Metadata.Name) + out, err := client.Output(cmd) + if err != nil { + return fmt.Errorf("error removing docker-swarm app, %s, %s, %v", cmd, out, err) + } + if err := DockerComposePorts(mf, dc); err != nil { + return err + } + log.DebugLog(log.DebugLevelMexos, "removing proxy and security rules", "name", mf.Metadata.Name) + if err = DeleteProxySecurityRules(rootLB, mf, masteraddr); err != nil { + log.DebugLog(log.DebugLevelMexos, "cannot create security rules", "error", err) + return err + } + log.DebugLog(log.DebugLevelMexos, "ok, docker stack removed", "name", mf.Metadata.Name) + // TODO add custom DNS entries per app service endpoints + return nil +} diff --git a/mexos/templates.go b/mexos/templates.go new file mode 100644 index 000000000..5d4e18e43 --- /dev/null +++ b/mexos/templates.go @@ -0,0 +1,306 @@ +package mexos + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "text/template" + + "github.com/ghodss/yaml" + "github.com/mobiledgex/edge-cloud/cloudcommon" + "github.com/mobiledgex/edge-cloud/edgeproto" + "github.com/mobiledgex/edge-cloud/log" +) + +type templateFill struct { + Name, Kind, Flavor, Tags, Tenant, Region, Zone, DNSZone string + ImageFlavor, Location, RootLB, Resource, ResourceKind, ResourceGroup string + StorageSpec, NetworkScheme, MasterFlavor, Topology string + NodeFlavor, Operator, Key, Image, Options string + ImageType, AppURI, ProxyPath, AgentImage string + ExternalNetwork, Project string + ExternalRouter, Flags, IPAccess, Swarm string + NumMasters, NumNodes int + Config templateConfig + Command []string + SpecPorts []PortDetail +} + +type templateConfig struct { + Base, Overlay, Deployment, Resources, Manifest, Template string +} + +var yamlMEXCluster = `apiVersion: v1 +kind: {{.ResourceKind}} +resource: {{.Resource}} +metadata: + name: {{.Name}} + tags: {{.Tags}} + kind: {{.Kind}} + tenant: {{.Tenant}} + operator: {{.Operator}} + region: {{.Region}} + zone: {{.Zone}} + location: {{.Location}} + project: {{.Project}} + dnszone: {{.DNSZone}} + swarm: {{.Swarm}} + resourcegroup: {{.ResourceGroup}} +spec: + flags: {{.Flags}} + flavor: {{.Flavor}} + key: {{.Key}} + dockerregistry: registry.mobiledgex.net:5000 + rootlb: {{.RootLB}} + networkscheme: {{.NetworkScheme}} +` + +var yamlMEXFlavor = `apiVersion: v1 +kind: {{.ResourceKind}} +resource: {{.Resource}} +metadata: + name: {{.Name}} + tags: {{.Tags}} + kind: {{.Kind}} +spec: + flags: {{.Flags}} + flavor: {{.Name}} + flavordetail: + name: {{.Name}} + nodes: {{.NumNodes}} + masters: {{.NumMasters}} + networkscheme: {{.NetworkScheme}} + masterflavor: {{.MasterFlavor}} + nodeflavor: {{.NodeFlavor}} + storagescheme: {{.StorageSpec}} + topology: {{.Topology}} +` + +var yamlMEXPlatform = `apiVersion: v1 +kind: {{.ResourceKind}} +resource: {{.Resource}} +metadata: + kind: {{.Kind}} + name: {{.Name}} + tags: {{.Tags}} + tenant: {{.Tenant}} + region: {{.Region}} + zone: {{.Zone}} + location: {{.Location}} + openrc: ~/.mobiledgex/openrc + dnszone: {{.DNSZone}} + operator: {{.Operator}} +spec: + flags: {{.Flags}} + flavor: {{.Flavor}} + rootlb: {{.RootLB}} + key: {{.Key}} + dockerregistry: registry.mobiledgex.net:5000 + externalnetwork: {{.ExternalNetwork}} + networkscheme: {{.NetworkScheme}} + externalrouter: {{.ExternalRouter}} + options: {{.Options}} + agent: + image: {{.AgentImage}} + status: active +` + +var yamlMEXApp = `apiVersion: v1 +kind: {{.ResourceKind}} +resource: {{.Resource}} +metadata: + kind: {{.Kind}} + name: {{.Name}} + tags: {{.Tags}} + tenant: {{.Tenant}} + operator: {{.Operator}} + dnszone: {{.DNSZone}} +config: + kind: + source: + detail: + resources: "{{.Config.Resources}}" + deployment: {{.Config.Deployment}} + manifest: "{{.Config.Manifest}}" + template: {{.Config.Template}} + base: {{.Config.Base}} + overlay: {{.Config.Overlay}} +spec: + flags: {{.Flags}} + key: {{.Key}} + rootlb: {{.RootLB}} + image: {{.Image}} + imagetype: {{.ImageType}} + imageflavor: {{.ImageFlavor}} + proxypath: {{.ProxyPath}} + flavor: {{.Flavor}} + uri: {{.AppURI}} + ipaccess: {{.IPAccess}} + networkscheme: {{.NetworkScheme}} + ports: +{{- range .SpecPorts}} + - {{.Name}} + {{.MexProto}} + {{.Proto}} + {{.InternalPort}} + {{.PublicPort}} + {{.PublicPath}} +{{- end}} + command: +{{- range .Command}} + - {{.}} +{{- end}} +` + +func fillPlatformTemplateCloudletKey(rootLB *MEXRootLB, cloudletKeyStr string) (*Manifest, error) { + log.DebugLog(log.DebugLevelMexos, "fill template cloudletkeystr", "cloudletkeystr", cloudletKeyStr) + clk := edgeproto.CloudletKey{} + err := json.Unmarshal([]byte(cloudletKeyStr), &clk) + if err != nil { + return nil, fmt.Errorf("can't unmarshal json cloudletkey %s, %v", cloudletKeyStr, err) + } + log.DebugLog(log.DebugLevelMexos, "unmarshalled cloudletkeystr", "cloudletkey", clk) + if clk.Name == "" || clk.OperatorKey.Name == "" { + log.DebugLog(log.DebugLevelMexos, "will not fill template with invalid cloudletkeystr", "cloudletkeystr", cloudletKeyStr) + return nil, fmt.Errorf("invalid cloudletkeystr %s", cloudletKeyStr) + } + + log.DebugLog(log.DebugLevelMexos, "using external network", "extNet", GetMEXExternalNetwork(rootLB.PlatConf)) + + data := templateFill{ + ResourceKind: "platform", + Resource: NormalizeName(clk.OperatorKey.Name), + Name: clk.Name, + Tags: clk.Name + "-tag", + Key: clk.Name + "-" + NormalizeName(clk.OperatorKey.Name), + Flavor: "x1.medium", + Operator: NormalizeName(clk.OperatorKey.Name), + Location: "buckhorn", + Region: "eu-central-1", + Zone: "eu-central-1c", + RootLB: rootLB.Name, + AgentImage: "registry.mobiledgex.net:5000/mobiledgex/mexosagent", + Kind: "mex-platform", + ExternalNetwork: GetMEXExternalNetwork(rootLB.PlatConf), + NetworkScheme: "priv-subnet,mex-k8s-net-1,10.101.X.0/24", + DNSZone: "mobiledgex.net", + ExternalRouter: "mex-k8s-router-1", + Options: "dhcp", + } + mf, err := templateUnmarshal(&data, yamlMEXPlatform) + if err != nil { + return nil, err + } + return mf, nil +} + +func fillAppTemplate(rootLB *MEXRootLB, appInst *edgeproto.AppInst, app *edgeproto.App, clusterInst *edgeproto.ClusterInst) (*Manifest, error) { + var err error + var mf *Manifest + log.DebugLog(log.DebugLevelMexos, "fill app template", "appinst", appInst, "clusterInst", clusterInst) + imageType, ok := edgeproto.ImageType_name[int32(app.ImageType)] + if !ok { + return nil, fmt.Errorf("cannot find imagetype in map") + } + if clusterInst.Flavor.Name == "" { + return nil, fmt.Errorf("will not fill app template, invalid cluster flavor name") + } + if verr := ValidateClusterKind(clusterInst.Key.CloudletKey.OperatorKey.Name); verr != nil { + return nil, verr + } + if appInst.Key.AppKey.Name == "" { + return nil, fmt.Errorf("will not fill app template, invalid appkey name") + } + ipAccess, ok := edgeproto.IpAccess_name[int32(appInst.IpAccess)] + if !ok { + return nil, fmt.Errorf("cannot find ipaccess in map") + } + if len(appInst.Key.AppKey.Name) < 3 { + log.DebugLog(log.DebugLevelMexos, "warning, very short appkey name", "name", appInst.Key.AppKey.Name) + } + config, err := cloudcommon.ParseAppConfig(app.Config) + if err != nil { + return nil, fmt.Errorf("error parsing appinst config %s, %v", app.Config, err) + } + log.DebugLog(log.DebugLevelMexos, "appinst config", "config", config) + appDeploymentType := app.Deployment + if err != nil { + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "app deploying", "imageType", imageType, "deploymentType", appDeploymentType) + if !cloudcommon.IsValidDeploymentForImage(app.ImageType, appDeploymentType) { + return nil, fmt.Errorf("deployment is not valid for image type") + } + vp := &rootLB.PlatConf.Values + data := templateFill{ + ResourceKind: "application", + Resource: NormalizeName(appInst.Key.AppKey.Name), + Kind: vp.Application.Kind, //"kubernetes", + Name: NormalizeName(appInst.Key.AppKey.Name), + Tags: NormalizeName(appInst.Key.AppKey.Name), + Key: clusterInst.Key.ClusterKey.Name, + Tenant: NormalizeName(appInst.Key.AppKey.Name), + DNSZone: vp.Network.DNSZone, // "mobiledgex.net", + Operator: NormalizeName(clusterInst.Key.CloudletKey.OperatorKey.Name), + RootLB: rootLB.Name, + Image: app.ImagePath, + ImageType: imageType, + ImageFlavor: appInst.Flavor.Name, + ProxyPath: NormalizeName(appInst.Key.AppKey.Name), + AppURI: appInst.Uri, + IPAccess: ipAccess, + NetworkScheme: vp.Network.Scheme, //XXX "external-ip," + GetMEXExternalNetwork(rootLB.PlatConf), + Config: templateConfig{ + Deployment: app.Deployment, //vp.Application.Deployment + Resources: config.Resources, + Manifest: app.DeploymentManifest, //XXX vp.Application.Manifest,controller passes entire YAML + Template: vp.Application.Template, + Base: vp.Application.Base, + Overlay: vp.Application.Overlay, + }, + SpecPorts: vp.Application.Ports, + Command: strings.Split(app.Command, " "), + } + mf, err = templateUnmarshal(&data, yamlMEXApp) + if err != nil { + return nil, err + } + switch appDeploymentType { + case cloudcommon.AppDeploymentTypeKubernetes: + case cloudcommon.AppDeploymentTypeDockerSwarm: + case cloudcommon.AppDeploymentTypeKVM: + case cloudcommon.AppDeploymentTypeHelm: + default: + return nil, fmt.Errorf("unknown image type %s", imageType) + } + log.DebugLog(log.DebugLevelMexos, "filled app manifest") + //XXX inconsistent addPorts after template fill + err = addPorts(mf, appInst) + if err != nil { + return nil, err + } + log.DebugLog(log.DebugLevelMexos, "added port to app manifest") + return mf, nil +} + +func templateUnmarshal(data *templateFill, yamltext string) (*Manifest, error) { + //log.DebugLog(log.DebugLevelMexos, "template unmarshal", "yamltext", string, "data", data) + tmpl, err := template.New("mex").Parse(yamltext) + if err != nil { + return nil, fmt.Errorf("can't create template for, %v", err) + } + var outbuffer bytes.Buffer + err = tmpl.Execute(&outbuffer, data) + if err != nil { + //log.DebugLog(log.DebugLevelMexos, "template data", "data", data) + return nil, fmt.Errorf("can't execute template, %v", err) + } + mf := &Manifest{} + err = yaml.Unmarshal(outbuffer.Bytes(), mf) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error yaml unmarshal, templated data") + return nil, fmt.Errorf("can't unmarshal templated data, %v, %s", err, outbuffer.String()) + } + return mf, nil +} diff --git a/mexos/types.go b/mexos/types.go new file mode 100644 index 000000000..4fb41edb4 --- /dev/null +++ b/mexos/types.go @@ -0,0 +1,226 @@ +package mexos + +//MetadataDetail has metadata +type MetadataDetail struct { + Name string `json:"name"` + Tags string `json:"tags"` + Tenant string `json:"tenant"` + Region string `json:"region"` + Zone string `json:"zone"` + Location string `json:"location"` + Project string `json:"project"` + ResourceGroup string `json:"resourcegroup"` + OpenRC string `json:"openrc"` + DNSZone string `json:"dnszone"` + Kind string `json:"kind"` + Operator string `json:"operator"` + Swarm string `json:"swarm"` +} + +//NetworkDetail has network data +type NetworkDetail struct { + Name string `json:"name"` + Kind string `json:"kind"` + CIDR string `json:"cidr"` + Options string `json:"options"` + Extra string `json:"extra"` +} + +//AgentDetail has data on agent +type AgentDetail struct { + Image string `json:"image"` + Status string `json:"status"` +} + +//FlavorDetail has data on flavor +type FlavorDetail struct { + Name string `json:"name"` + Favorite string `json:"favorite"` + Memory string `json:"memory"` + Topology string `json:"topology"` + NodeFlavor string `json:"nodeflavor"` + MasterFlavor string `json:"masterflavor"` + NetworkScheme string `json:"networkscheme"` + Storage string `json:"storage"` + StorageScheme string `json:"storagescheme"` + CPUs int `json:"cpus"` + Masters int `json:"masters"` + Nodes int `json:"nodes"` +} + +type FlavorDetailInfo struct { + Name string `json:"name"` + Nodes int `json:"nodes"` + Masters int `json:"masters"` + NetworkScheme string `json:"networkscheme"` + MasterFlavor string `json:"masterflavor"` + NodeFlavor string `json:"nodeflavor"` + StorageScheme string `json:"storagescheme"` + Topology string `json:"topology"` +} + +type PortDetail struct { + Name string `json:"name"` + MexProto string `json:"mexproto"` + Proto string `json:"proto"` + InternalPort int `json:"internalport"` + PublicPort int `json:"publicport"` + PublicPath string `json:"publicpath"` +} + +//SpecDetail holds spec block +type SpecDetail struct { + Flavor string `json:"flavor"` // appInst flavor? + FlavorDetail FlavorDetailInfo `json:"flavordetail"` + Flags string `json:"flags"` + RootLB string `json:"rootlb"` + Image string `json:"image"` + ImageFlavor string `json:"imageflavor"` + ImageType string `json:"imagetype"` + DockerRegistry string `json:"dockerregistry"` + ExternalNetwork string `json:"externalnetwork"` + ExternalRouter string `json:"externalrouter"` + Options string `json:"options"` + ProxyPath string `json:"proxypath"` + Ports []PortDetail `json:"ports"` + Command []string `json:"command"` + IPAccess string `json:"ipaccess"` + URI string `json:"uri"` + Key string `json:"key"` + NetworkScheme string `json:"networkscheme"` + Agent AgentDetail `json:"agent"` +} + +type AppInstConfigDetail struct { + Deployment string `json:"deployment"` + Resources string `json:"resources"` + Template string `json:"template"` + Manifest string `json:"manifest"` + Base string `json:"base"` +} + +type AppInstConfig struct { + Kind string `json:"kind"` + Source string `json:"source"` + ConfigDetail AppInstConfigDetail `json:"detail"` +} + +type AgentValue struct { + Image string `json:"image"` + Port string `json:"port"` + Status string `json:"status"` +} + +type AppValue struct { + Deployment string `json:"deployment"` + Name string `json:"name"` + Kind string `json:"kind"` + Manifest string `json:"manifest"` + Image string `json:"image"` + ImageType string `json:"imagetype"` + ProxyPath string `json:"proxypath"` + Template string `json:"template"` + Base string `json:"base"` + Overlay string `json:"overlay"` + Ports []PortDetail `json:"ports"` +} + +type ClusterValue struct { + Flavor string `json:"flavor"` + Kind string `json:"kind"` + Name string `json:"name"` + Zone string `json:"zone"` + Region string `json:"region"` + Location string `json:"location"` + OSImage string `json:"osimage"` + Tenant string `json:"tenant"` + Swarm string `json:"swarm"` +} + +type StorageValue struct { + Name string `json:"name"` + Scheme string `json:"scheme"` +} + +type EnvironmentValue struct { + OpenRC string `json:"openrc"` + MexEnv string `json:"mexenv"` +} + +type NetworkValue struct { + Router string `json:"router"` + Options string `json:"options"` + Name string `json:"name"` + External string `json:"external"` + IPAccess string `json:"ipaccess"` + SecurityRule string `json:"securityrule"` + Scheme string `json:"scheme"` + DNSZone string `json:"dnszone"` + HolePunch string `json:"holepunch"` +} + +type OperatorValue struct { + Name string `json:"name"` + Kind string `json:"kind"` +} + +type RegistryValue struct { + Name string `json:"name"` + Docker string `json:"docker"` + Update string `json:"update"` +} + +type ResourceValue struct { + Group string `json:"group"` + Project string `json:"project"` +} + +type ValueDetail struct { + Name string `json:"name"` + Kind string `json:"kind"` + Base string `json:"base"` + Application AppValue `json:"application"` + Agent AgentValue `json:"agent"` + Cluster ClusterValue `json:"cluster"` + Network NetworkValue `json:"network"` + Registry RegistryValue `json:"registry"` + Operator OperatorValue `json:"operator"` + Resource ResourceValue `json:"resource"` + Environment EnvironmentValue `json:"environment"` + VaultEnvMap map[string]string +} + +type EnvData struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type VaultDataDetail struct { + Env []EnvData `json:"env"` +} + +type VaultData struct { + Detail VaultDataDetail `json:"data"` +} + +type VaultResponse struct { + Data VaultData `json:"data"` +} + +//Manifest is general container for the manifest yaml used by `mex` +type Manifest struct { + Name string `json:"name"` + APIVersion string `json:"apiVersion"` + Base string `json:"base"` + Kind string `json:"kind"` + Resource string `json:"resource"` + Metadata MetadataDetail `json:"metadata"` + Spec SpecDetail `json:"spec"` + Config AppInstConfig `json:"config"` + Values ValueDetail `json:"values"` +} + +type NetSpecInfo struct { + Kind, Name, CIDR, Options string + Extra []string +} diff --git a/mexos/uri.go b/mexos/uri.go new file mode 100644 index 000000000..48f43ad8e --- /dev/null +++ b/mexos/uri.go @@ -0,0 +1,107 @@ +package mexos + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + + sh "github.com/codeskyblue/go-sh" + "github.com/mobiledgex/edge-cloud/log" +) + +//validateDomain does strange validation, not strictly domain, due to the data passed from controller. +// if it is FQDN it is valid. And if it starts with http:// or https:// and followed by fqdn, it is valid. +func validateDomain(uri string) error { + if isDomainName(uri) { + return nil + } + fqdn := uri2fqdn(uri) + if isDomainName(fqdn) { + return nil + } + return fmt.Errorf("URI %s is not a valid domain name", uri) +} + +func GetURIFile(mf *Manifest, uri string) ([]byte, error) { + log.DebugLog(log.DebugLevelMexos, "attempt to get uri file", "uri", uri) + // if _, err := url.ParseRequestURI(uri); err != nil { + // return nil, err + // } + if strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://") { + res, err := GetHTTPFile(mf, uri) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error getting http uri file", "uri", uri, "error", err) + return nil, err + } + return res, nil + } + if strings.HasPrefix(uri, "scp://") { + res, err := GetSCPFile(mf, uri) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error getting scp uri file", "uri", uri, "error", err) + return nil, err + } + return res, nil + } + if strings.HasPrefix(uri, "file:///") { + uri = strings.Replace(uri, "file:///", "", -1) + } + // if err := validateDomain(uri); err != nil { + // return ioutil.ReadFile(uri) + // } + log.DebugLog(log.DebugLevelMexos, "attempt to read uri as normal file", "uri", uri) + res, err := ioutil.ReadFile(uri) + if err != nil { + log.DebugLog(log.DebugLevelMexos, "error getting file uri file", "uri", uri, "error", err) + return nil, err + } + return res, nil +} + +func GetHTTPFile(mf *Manifest, uri string) ([]byte, error) { + log.DebugLog(log.DebugLevelMexos, "attempt to get http uri file", "uri", uri) + resp, err := http.Get(uri) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK { + res, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return res, nil + } + return nil, fmt.Errorf("http status not OK, %v", resp.StatusCode) +} + +func GetSCPFile(mf *Manifest, uri string) ([]byte, error) { + log.DebugLog(log.DebugLevelMexos, "attempt to get scp uri file", "uri", uri) + part1 := strings.Replace(uri, "scp://", "mobiledgex@", -1) + slashindex := strings.Index(part1, "/") + if slashindex < 0 { + return nil, fmt.Errorf("malformed uri, missing /") + } + addr := part1[:slashindex] + if len(part1) < (slashindex + 1) { + return nil, fmt.Errorf("malformed uri, too short") + } + fn := part1[slashindex+1:] + if len(fn) < 1 { + return nil, fmt.Errorf("malformed uri, fn too short") + } + return sh.Command("ssh", "-o", sshOpts[0], "-o", sshOpts[1], "-i", PrivateSSHKey(), addr, "cat", fn).Output() +} + +// func CopyURIFile(mf *Manifest, uri string, fn string) error { +// res, err := GetURIFile(mf, uri) +// if err != nil { +// return err +// } +// err = ioutil.WriteFile(fn, res, 0644) +// if err != nil { +// return err +// } +// return nil +// } diff --git a/mexos/valid.go b/mexos/valid.go new file mode 100644 index 000000000..77d9013d0 --- /dev/null +++ b/mexos/valid.go @@ -0,0 +1,171 @@ +package mexos + +import ( + "fmt" + "os" + "reflect" + "strings" + + "github.com/mobiledgex/edge-cloud/log" +) + +var validMEXOSEnv = false + +func CheckManifest(mf *Manifest) error { + if mf.APIVersion == "" { + return fmt.Errorf("mf apiversion not set") + } + if mf.APIVersion != APIversion { + return fmt.Errorf("invalid api version") + } + return CheckManifestValues(mf) +} + +func CheckManifestValues(mf *Manifest) error { + log.DebugLog(log.DebugLevelMexos, "checking manifest values") + return CheckVals(mf.Values) +} + +func CheckVals(vals interface{}) error { + m := reflect.ValueOf(vals) + if strings.HasPrefix(m.Type().String(), "map[") { + return nil + } + for i := 0; i < m.NumField(); i++ { + n := m.Type().Field(i).Name + t := m.Type().Field(i).Type + v := m.Field(i).Interface() + log.DebugLog(log.DebugLevelMexos, "checkvals", "name", n, "type", m.Type(), "field type", t, "value", v) + if t.String() == "[]mexos.PortDetail" { // XXX skip ports, TODO check !nil and validate + continue + } + if t.String() == "string" { + if v == "" { + log.DebugLog(log.DebugLevelMexos, "checkvals, warning, empty field", "name", n, "type", m.Type(), "field type", t, "value", v) + } + //log.DebugLog(log.DebugLevelMexos, "ok", "name", n, "type", t, "value", v) + } else { + if err := CheckVals(m.Field(i).Interface()); err != nil { + return err + } + } + } + validMEXOSEnv = true + return nil +} + +func IsValidMEXOSEnv() bool { + if os.Getenv("MEX_CF_KEY") == "" { // XXX + return false + } + return validMEXOSEnv +} + +//ValidateNetSpec parses and validates the netSpec +func ValidateNetSpec(netSpec string) error { + // TODO if _,err:=ParseNetSpec(netSpec); err!=nil{ return err} + if netSpec == "" { + return fmt.Errorf("empty netspec") + } + return nil +} + +//ValidateTags parses and validates tags +func ValidateTags(tags string) error { + // TODO a=b,c=d,... + if tags == "" { + return fmt.Errorf("empty tags") + } + return nil +} + +//ValidateTenant parses and validates tenant +func ValidateTenant(tenant string) error { + // TODO suffix -tenant + if tenant == "" { + return fmt.Errorf("emtpy tenant") + } + return nil +} + +func ValidateClusterKind(kind string) error { + // TODO list of acceptable cluster kinds to be provided elsewhere + log.DebugLog(log.DebugLevelMexos, "cluster kind", "kind", kind) + if kind == "" { + return fmt.Errorf("empty cluster kind") + } + //TODO: add more kinds of clusters + for _, k := range []string{ + "gcp", + "azure", + //"gddt", + } { + if kind == k { + return nil + } + } + // if strings.HasPrefix(kind, "mex-") { + // return nil + // } + // log.DebugLog(log.DebugLevelMexos, "warning, cluster kind, operator has no mex- prefix", "kind", kind) + return nil +} + +func ValidateMetadata(mf *Manifest) error { + //TODO acceptable name patterns to be provided elsewhere + if mf.Metadata.Name == "" { + return fmt.Errorf("missing name for the deployment") + } + return nil +} + +func ValidateKey(mf *Manifest) error { + // TODO use of spec key as cluster name may need to change + if mf.Spec.Key == "" { + return fmt.Errorf("empty spec key name") + } + return nil +} + +func ValidateProxyPath(mf *Manifest) error { + // XXX is it error if this is empty? + if mf.Spec.ProxyPath == "" { + return fmt.Errorf("empty proxy path") + } + return nil +} + +func ValidateImage(mf *Manifest) error { + // TODO check valid list of images, provided elsewhere + if mf.Spec.Image == "" { + return fmt.Errorf("empty image") + } + return nil +} + +func ValidateDNSZone(mf *Manifest) error { + // TODO check against mobiledgex.net and other zones acceptable as listed in DB + if mf.Metadata.DNSZone == "" { + return fmt.Errorf("missing DNS zone, metadata %v", mf.Metadata) + } + return nil +} + +func ValidateCommon(mf *Manifest) error { + if err := ValidateMetadata(mf); err != nil { + return err + } + if err := ValidateKey(mf); err != nil { + return err + } + if err := ValidateProxyPath(mf); err != nil { + return err + } + if err := ValidateImage(mf); err != nil { + return err + } + if err := ValidateDNSZone(mf); err != nil { + return err + } + return nil +} diff --git a/mexos/vault.go b/mexos/vault.go new file mode 100644 index 000000000..460b37fe0 --- /dev/null +++ b/mexos/vault.go @@ -0,0 +1,148 @@ +package mexos + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + + "github.com/ghodss/yaml" + "github.com/mobiledgex/edge-cloud/log" +) + +func GetVaultData(url string) ([]byte, error) { + vault_token := os.Getenv("VAULT_TOKEN") + if vault_token == "" { + res, err := ioutil.ReadFile(os.Getenv("HOME") + "/.mobiledgex/vault.txt") + if err != nil { + return nil, fmt.Errorf("no vault token") + } + vault_token = strings.TrimSpace(string(res)) + if vault_token == "" { + return nil, fmt.Errorf("missing vault token") + } + } + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Vault-Token", vault_token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + contents, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return contents, nil +} + +func GetVaultEnvResponse(contents []byte) (*VaultResponse, error) { + vr := &VaultResponse{} + err := yaml.Unmarshal(contents, vr) + if err != nil { + return nil, err + } + return vr, nil +} + +var home = os.Getenv("HOME") + +func interpolate(val string) string { + if strings.HasPrefix(val, "$HOME") { + val = strings.Replace(val, "$HOME", home, -1) + } + return val +} + +func internEnv(envs []EnvData) error { + for _, e := range envs { + val := interpolate(e.Value) + err := os.Setenv(e.Name, val) + if err != nil { + return err + } + //log.DebugLog(log.DebugLevelMexos, "setenv", "name", e.Name, "value", val) + } + return nil +} + +func InternVaultEnv(mf *Manifest) error { + //log.DebugLog(log.DebugLevelMexos, "interning vault env var") + mf.Values.VaultEnvMap = make(map[string]string) + for _, u := range []string{mf.Values.Environment.OpenRC, mf.Values.Environment.MexEnv} { + if u == "" { + continue + } + dat, err := GetVaultData(u) + if err != nil { + return err + } + vr, err := GetVaultEnvResponse(dat) + if err != nil { + return err + } + for _, e := range vr.Data.Detail.Env { + mf.Values.VaultEnvMap[e.Name] = e.Value + } + //log.DebugLog(log.DebugLevelMexos, "interning vault data", "data", vr) + err = internEnv(vr.Data.Detail.Env) + if err != nil { + return err + } + } + log.DebugLog(log.DebugLevelMexos, "vault env var map", "vault env map", mf.Values.VaultEnvMap) + return nil +} + +func CheckPlatformEnv(platformType string) error { + // if !strings.Contains(platformType, "openstack") { // TODO gcp,azure,... + // log.DebugLog(log.DebugLevelMexos, "warning, unsupported, skip check platform environment", "platform", platformType) + // return nil + // } + // for _, n := range []struct { + // name string + // getter func() string + // }{ + // {"MEX_EXT_NETWORK", GetMEXExternalNetwork}, + // {"MEX_EXT_ROUTER", GetMEXExternalRouter}, + // {"MEX_NETWORK", GetMEXNetwork}, + // {"MEX_SECURITY_RULE", GetMEXSecurityRule}, + // } { + // ev := os.Getenv(n.name) + // if ev == "" { + // ev = n.getter() + // } + // if ev == "" { + // return fmt.Errorf("missing " + n.name) + // } + // } + // log.DebugLog(log.DebugLevelMexos, "doing oscli sanity check") + // _, err := ListImages() + // if err != nil { + // return fmt.Errorf("oscli sanity check failed, %v", err) + // } + return nil +} + +func GetVaultEnv(mf *Manifest, uri string) error { + dat, err := GetURIFile(mf, mf.Base+"/"+uri) + if err != nil { + return err + } + err = yaml.Unmarshal(dat, mf) + if err != nil { + return err + } + log.DebugLog(log.DebugLevelMexos, "about to intern vault env", "mf", mf) + if err := InternVaultEnv(mf); err != nil { + return err + } + if err := CheckPlatformEnv(mf.Values.Operator.Kind); err != nil { + return err + } + return err +} diff --git a/mgmt/registry/install-files.sh b/mgmt/registry/install-files.sh new file mode 100755 index 000000000..fc32bcdef --- /dev/null +++ b/mgmt/registry/install-files.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -x +dir=~/src/github.com/mobiledgex/edge-cloud-infra/k8s-prov +dest=registry.mobiledgex.net:files-repo/mobiledgex +for f in install-k8s-*.sh; do + scp $dir/$f $dest +done +scp ~/src/github.com/mobiledgex/edge-cloud-infra/openstack-tenant/qcow2/mobiledgex-16.04-qcow2/mobiledgex-init.sh $dest +# examples deployed. Change to production when ready. +dir=~/src/github.com/mobiledgex/edge-cloud-infra/deployments/examples +for d in application cluster platform kustomize; do + scp -r $dir/$d $dest +done + diff --git a/openstack-prov/oscliapi/mex_test.go b/openstack-prov/oscliapi/mex_test.go deleted file mode 100644 index 73e01053a..000000000 --- a/openstack-prov/oscliapi/mex_test.go +++ /dev/null @@ -1,275 +0,0 @@ -package oscli - -import ( - "fmt" - //"github.com/rs/xid" - "os" - "strings" - "testing" -) - -var mexTestInfra1 = os.Getenv("MEX_TEST_INFRA") - -var tenant = "test-tenant" - -var roleAgent = "mex-agent-node" //installs docker -var agentName = "" - -func TestCreateMEXAgent(t *testing.T) { - if mexTestInfra1 == "" { - return - } - //guid := xid.New() - //agentName = "mex-agent-test-" + guid.String() - agentName = "mex-agent-test-1" - - // XXX 10.101.X.X/24 is not used. Just place-holder for now. - // id 1 is not used either. DHCP is used. - err := CreateMEXKVM(agentName, roleAgent, "external-ip,external-network-shared,10.101.X.X/24,dhcp", roleAgent, tenant, 1) - //XXX use roleAgent as tags, for now - - if err != nil { - t.Errorf("can't create kvm, %v", err) - return - } - fmt.Println("created kvm") -} - -func TestDestroyMEXAgent(t *testing.T) { - if mexTestInfra1 == "" { - return - } - if agentName == "" { - sl, err := ListServers() - if err != nil { - t.Error(err) - return - } - for _, s := range sl { - if strings.HasPrefix(s.Name, "mex-agent-test-") { - agentName = s.Name - } - } - } - - err := DestroyMEXKVM(agentName, roleAgent) - if err != nil { - t.Error(err) - return - } -} - -var roleMaster = "k8s-master" //installs k8s master -var masterName = "" - -var test1Tags = "test-1" - -func TestCreateKubernetesMaster(t *testing.T) { - if mexTestInfra1 == "" { - return - } - //guid := xid.New() - //masterName = "mex-" + roleMaster + "-" + guid.String() - masterName = "mex-" + roleMaster + "-1" - //Master always has X.X.X.2 - err := CreateMEXKVM(masterName, roleMaster, "priv-subnet,mex-k8s-net1,10.101.X.0/24", test1Tags, tenant, 1) - if err != nil { - t.Errorf("can't create kubernetes master node, %v", err) - return - } - fmt.Println("created k8s-master") -} - -var roleNode1 = "k8s-node" //installs kubectl -var node1Name = "" - -func TestCreateKubernetesNode1(t *testing.T) { - if mexTestInfra1 == "" { - return - } - //guid := xid.New() - //node1Name = "mex-" + roleNode1 + "-" + guid.String() - node1Name = "mex-" + roleNode1 + "-1" - err := CreateMEXKVM(node1Name, roleNode1, "priv-subnet,mex-k8s-net-1,10.101.X.0/24", test1Tags, tenant, 1) - if err != nil { - t.Errorf("can't create kubernetes node 1, %v", err) - } - fmt.Println("created k8s-node 1") -} - -var roleNode2 = "k8s-node" -var node2Name = "" - -func TestCreateKubernetesNode2(t *testing.T) { - if mexTestInfra1 == "" { - return - } - //guid := xid.New() - //node2Name = "mex-" + roleNode2 + "-" + guid.String() - node2Name = "mex-" + roleNode2 + "-1" - err := CreateMEXKVM(node2Name, roleNode2, "priv-subnet,mex-k8s-net-1,10.101.X.0/24", test1Tags, tenant, 2) - if err != nil { - t.Errorf("can't create kubernetes node 2, %v", err) - } - fmt.Println("created k8s-node 2") -} - -func TestDeleteAgent(t *testing.T) { - if mexTestInfra1 == "" { - return - } - err := DeleteServer(agentName) - if err != nil { - t.Errorf("can't delete agent kvm") - return - } - - fmt.Println("delete", agentName) -} - -func TestDeleteAgentByRole(t *testing.T) { - if mexTestInfra1 == "" { - return - } - sl, err := ListServers() - if err != nil { - t.Errorf("can't get list of servers, %v", err) - return - } - for _, s := range sl { - if strings.HasPrefix(s.Name, "mex-") { - sd, err := GetServerDetails(s.Name) - if err != nil { - t.Errorf("can't get server detail for %s, %v", s.Name, err) - return - } - prop := sd.Properties - props := strings.Split(prop, ",") - for _, p := range props { - kv := strings.Split(p, "=") - if strings.Contains(kv[0], "role") { // extra space in front - if strings.Contains(kv[1], roleAgent) { // single quotes - err := DeleteServer(s.Name) - if err != nil { - t.Errorf("can't delete %s, %v", s.Name, err) - return - } - fmt.Println("delete", s.Name) - } - } - } - } - } -} - -//because of ephemeral node names these cannot be run individually - -//delete all nodes before master -func TestDestroyKubernetesNode1(t *testing.T) { - if mexTestInfra1 == "" { - return - } - err := DestroyMEXKVM(node1Name, roleNode1) - if err != nil { - t.Errorf("can't destroy %s, %v", node1Name, err) - return - } -} - -func TestDestroyKubernetesNode2(t *testing.T) { - if mexTestInfra1 == "" { - return - } - err := DestroyMEXKVM(node2Name, roleNode2) - if err != nil { - t.Errorf("can't destroy %s, %v", node2Name, err) - return - } -} - -func TestDestroyKubernetesMaster(t *testing.T) { - if mexTestInfra1 == "" { - return - } - err := DestroyMEXKVM(masterName, roleMaster) - if err != nil { - t.Errorf("can't destroy %s, %v", masterName, err) - return - } -} - -func TestDestroyKubernetesByTags(t *testing.T) { - if mexTestInfra1 == "" { - return - } - sl, err := ListServers() - if err != nil { - t.Errorf("can't get list of servers, %v", err) - return - } - //really should delete k8s-nodes first and then k8s-master. but since k8s-master - // are created first this sort of works. - // maybe fix later TODO - for _, s := range sl { - if strings.HasPrefix(s.Name, "mex-k8s-") { - sd, err := GetServerDetails(s.Name) - if err != nil { - t.Errorf("can't get server detail for %s, %v", s.Name, err) - return - } - prop := sd.Properties - props := strings.Split(prop, ",") - for _, p := range props { - kv := strings.Split(p, "=") - if strings.Contains(kv[0], "tags") { // extra space in front - if strings.Contains(kv[1], test1Tags) { // single quotes - err := DeleteServer(s.Name) - if err != nil { - t.Errorf("can't delete %s, %v", s.Name, err) - return - } - fmt.Println("delete", s.Name) - } - } - } - } - } -} - -func TestDestroyKubernetesByTenant(t *testing.T) { - if mexTestInfra1 == "" { - return - } - sl, err := ListServers() - if err != nil { - t.Errorf("can't get list of servers, %v", err) - return - } - //really should delete k8s-nodes first and then k8s-master. but since k8s-master - // are created first this sort of works. - // maybe fix later TODO - for _, s := range sl { - if strings.HasPrefix(s.Name, "mex-k8s-") { - sd, err := GetServerDetails(s.Name) - if err != nil { - t.Errorf("can't get server detail for %s, %v", s.Name, err) - return - } - prop := sd.Properties - props := strings.Split(prop, ",") - for _, p := range props { - kv := strings.Split(p, "=") - if strings.Contains(kv[0], "tenant") { // extra space in front - if strings.Contains(kv[1], tenant) { // single quotes - err := DeleteServer(s.Name) - if err != nil { - t.Errorf("can't delete %s, %v", s.Name, err) - return - } - fmt.Println("delete", s.Name) - } - } - } - } - } -} diff --git a/openstack-prov/oscliapi/oscli.go b/openstack-prov/oscliapi/oscli.go deleted file mode 100644 index 820ae2070..000000000 --- a/openstack-prov/oscliapi/oscli.go +++ /dev/null @@ -1,775 +0,0 @@ -package oscli - -import ( - "encoding/json" - "fmt" - "net" - "strings" - "time" - - sh "github.com/codeskyblue/go-sh" - "github.com/mobiledgex/edge-cloud/log" -) - -// There are issues with x509 certfication and token retrieval when using ../api -// with certain Openstack cloudlets. The issues are handled correctly by -// openstack CLI python code. But not in the gophercloud library. -// So the existence of this code is for: (1) avoid all the issues that are -// time-wasting and go directly to something that always works, -// (2) show clear examples, as a tutorial, to those who are not familiar -// with openstack. - -// Limit name,value pairs of Openstack tenant level limits. The only platform -// level stats available to us reliably at some cloudlets. -type Limit struct { - Name string - Value int -} - -// Server is output of 'openstack server list'. In Openstack, 'server' means -// an instance of KVM. -type Server struct { - Status, Name, Image, ID, Flavor, Networks string -} - -// ServerOpt is used to specify options when creating servers. -type ServerOpt struct { - AvailabilityZone string //XXX not used yet - Name, Image, Flavor string - UserData string - NetIDs []string - Properties []string -} - -// ServerDetail is used with output of 'openstack server show' to list -// gory details per server. -type ServerDetail struct { - TaskState string `json:"OS-EXT-STS:task_state"` - Addresses string `json:"addresses"` - Image string `json:"image"` - VMState string `json:"OS-EXT-STS:vm_state"` - LaunchedAt string `json:"OS-SRV-USG:launched_at"` - Flavor string `json:"flavor"` - ID string `json:"id"` - SecurityGroups string `json:"security_groups"` - VolumesAttached string `json:"volumes_attached"` - UserID string `json:"user_id"` - DiskConfig string `json:"OS-DCF:diskConfig"` - AccessIPv4 string `json:"accessIPv4"` - AccessIPv6 string `json:"accessIPv6"` - Progress int `json:"progress"` - PowerState string `json:"OS-EXT-STS:power_state"` - ProjectID string `json:"project_id"` - ConfigDrive string `json:"config_drive"` - Status string `json:"status"` - Updated string `json:"updated"` - HostID string `json:"hostId"` - TerminatedAt string `json:"OS-SRV-USG:terminated_at"` - KeyName string `json:"key_name"` - AvailabilityZone string `json:"OS-EXT-AZ:availability_zone"` - Name string `json:"name"` - Created string `json:"created"` - Properties string `json:"properties"` -} - -// Image is used with 'openstack image list' -type Image struct { - Status, ID, Name string -} - -// Network is used with 'openstack network list' -// A network in openstack is created as needed. Some are -// created by the provider and has extra features. Such -// as external connectivity. -type Network struct { - Subnets, ID, Name string -} - -// NetworkDetail has a informational data for the network. -type NetworkDetail struct { - ID string `json:"id"` - Name string `json:"name"` - ProviderPhysicalNetwork string `json:"provider:physical_network"` - IPv6AddressScope string `json:"ipv6_address_scope"` - DNSDomain string `json:"dns_domain"` - IsVLANTransparent string `json:"is_vlan_transparent"` - ProviderNetworkType string `json:"provider:network_type"` - External string `json:"router:external"` - AvailabilityZoneHints string `json:"availability_zone_hints"` - AvailabilityZones string `json:"availability_zones"` - Segments string `json:"segments"` - IPv4AddressScope string `json:"ipv4_address_scope"` - ProjectID string `json:"project_id"` - Status string `json:"status"` - Subnets string `json:"subnets"` - Description string `json:"description"` - Tags string `json:"tags"` - UpdatedAt string `json:"updated_at"` - ProviderSegmentationID string `json:"provider:segmentation_id"` - QOSPolicyID string `json:"qos_policy_id"` - AdminStateUp string `json:"admin_state_up"` - CreatedAt string `json:"created_at"` - RevisionNumber int `json:"revision_number"` - MTU int `json:"mtu"` - PortSecurityEnabled bool `json:"port_security_enabled"` - Shared bool `json:"shared"` - IsDefault bool `json:"is_default"` -} - -// Flavor is used with 'openstack flavor list' -type Flavor struct { - Name, ID string - RAM, Ephemeral, VCPUs, Disk int -} - -// Subnet is used with 'openstack subnet list' -// In openstack, each 'network' contains 'subnets'. -// Networks have higher abstractions. The subnets -// actually contain network ranges, etc. -type Subnet struct { - Name, ID, Network, Subnet string -} - -//SubnetDetail contains details about a given subnet. -// ID,Name and GatewayIP are useful. -type SubnetDetail struct { - ID string `json:"id"` - Name string `json:"name"` - ServiceTypes string `json:"service_types"` - Description string `json:"description"` - EnableDHCP bool `json:"enable_dhcp"` - SegmentID string `json:"segment_id"` - NetworkID string `json:"network_id"` - CreatedAt string `json:"created_at"` - Tags string `json:"tags"` - DNSNameServers string `json:"dns_nameservers"` - UpdatedAt string `json:"updated_at"` - IPv6RAMode string `json:"ipv6_ra_mode"` - AllocationPools string `json:"allocation_pools"` - GatewayIP string `json:"gateway_ip"` - RevisionNumber int `json:"revision_number"` - IPv6AddressMode string `json:"ipv6_address_mode"` - IPVersion int `json:"ip_version"` - HostRoutes string `json:"host_routes"` - CIDR string `json:"cidr"` - ProjectID string `json:"project_id"` - SubnetPoolID string `json:"subnetpool_id"` -} - -//Router is used with 'openstack router' commands. -// In openstack router is virtually created to connect different subnets -// and possible to external networks. -type Router struct { - Name, ID, Status, State, Project string - HA, Distributed bool -} - -//RouterDetail lists more info per router -type RouterDetail struct { - ID string `json:"id"` - Name string `json:"name"` - ExternalGatewayInfo string `json:"external_gateway_info"` - Status string `json:"status"` - AvailabilityZoneHints string `json:"availability_zone_hints"` - AvailabilityZones string `json:"availability_zones"` - Description string `json:"description"` - AdminStateUp string `json:"admin_state_up"` - CreatedAt string `json:"created_at"` - Tags string `json:"tags"` - UpdatedAt string `json:"updated_at"` - InterfacesInfo string `json:"interfaces_info"` - ProjectID string `json:"project_id"` - FlavorID string `json:"flavor_id"` - Routes string `json:"routes"` - Distributed bool `json:"distributed"` - HA bool `json:"ha"` - RevisionNumber int `json:"revision_number"` -} - -//ExternalGateway details the info inside RouterDetail. The ExternalGatewayInfo is -// a string, which has to be double unmarshal'ed -type ExternalGateway struct { - NetworkID string `json:"network_id"` //subnet of external net - EnableSNAT bool `json:"enable_snat"` - ExternalFixedIPs []ExternalFixedIP `json:"external_fixed_ips"` //gateway between extnet and privnet -} - -//ExternalFixedIP similarly needs to be double unmarshalled as part of the above -type ExternalFixedIP struct { - SubnetID string `json:"subnet_id"` - IPAddress string `json:"ip_address"` -} - -//RouterInterface also needs to be used for double unmarshal -type RouterInterface struct { - SubnetID string `json:"subnet_id"` //attached privnet - IPAddress string `json:"ip_address"` //router for the privnet side on the subnet CIDR, usually X.X.X.1 but should really confirm by reading this - PortID string `json:"port_id"` -} - -// "Is Public" field is missing in 'Router' - -//GetLimits is used to retrieve tenant level platform stats -func GetLimits() ([]Limit, error) { - //err := sh.Command("openstack", "limits", "show", "--absolute", "-f", "json", sh.Dir("/tmp")).WriteStdout("os-out.txt") - out, err := sh.Command("openstack", "limits", "show", "--absolute", "-f", "json", sh.Dir("/tmp")).Output() - if err != nil { - err = fmt.Errorf("cannot get limits from openstack, %v", err) - return nil, err - } - var limits []Limit - err = json.Unmarshal(out, &limits) - if err != nil { - err = fmt.Errorf("cannot unmarshal, %v", err) - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "get limits", "limits", limits) - return limits, nil -} - -//ListServers returns list of servers, KVM instances, running on the system -func ListServers() ([]Server, error) { - out, err := sh.Command("openstack", "server", "list", "-f", "json").Output() - if err != nil { - err = fmt.Errorf("cannot get server list, %v", err) - return nil, err - } - var servers []Server - err = json.Unmarshal(out, &servers) - if err != nil { - err = fmt.Errorf("cannot unmarshal, %v", err) - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "list servers", "servers", servers) - return servers, nil -} - -//ListImages lists avilable images in glance -func ListImages() ([]Image, error) { - out, err := sh.Command("openstack", "image", "list", "-f", "json").Output() - if err != nil { - err = fmt.Errorf("cannot get image list, %v", err) - return nil, err - } - var images []Image - err = json.Unmarshal(out, &images) - if err != nil { - err = fmt.Errorf("cannot unmarshal, %v", err) - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "list images", "images", images) - return images, nil -} - -//ListNetworks lists networks known to the platform. Some created by the operator, some by users. -func ListNetworks() ([]Network, error) { - out, err := sh.Command("openstack", "network", "list", "-f", "json").Output() - if err != nil { - err = fmt.Errorf("cannot get network list, %v", err) - return nil, err - } - var networks []Network - err = json.Unmarshal(out, &networks) - if err != nil { - err = fmt.Errorf("cannot unmarshal, %v", err) - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "list networks", "networks", networks) - return networks, nil -} - -//ListFlavors lists flavors known to the platform. These vary. On Buckhorn cloudlet the are m4. prefixed. -func ListFlavors() ([]Flavor, error) { - out, err := sh.Command("openstack", "flavor", "list", "-f", "json").Output() - if err != nil { - err = fmt.Errorf("cannot get flavor list, %v", err) - return nil, err - } - var flavors []Flavor - err = json.Unmarshal(out, &flavors) - if err != nil { - err = fmt.Errorf("cannot unmarshal, %v", err) - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "list flavors", flavors) - return flavors, nil -} - -//CreateServer instantiates a new server instance, which is a KVM instance based on a qcow2 image from glance -func CreateServer(opts *ServerOpt) error { - args := []string{ - "server", "create", - "--config-drive", "true", //XXX always - "--image", opts.Image, "--flavor", opts.Flavor, - } - if opts.UserData != "" { - args = append(args, "--user-data", opts.UserData) - } - for _, p := range opts.Properties { - args = append(args, "--property", p) - // `p` should be like: "key=value" - } - for _, n := range opts.NetIDs { - args = append(args, "--nic", "net-id="+n) - // `n` should be like: "public,v4-fixed-ip=172.24.4.201" - } - args = append(args, opts.Name) - //TODO additional args - iargs := make([]interface{}, len(args)) - for i, v := range args { - iargs[i] = v - } - log.DebugLog(log.DebugLevelMexos, "openstack create server", "opts", opts, "iargs", iargs) - out, err := sh.Command("openstack", iargs...).Output() - if err != nil { - err = fmt.Errorf("cannot create server, %v, '%s'", err, out) - return err - } - return nil -} - -// GetServerDetails returns details of the KVM instance -func GetServerDetails(name string) (*ServerDetail, error) { - active := false - srvDetail := &ServerDetail{} - for i := 0; i < 10; i++ { - out, err := sh.Command("openstack", "server", "show", "-f", "json", name).Output() - if err != nil { - err = fmt.Errorf("can't show server %s, %s, %v", name, out, err) - return nil, err - } - //fmt.Printf("%s\n", out) - err = json.Unmarshal(out, srvDetail) - if err != nil { - err = fmt.Errorf("cannot unmarshal while getting server detail, %v", err) - return nil, err - } - if srvDetail.Status == "ACTIVE" { - active = true - break - } - log.DebugLog(log.DebugLevelMexos, "wait for server to become ACTIVE", "server detail", srvDetail) - time.Sleep(30 * time.Second) - } - if !active { - return nil, fmt.Errorf("while getting server detail, waited but server %s is too slow getting to active state", name) - } - log.DebugLog(log.DebugLevelMexos, "server detail", "server detail", srvDetail) - return srvDetail, nil -} - -//DeleteServer destroys a KVM instance -// sometimes it is not possible to destroy. Like most things in Openstack, try again. -func DeleteServer(id string) error { - log.DebugLog(log.DebugLevelMexos, "deleting server", "id", id) - out, err := sh.Command("openstack", "server", "delete", id).Output() - if err != nil { - err = fmt.Errorf("can't delete server %s, %s, %v", id, out, err) - return err - } - return nil -} - -// CreateNetwork creates a network with a name. -func CreateNetwork(name string) error { - log.DebugLog(log.DebugLevelMexos, "creating network", "network", name) - out, err := sh.Command("openstack", "network", "create", name).Output() - if err != nil { - err = fmt.Errorf("can't create network %s, %s, %v", name, out, err) - return err - } - return nil -} - -//DeleteNetwork destroys a named network -// Sometimes it will fail. Openstack will refuse if there are resources attached. -func DeleteNetwork(name string) error { - log.DebugLog(log.DebugLevelMexos, "deleting network", "network", name) - out, err := sh.Command("openstack", "network", "delete", name).Output() - if err != nil { - err = fmt.Errorf("can't delete network %s, %s, %v", name, out, err) - return err - } - return nil -} - -//NeutronErrorDetail holds neturon error -type NeutronErrorDetail struct { - Message string `json:"message"` - Type string `json:"type"` - Detail string `json:"detail"` -} - -//NeutronErrorType container for the NeutronErrorDetail -type NeutronErrorType struct { - NeutronError NeutronErrorDetail -} - -//CreateSubnet creates a subnet within a network. A subnet is assigned ranges. Optionally DHCP can be enabled. -func CreateSubnet(netRange, networkName, gatewayAddr, subnetName string, dhcpEnable bool) error { - var dhcpFlag string - if dhcpEnable { - dhcpFlag = "--dhcp" - } else { - dhcpFlag = "--no-dhcp" - } - out, err := sh.Command("openstack", "subnet", "create", - "--subnet-range", netRange, // e.g. 10.101.101.0/24 - "--network", networkName, // mex-k8s-net-1 - dhcpFlag, - "--gateway", gatewayAddr, // e.g. 10.101.101.1 - subnetName).CombinedOutput() // e.g. mex-k8s-subnet-1 - if err != nil { - nerr := &NeutronErrorType{} - if ix := strings.Index(string(out), `{"NeutronError":`); ix > 0 { - neutronErr := out[ix:] - if jerr := json.Unmarshal(neutronErr, nerr); jerr != nil { - err = fmt.Errorf("can't create subnet %s, %s, %v, error while parsing neutron error, %v", subnetName, out, err, jerr) - return err - } - if strings.Index(nerr.NeutronError.Message, "overlap") > 0 { - sd, serr := GetSubnetDetail(subnetName) - if serr != nil { - return fmt.Errorf("cannot get subnet detail for %s, while fixing overlap error, %v", subnetName, serr) - } - log.DebugLog(log.DebugLevelMexos, "create subnet, existing subnet detail", "subnet detail", sd) - - //XXX do more validation - - log.DebugLog(log.DebugLevelMexos, "create subnet, reusing existing subnet", "result", out, "error", err) - return nil - } - } - err = fmt.Errorf("can't create subnet %s, %s, %v", subnetName, out, err) - return err - } - return nil -} - -//DeleteSubnet deletes the subnet. If this fails, remove any attached resources, like router, and try again. -func DeleteSubnet(subnetName string) error { - log.DebugLog(log.DebugLevelMexos, "deleting subnet", "name", subnetName) - out, err := sh.Command("openstack", "subnet", "delete", subnetName).Output() - if err != nil { - err = fmt.Errorf("can't delete subnet %s, %s, %v", subnetName, out, err) - return err - } - return nil -} - -//CreateRouter creates new router. A router can be attached to network and subnets. -func CreateRouter(routerName string) error { - log.DebugLog(log.DebugLevelMexos, "creating router", "name", routerName) - out, err := sh.Command("openstack", "router", "create", routerName).Output() - if err != nil { - err = fmt.Errorf("can't create router %s, %s, %v", routerName, out, err) - return err - } - return nil -} - -//DeleteRouter removes the named router. The router needs to not be in use at the time of deletion. -func DeleteRouter(routerName string) error { - log.DebugLog(log.DebugLevelMexos, "deleting router", "name", routerName) - out, err := sh.Command("openstack", "router", "delete", routerName).Output() - if err != nil { - err = fmt.Errorf("can't delete router %s, %s, %v", routerName, out, err) - return err - } - return nil -} - -//SetRouter assigns the router to a particular network. The network needs to be attached to -// a real external network. This is intended only for routing to external network for now. No internal routers. -// Sometimes, oftentimes, it will fail if the network is not external. -func SetRouter(routerName, networkName string) error { - log.DebugLog(log.DebugLevelMexos, "setting router to network", "router", routerName, "network", networkName) - out, err := sh.Command("openstack", "router", "set", routerName, "--external-gateway", networkName).Output() - if err != nil { - err = fmt.Errorf("can't set router %s to %s, %s, %v", routerName, networkName, out, err) - return err - } - return nil -} - -//AddRouterSubnet will connect subnet to another network, possibly external, via a router -func AddRouterSubnet(routerName, subnetName string) error { - log.DebugLog(log.DebugLevelMexos, "adding router to subnet", "router", routerName, "network", subnetName) - out, err := sh.Command("openstack", "router", "add", "subnet", routerName, subnetName).Output() - if err != nil { - err = fmt.Errorf("can't add router %s to subnet %s, %s, %v", routerName, subnetName, out, err) - return err - } - return nil -} - -//RemoveRouterSubnet is useful to remove the router from the subnet before deletion. Otherwise subnet cannot -// be deleted. -func RemoveRouterSubnet(routerName, subnetName string) error { - log.DebugLog(log.DebugLevelMexos, "removing router from subnet", "router", routerName, "subnet", subnetName) - out, err := sh.Command("openstack", "router", "remove", "subnet", routerName, subnetName).Output() - if err != nil { - err = fmt.Errorf("can't remove router %s from subnet %s, %s, %v", routerName, subnetName, out, err) - return err - } - return nil -} - -//ListSubnets returns a list of subnets available -func ListSubnets(netName string) ([]Subnet, error) { - var err error - var out []byte - - if netName != "" { - out, err = sh.Command("openstack", "subnet", "list", "--network", netName, "-f", "json").Output() - } else { - out, err = sh.Command("openstack", "subnet", "list", "-f", "json").Output() - } - if err != nil { - err = fmt.Errorf("can't get a list of subnets, %v", err) - return nil, err - } - subnets := []Subnet{} - err = json.Unmarshal(out, &subnets) - if err != nil { - err = fmt.Errorf("can't unmarshal subnets, %v", err) - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "list subnets", "subnets", subnets) - return subnets, nil -} - -//ListRouters returns a list of routers available -func ListRouters() ([]Router, error) { - out, err := sh.Command("openstack", "router", "list", "-f", "json").Output() - if err != nil { - err = fmt.Errorf("can't get a list of routers, %s, %v", out, err) - return nil, err - } - routers := []Router{} - err = json.Unmarshal(out, &routers) - if err != nil { - err = fmt.Errorf("can't unmarshal routers, %v", err) - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "list routers", "routers", routers) - return routers, nil -} - -//GetRouterDetail returns details per router -func GetRouterDetail(routerName string) (*RouterDetail, error) { - out, err := sh.Command("openstack", "router", "show", "-f", "json", routerName).Output() - if err != nil { - err = fmt.Errorf("can't get router details for %s, %s, %v", routerName, out, err) - return nil, err - } - routerDetail := &RouterDetail{} - err = json.Unmarshal(out, routerDetail) - if err != nil { - err = fmt.Errorf("can't unmarshal router detail, %v", err) - return nil, err - } - log.DebugLog(log.DebugLevelMexos, "router detail", "router detail", routerDetail) - return routerDetail, nil -} - -//CreateServerImage snapshots running service into a qcow2 image -func CreateServerImage(serverName, imageName string) error { - log.DebugLog(log.DebugLevelMexos, "creating image snapshot from server", "server", serverName, "image", imageName) - out, err := sh.Command("openstack", "server", "image", "create", serverName, "--name", imageName).Output() - if err != nil { - err = fmt.Errorf("can't create image from %s into %s, %s, %v", serverName, imageName, out, err) - return err - } - return nil -} - -//CreateImage puts images into glance -func CreateImage(imageName, qcowFile string) error { - log.DebugLog(log.DebugLevelMexos, "creating image in glance", "image", imageName, "qcow", qcowFile) - out, err := sh.Command("openstack", "image", "create", - imageName, - "--disk-format", "qcow2", - "--container-format", "bare", - "--file", qcowFile).Output() - if err != nil { - err = fmt.Errorf("can't create image in glace, %s, %s, %s, %v", imageName, qcowFile, out, err) - return err - } - return nil -} - -//SaveImage takes the image name available in glance, as a result of for example the above create image. -// It will then save that into a local file. The image transfer happens from glance into your own laptop -// or whatever. -// This can take a while, transferring all the data. -func SaveImage(saveName, imageName string) error { - log.DebugLog(log.DebugLevelMexos, "saving image", "save name", saveName, "image name", imageName) - out, err := sh.Command("openstack", "image", "save", "--file", saveName, imageName).Output() - if err != nil { - err = fmt.Errorf("can't save image from %s to file %s, %s, %v", imageName, saveName, out, err) - return err - } - return nil -} - -//DeleteImage deletes the named image from glance. Sometimes backing store is still busy and -// will refuse to honor the request. Like most things in Openstack, wait for a while and try -// again. -func DeleteImage(imageName string) error { - log.DebugLog(log.DebugLevelMexos, "deleting image", "name", imageName) - out, err := sh.Command("openstack", "image", "delete", imageName).Output() - if err != nil { - err = fmt.Errorf("can't delete image %s, %s, %v", imageName, out, err) - return err - } - return nil -} - -//GetSubnetDetail returns details for the subnet. This is useful when getting router/gateway -// IP for a given subnet. The gateway info is used for creating a server. -// Also useful in general, like other `detail` functions, to get the ID map for the name of subnet. -func GetSubnetDetail(subnetName string) (*SubnetDetail, error) { - out, err := sh.Command("openstack", "subnet", "show", "-f", "json", subnetName).Output() - if err != nil { - err = fmt.Errorf("can't get subnet details for %s, %s, %v", subnetName, out, err) - return nil, err - } - subnetDetail := &SubnetDetail{} - err = json.Unmarshal(out, subnetDetail) - if err != nil { - return nil, fmt.Errorf("can't unmarshal subnet detail, %v", err) - } - log.DebugLog(log.DebugLevelMexos, "get subnet detail", "subnet detail", subnetDetail) - return subnetDetail, nil -} - -//GetNetworkDetail returns details about a network. It is used, for example, by GetExternalGateway. -func GetNetworkDetail(networkName string) (*NetworkDetail, error) { - out, err := sh.Command("openstack", "network", "show", "-f", "json", networkName).Output() - if err != nil { - err = fmt.Errorf("can't get details for network %s, %s, %v", networkName, out, err) - return nil, err - } - networkDetail := &NetworkDetail{} - err = json.Unmarshal(out, networkDetail) - if err != nil { - return nil, fmt.Errorf("can't unmarshal network detail, %v", err) - } - log.DebugLog(log.DebugLevelMexos, "get network detail", "network detail", networkDetail) - return networkDetail, nil -} - -//GetExternalGateway retrieves Gateway IP from the external network information. It first gets external -// network information. Using that it further gets subnet information. Inside that subnet information -// there should be gateway IP if the network is set up correctly. -// Not to be confused with GetRouterDetailExternalGateway. -func GetExternalGateway(extNetName string) (string, error) { - nd, err := GetNetworkDetail(extNetName) - if err != nil { - return "", fmt.Errorf("can't get details for external network %s, %v", extNetName, err) - } - - if nd.Status != "ACTIVE" { - return "", fmt.Errorf("network %s is not active, status %s", extNetName, nd.Status) - } - if nd.AdminStateUp != "UP" { - return "", fmt.Errorf("network %s is not admin-state set to up", extNetName) - } - subnets := strings.Split(nd.Subnets, ",") - //XXX beware of extra spaces - if len(subnets) < 1 { - return "", fmt.Errorf("no subnets for %s", extNetName) - } - //XXX just use first subnet -- may not work in all cases, but there is no tagging done rightly yet - sd, err := GetSubnetDetail(subnets[0]) - if err != nil { - return "", fmt.Errorf("cannot get details for subnet %s, %v", subnets[0], err) - } - //TODO check status of subnet - if sd.GatewayIP == "" { - return "", fmt.Errorf("cannot get external network's gateway IP") - } - log.DebugLog(log.DebugLevelMexos, "get external gatewayIP", "gatewayIP", sd.GatewayIP, "subnet detail", sd) - return sd.GatewayIP, nil -} - -//GetNextSubnetRange will find the CIDR for the next range of subnet that can be created. For example, -// if the subnet detail we get has 10.101.101.0/24 then the next one can be 10.101.102.0/24 -func GetNextSubnetRange(subnetName string) (string, error) { - sd, err := GetSubnetDetail(subnetName) - if err != nil { - return "", err - } - if sd.CIDR == "" { - return "", fmt.Errorf("missing CIDR in subnet %s", subnetName) - } - _, ipv4Net, err := net.ParseCIDR(sd.CIDR) - if err != nil { - return "", fmt.Errorf("can't parse CIDR %s, %v", sd.CIDR, err) - } - i := strings.Index(sd.CIDR, "/") - suffix := sd.CIDR[i:] - v4 := ipv4Net.IP.To4() - ipnew := net.IPv4(v4[0], v4[1], v4[2]+1, v4[3]) - log.DebugLog(log.DebugLevelMexos, "get next subnet range", "new ip range", ipnew, "suffix", suffix) - return ipnew.String() + suffix, nil -} - -//GetRouterDetailExternalGateway is different than GetExternalGateway. This function gets -// the gateway interface in the subnet within external network. This is -// accessible from private networks to route packets to the external network. -// The GetExternalGateway gets the gateway for the outside network. This is -// for the packets to be routed out to the external network, i.e. internet. -func GetRouterDetailExternalGateway(rd *RouterDetail) (*ExternalGateway, error) { - if rd.ExternalGatewayInfo == "" { - return nil, fmt.Errorf("empty external gateway info") - } - externalGateway := &ExternalGateway{} - err := json.Unmarshal([]byte(rd.ExternalGatewayInfo), externalGateway) - if err != nil { - return nil, fmt.Errorf("can't get unmarshal external gateway info, %v", err) - } - log.DebugLog(log.DebugLevelMexos, "get router detail external gateway", "external gateway", externalGateway) - return externalGateway, nil -} - -// GetRouterDetailInterfaces gets the list of interfaces on the router. For example, each private -// subnet connected to the router will be listed here with own interface definition. -func GetRouterDetailInterfaces(rd *RouterDetail) ([]RouterInterface, error) { - if rd.InterfacesInfo == "" { - return nil, fmt.Errorf("missing interfaces info in router details") - } - interfaces := []RouterInterface{} - err := json.Unmarshal([]byte(rd.InterfacesInfo), &interfaces) - if err != nil { - return nil, fmt.Errorf("can't unmarshal router detail interfaces") - } - log.DebugLog(log.DebugLevelMexos, "get router detail interfaces", "interfaces", interfaces) - return interfaces, nil -} - -//SetServerProperty sets properties for the server -func SetServerProperty(name, property string) error { - if name == "" { - return fmt.Errorf("empty name") - } - if property == "" { - return fmt.Errorf("empty property") - } - out, err := sh.Command("openstack", "server", "set", "--property", property, name).Output() - if err != nil { - return fmt.Errorf("can't set property %s on server %s, %s, %v", property, name, out, err) - } - log.DebugLog(log.DebugLevelMexos, "set server property", "name", name, "property", property) - return nil -} - -func AddSecurityRuleCIDR(cidr string, proto string, name string, port int) error { - portStr := fmt.Sprintf("%d", port) - out, err := sh.Command("openstack", "security", "group", "rule", "create", "--remote-ip", cidr, "--proto", proto, "--dst-port", portStr, "--ingress", name).Output() - if err != nil { - return fmt.Errorf("can't add security group rule for port %d to %s,%s,%v", port, name, out, err) - } - return nil -} diff --git a/openstack-prov/oscliapi/oscli_test.go b/openstack-prov/oscliapi/oscli_test.go deleted file mode 100644 index cc502b2f9..000000000 --- a/openstack-prov/oscliapi/oscli_test.go +++ /dev/null @@ -1,472 +0,0 @@ -package oscli - -import ( - "fmt" - "os" - "testing" - //log "github.com/sirupsen/logrus" -) - -var mexTestInfra2 = os.Getenv("MEX_TEST_INFRA") - -// The order of tests are important here. Also 'source openrc' or the equivalent to set up -// the OS_XXX env variables before running this. -// There is no mock for this for now. -// For CICD, either create a real live test cluster for that use, or -// write a bunch of fake mock returns -- will not do this here. -// During development, each of the tests were run individually against a live cluster. - -var testServerName = "go-test1" -var testImageName = "mobiledgex-16.04-2" -var testFlavor = "m4.small" -var testUserData = "userdata.txt" -var testNetwork = "test-external-network-shared" -var testSubnet = "test-internal-network" -var testRange = "10.102.102.0/24" -var testGateway = "10.102.102.1" -var testRouter = "test-router-1" - -func TestInit(t *testing.T) { - if mexTestInfra2 == "" { - return - } - //log.SetLevel(log.DebugLevel) -} - -func TestGetLimits(t *testing.T) { - if mexTestInfra2 == "" { - return - } - out, err := GetLimits() - if err != nil { - t.Errorf("cannot GetLimits, %v", err) - } else { - fmt.Println(out) - } -} - -func TestListServers(t *testing.T) { - if mexTestInfra2 == "" { - return - } - out, err := ListServers() - if err != nil { - t.Errorf("cannot ListServers, %v", err) - } else { - fmt.Println(out) - } -} - -func TestListImages(t *testing.T) { - if mexTestInfra2 == "" { - return - } - out, err := ListImages() - if err != nil { - t.Errorf("cannot ListImages, %v", err) - } else { - fmt.Println(out) - } -} - -func TestListNetworks(t *testing.T) { - if mexTestInfra2 == "" { - return - } - out, err := ListNetworks() - if err != nil { - t.Errorf("cannot ListNetworks, %v", err) - } else { - fmt.Println(out) - } -} - -func TestListFlavors(t *testing.T) { - if mexTestInfra2 == "" { - return - } - out, err := ListFlavors() - if err != nil { - t.Errorf("cannot ListFlavors, %v", err) - } else { - fmt.Println(out) - } -} - -func TestCreateServer(t *testing.T) { - if mexTestInfra2 == "" { - return - } - opts := &ServerOpt{ - Name: testServerName, - Image: testImageName, - Flavor: testFlavor, - UserData: testUserData, - NetIDs: []string{testNetwork}, - Properties: []string{}, - } - err := CreateServer(opts) - if err != nil { - t.Errorf("%v", err) - } else { - fmt.Println("created", opts) - } - - opts = &ServerOpt{ - Name: testServerName, - Image: testImageName, - Flavor: testFlavor, - UserData: testUserData, - NetIDs: []string{testNetwork}, - Properties: []string{}, - } - err = CreateServer(opts) - if err != nil { - fmt.Println("correctly failed to create", opts) - } else { - t.Errorf("should have failed to create server with the same name") - } -} - -func TestGetServerDetails(t *testing.T) { - if mexTestInfra2 == "" { - return - } - sd, err := GetServerDetails(testServerName) - if err != nil { - t.Errorf("server show err, %v", err) - return - } - fmt.Println("server", sd) - - _, err = GetServerDetails(testServerName + "xxx") - if err == nil { - t.Errorf("should have failed") - return - } - fmt.Println("correctly failed to get details for bogus server name") -} - -func TestDeleteServer(t *testing.T) { - if mexTestInfra2 == "" { - return - } - sd, err := GetServerDetails(testServerName) - if err != nil { - t.Errorf("cannot show server %s, %v", testServerName, err) - } - - if sd.Name != testServerName { //Xxx never happen - t.Errorf("name mismatch") - } - - //log.Debugln("delete server %s %s", sd.Name, sd.ID) - - err = DeleteServer(sd.ID) - if err != nil { - t.Errorf("cannot delete server %s %s, %v", sd.Name, sd.ID, err) - } else { - fmt.Println("server", sd.Name, "deleted ok") - } -} - -func TestCreateNetwork(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := CreateNetwork(testNetwork) - if err != nil { - t.Errorf("cannot create network, %v", err) - return - } - fmt.Println("network", testNetwork, "created") - - err = CreateNetwork(testNetwork) - if err == nil { - t.Errorf("should have failed to create network with existing name") - return - } - fmt.Println("correctly failed to create network with duplicate name") -} - -func TestCreateSubnet(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := CreateSubnet(testRange, testNetwork, testGateway, testSubnet, false) - if err != nil { - t.Errorf("cannot create subnet, %v", err) - return - } - fmt.Println("created subnet ", testSubnet) - - err = CreateSubnet(testRange, testNetwork, testGateway, testSubnet, false) - if err == nil { - t.Errorf("should have failed to create subnet with duplicate name") - return - } - fmt.Println("correctly failed to create duplicate subnet") -} - -func TestCreateRouter(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := CreateRouter(testRouter) - if err != nil { - t.Errorf("can't create router, %v", err) - return - } - fmt.Println("created router ", testRouter) - - err = CreateRouter(testRouter) - if err == nil { - t.Errorf("should have failed to create duplicate router") - return - } - fmt.Println("correctly failed to create dup router") -} - -func TestSetRouter(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := SetRouter(testRouter, testNetwork) - if err != nil { - fmt.Printf("can't set router, %v", err) - fmt.Printf("not an error.") - //because testNetwork is not a real external network - return - } - fmt.Printf("set router %s in net %s\n", testRouter, testNetwork) - - err = SetRouter(testRouter, testNetwork) - if err == nil { - t.Errorf("should have failed to set router again") - return - } - fmt.Printf("correctly failed to set router again") -} - -func TestAddRouterSubnet(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := AddRouterSubnet(testRouter, testSubnet) - if err != nil { - t.Errorf("can't add router to subnet, %v", err) - return - } - fmt.Printf("added router %s to subnet %s\n", testRouter, testSubnet) - - err = AddRouterSubnet(testRouter, testSubnet) - if err == nil { - t.Errorf("should have failed to add dup router to subnet") - return - } - fmt.Printf("correctly failed to add dup router to subnet") -} - -func TestListSubnets(t *testing.T) { - if mexTestInfra2 == "" { - return - } - subnets, err := ListSubnets("") //list all - if err != nil { - t.Errorf("can't list subnets, %v", err) - return - } - fmt.Printf("subnets %v\n", subnets) -} - -func TestListRouters(t *testing.T) { - if mexTestInfra2 == "" { - return - } - routers, err := ListRouters() - if err != nil { - t.Errorf("can't list routers, %v", err) - return - } - fmt.Printf("routers %v\n", routers) -} - -// The order of the following sequence of tests are particularly important -// For example, it is good idea to remove the router assigned to a subnet -// before removing subnet. - -func TestRemoveRouterSubnet(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := RemoveRouterSubnet(testRouter, testSubnet) - if err != nil { - t.Errorf("can't remove router from subnet, %v", err) - return - } - fmt.Printf("removed router %s from subnet %s\n", testRouter, testSubnet) - - err = RemoveRouterSubnet(testRouter, testSubnet) - if err == nil { - t.Errorf("should have failed to remove router from subnet again") - return - } - fmt.Printf("correctly failed to remove the router from subnet again") -} - -func TestDeleteRouter(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := DeleteRouter(testRouter) - if err != nil { - t.Errorf("can't delete router, %v", err) - return - } - fmt.Println("deleted router ", testRouter) - - err = DeleteRouter(testRouter) - if err == nil { - t.Errorf("should have failed to delete router again") - return - } - fmt.Println("correctly failed to remove router again") - -} -func TestDeleteSubnet(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := DeleteSubnet(testSubnet) - if err != nil { - t.Errorf("cannot delete subnet , %v", err) - return - } - fmt.Println("deleted subnet s", testSubnet) - - err = DeleteSubnet(testSubnet) - if err == nil { - t.Errorf("should have failed to delete subnet again") - return - } - fmt.Println("correctly failed to remove subnet again") -} - -func TestDeleteNetwork(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := DeleteNetwork(testNetwork) - if err != nil { - t.Errorf("cannot delete network, %v", err) - return - } - fmt.Println("deleted network ", testNetwork) - - err = DeleteNetwork(testNetwork) - if err == nil { - t.Errorf("should have failed to delete network again") - return - } - fmt.Println("correctly failed to remove the network again") -} - -// The orders are not preserved here. The server should be running. -// But it won't be when we run tests serially. - -var testImage = "test-image-1" - -// TestCreateImage is kind of `snapshotting` the running KVM image into glance -func TestCreateImage(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := CreateImage(testServerName, testImage) - if err != nil { - t.Errorf("cannot create image , %v", err) - return - } - fmt.Println("created image", testImage) - - err = CreateImage(testServerName, testImage) - if err == nil { - t.Errorf("should have failed to create image again") - return - } - fmt.Println("correctly failed to create image again") -} - -var saveImageFile = "test-save-image.qcow2" // will be created locally. Potentially very large. - -//Saving image that has been tagged into glance with a name before. The saving is -// actually retrieving the image over network from cloudlet, into local storage. -// It can take some time and storage. -func TestSaveImage(t *testing.T) { - if mexTestInfra2 == "" { - return - } - //This can take a while - err := SaveImage(saveImageFile, testImage) - if err != nil { - t.Errorf("cannot save image , %v", err) - return - } - fmt.Println("saved image", testImage+"XXX") - - err = SaveImage(saveImageFile, testImage) - if err == nil { - t.Errorf("should have failed to save bogus image locally") - return - } - fmt.Println("correctly failed to save bogus image") - - //TODO: it is possible for the platform to refuse this request when - // the platform is slow and the tasks ongoing are queued for whatever reason, such - // as slow storage. So we can test for retries and legit fails. But we don't for now. -} - -func TestDeleteImage(t *testing.T) { - if mexTestInfra2 == "" { - return - } - err := DeleteImage(testImage) - if err != nil { - t.Errorf("cannot delete image , %v", err) - return - } - fmt.Println("deleted image", testImage) - - err = DeleteImage(testImage) - if err == nil { - t.Errorf("should have failed to delete non existing image") - return - } - fmt.Println("correctly failed to delete non existing image") -} - -var testExternalNetwork = "external-network-shared" - -func TestGetExternalGateway(t *testing.T) { - if mexTestInfra2 == "" { - return - } - eg, err := GetExternalGateway(testExternalNetwork) - if err != nil { - t.Errorf("can't get external gateway for %s, %v", testExternalNetwork, err) - return - } - - fmt.Println("external gateway for", testExternalNetwork, eg) -} - -func TestSetServerProperty(t *testing.T) { - if mexTestInfra2 == "" { - return - } - nm := os.Getenv("MEX_TEST_MN") - - err := SetServerProperty(nm, "mex-flavor=x1.medium") - if err != nil { - t.Error(err) - } -} diff --git a/openstack-prov/oscliapi/userdata.txt b/openstack-prov/oscliapi/userdata.txt deleted file mode 100644 index b2c444aae..000000000 --- a/openstack-prov/oscliapi/userdata.txt +++ /dev/null @@ -1,13 +0,0 @@ -#cloud-config -bootcmd: - - echo MOBILEDGEX CLOUD CONFIG START -#hostname: master-node-1 -password: mobiledgex -ssh_authorized_keys: - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDZiZ16uwmHOuafD6a9AmZ5kYF9LqtfyrUOIVMF1eoRJCMALQrWbzNz/NOnqi5h5dwhPn+49oWMU16BKDkEgDik2jgNUOSZ69oZM4/ovPsB8yL55qdNBTx32kov5O8NkSwMEDter2mAPi9czCEv18MRC1qkiZCUxmfFs0BBgXtNfE42Utr97YcKFtvutLDGA1hoFVjon0Yk7wSMNZfwkBznVoShRISCzMvG5uVtf6miJwIIA9+SiwA/aa2OjCRQaiPCKJrPzHMcuLg4oZcs0ltd1CaIVLtMGaqpEoIvDumXEpuk0TSBJwxWUDAEgO5ILmVxi2fSLKa0yuLala6bcfwJ stack@sv1.mobiledgex.com -chpasswd: { expire: False } -ssh_pwauth: True -timezone: US/Pacific -runcmd: - - [ echo, MOBILEDGEX, doing, ifconfig ] - - [ ifconfig, -a ] diff --git a/openstack-tenant/agent/README.md b/openstack-tenant/agent/README.md index cf6eb7ce0..7bcce4795 100644 --- a/openstack-tenant/agent/README.md +++ b/openstack-tenant/agent/README.md @@ -27,7 +27,10 @@ TODO: establish a private docker repo. Automate push and pull and relaunch. The agent runs on a node with sufficient access to the Internet and the internal network in which kubernetes cluster and containers are hosted. The openstack network has to be setup properly for this. An instance of openstack router has to be created and directed to route between the public and private network. Typically a private network is created for the kubernetes cluster which is isolated from the internet. The traffic from the private network has to be NAT'ed and routed via a router. -The agent is deployed as a docker container image. Pull the image on the agent node: +The agent is deployed as a docker container image. *NOTE THAT WE NO LONGER DEPLOY AGENT AS DOCKER. WE USE SERVICES. SEE BELOW* + +Pull the image on the agent node: + ``` docker pull mobiledgex/mexosagent diff --git a/mexosagent.service b/openstack-tenant/agent/mexosagent.service similarity index 83% rename from mexosagent.service rename to openstack-tenant/agent/mexosagent.service index c74398535..bd85e7064 100644 --- a/mexosagent.service +++ b/openstack-tenant/agent/mexosagent.service @@ -4,7 +4,7 @@ Wants=network.target After=network.target [Service] -ExecStart=/bin/bash -ce "exec /usr/local/bin/mexosagent -debug -cert /root >> /root/mexosagent.log 2>&1" +ExecStart=/bin/bash -ce "exec /usr/local/bin/mexosagent -debug -cert /home/ubuntu >> /var/log/mexosagent.log 2>&1" WorkingDirectory=/root Restart=always diff --git a/openstack-tenant/agent/server/handlers.go b/openstack-tenant/agent/server/handlers.go index aed975baf..a07252a47 100644 --- a/openstack-tenant/agent/server/handlers.go +++ b/openstack-tenant/agent/server/handlers.go @@ -391,7 +391,6 @@ func CreateNginx(name string, ports []*api.NginxPort) error { if !fileExists(pwd + "/key.pem") { log.Debugln("key.pem does not exist") return fmt.Errorf("while creating nginx %s, key.pem does not exist", name) - return err } errlogFile := dir + "/err.log" f, err := os.Create(errlogFile) diff --git a/openstack-tenant/agent/server/server.go b/openstack-tenant/agent/server/server.go index e1723f07b..e411e4e47 100644 --- a/openstack-tenant/agent/server/server.go +++ b/openstack-tenant/agent/server/server.go @@ -52,5 +52,5 @@ func ListenAndServeREST(restAddress, grpcAddress string) error { return fmt.Errorf("could not register REST service: %s", err) } - return http.ListenAndServe(restAddress, mux) + return http.ListenAndServe(restAddress, mux) //TODO TLS } diff --git a/openstack-tenant/packer/packer_template.mobiledgex.json b/openstack-tenant/packer/packer_template.mobiledgex.json index 455e459cf..a3ccee9e7 100644 --- a/openstack-tenant/packer/packer_template.mobiledgex.json +++ b/openstack-tenant/packer/packer_template.mobiledgex.json @@ -1,3 +1,4 @@ + { "builders": [{ "type": "openstack", @@ -5,8 +6,8 @@ "ssh_username": "ubuntu", "region": "RegionOne", "image_name": "mobiledgex", - "source_image": "6a7888da-2112-4660-8d5a-8c06ab845d52", - "networks": "f99dfbb9-161c-4929-b0de-2eb00e765725", + "source_image": "a4da5673-bfdb-423f-9020-9e072b851b39", + "networks": "d5d7d31c-721b-418e-be85-b1dad361f195", "security_groups": ["default" ] }], "provisioners": [{ @@ -14,3 +15,4 @@ "script": "setup.sh" }] } + diff --git a/openstack-tenant/packer/setup.sh b/openstack-tenant/packer/setup.sh index 513e45bc2..1fa9ec9cc 100644 --- a/openstack-tenant/packer/setup.sh +++ b/openstack-tenant/packer/setup.sh @@ -1,51 +1,64 @@ -whoami -pwd +sudo mkdir -p /etc/mobiledgex +sudo chmod 700 /etc/mobiledgex +echo starting setup.sh | sudo tee -a /etc/mobiledgex/creation_log.txt +pwd | sudo tee -a /etc/mobiledgex/creation_log.txt echo 127.0.1.1 `hostname` | sudo tee -a /etc/hosts -cat /etc/hosts +cat /etc/hosts | sudo tee -a /etc/mobiledgex/creation_log.txt echo nameserver 1.1.1.1 | sudo tee -a /etc/resolv.conf -cat /etc/resolv.conf -sudo dhclient ens3 -ip a -ip r +cat /etc/resolv.conf | sudo tee -a /etc/mobiledgex/creation_log.txt +sudo dhclient ens3 | sudo tee -a /etc/mobiledgex/creation_log.txt +ip a | sudo tee -a /etc/mobiledgex/creation_log.txt +ip r | sudo tee -a /etc/mobiledgex/creation_log.txt sudo apt-get update -sudo apt-get install -y jq +sudo apt-get install -y jq ipvsadm +sudo curl -s -o /etc/mobiledgex/holepunch https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/holepunch +sudo curl -s -o /etc/mobiledgex/holepunch.json https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/holepunch.json +sudo chmod a+rx /etc/mobiledgex/holepunch +sudo chmod a+r /etc/mobiledgex/holepunch.json sudo curl -s -o /usr/local/bin/mobiledgex-init.sh https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/mobiledgex-init.sh sudo chmod a+rx /usr/local/bin/mobiledgex-init.sh +echo copied mobiledgex-init.sh | sudo tee -a /etc/mobiledgex/creation_log.txt sudo curl -s -o /etc/systemd/system/mobiledgex.service https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/mobiledgex.service +echo copied mobiledgex.serivce | sudo tee -a /etc/mobiledgex/creation_log.txt sudo chmod a+rx /etc/systemd/system/mobiledgex.service sudo systemctl enable mobiledgex -#sudo mkdir -p /root/.ssh -#sudo ls -al /root -#sudo ls -al /root/.ssh +echo enabled mobiledgex service | sudo tee -a /etc/mobiledgex/creation_log.txt sudo curl -s -o /tmp/id_rsa_mex.pub https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/id_rsa_mex.pub +sudo curl -s -o /etc/mobiledgex/id_rsa_mex https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/id_rsa_mex +sudo chmod 600 /etc/mobiledgex/id_rsa_mex +sudo cp /etc/mobiledgex/id_rsa_mex /root/id_rsa_mex +sudo chmod 600 /root/id_rsa_mex sudo curl -s -o /tmp/id_rsa_mobiledgex.pub https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/id_rsa_mobiledgex.pub -sudo cat /tmp/id_rsa_mex.pub /tmp/id_rsa_mobiledgex | sudo tee /root/.ssh/authorized_keys +sudo cat /tmp/id_rsa_mex.pub /tmp/id_rsa_mobiledgex.pub | sudo tee /root/.ssh/authorized_keys sudo chmod 700 /root/.ssh sudo chmod 600 /root/.ssh/authorized_keys sudo curl -s -o /root/.ssh/config https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/ssh.config sudo chmod 600 /root/.ssh/config sudo rm /root/.ssh/known_hosts -#sudo cat /tmp/id_rsa_mex.pub | sudo tee -a ~ubuntu/.ssh/authorized_keys -#sudo ls -alR ~ubuntu/ -sudo curl -s -o /root/install-k8s-base.sh https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/install-k8s-base.sh -sudo chmod a+rx /root/install-k8s-base.sh -sudo curl -s -o /root/install-k8s-master.sh https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/install-k8s-master.sh -sudo chmod a+rx /root/install-k8s-master.sh -sudo curl -s -o /root/install-k8s-node.sh https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/install-k8s-node.sh -sudo chmod a+rx /root/install-k8s-node.sh -#sudo ls -alR /root -#sudo sed -e 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config > /tmp/xxx -#sudo mv /tmp/xxx /etc/ssh/sshd_config +echo set up ssh | sudo tee -a /etc/mobiledgex/creation_log.txt +sudo curl -s -o /etc/mobiledgex/install-k8s-base.sh https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/install-k8s-base.sh +sudo chmod a+rx /etc/mobiledgex/install-k8s-base.sh +sudo curl -s -o /etc/mobiledgex/install-k8s-master.sh https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/install-k8s-master.sh +sudo chmod a+rx /etc/mobiledgex/install-k8s-master.sh +sudo curl -s -o /etc/mobiledgex/install-k8s-node.sh https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/install-k8s-node.sh +sudo chmod a+rx /etc/mobiledgex/install-k8s-node.sh +echo copied k8s install scripts | sudo tee -a /etc/mobiledgex/creation_log.txt echo root:sandhill | sudo chpasswd -#sudo useradd -m -s /bin/bash mobiledgex -#sudo mkdir -p /home/mobiledgex/.ssh -#sudo chown mobiledgex /home/mobiledgex -#sudo chown mobiledgex /home/mobiledgex/.ssh -#sudo usermod -aG sudo mobiledgex -#echo mobiledgex:sandhill | sudo chpasswd -#sudo mkdir -p /home/mobiledgex/.ssh -#sudo cat /tmp/id_rsa_mex.pub | sudo tee -a ~mobiledgex/.ssh/authorized_keys -#sudo chown mobiledgex ~mobiledgex/.ssh/authorized_keys -#echo 'mobiledgex ALL=(ALL:ALL) NOPASSWD:ALL' | sudo tee -a /etc/sudoers -#sudo cat /etc/ssh/sshd_config -echo created at `date` | sudo tee -a /root/creation_date.txt +echo set root passwd | sudo tee -a /etc/mobiledgex/creation_log.txt +echo starting install of k8s base | sudo tee -a /etc/mobiledgex/creation_log.txt +sudo sh -x /etc/mobiledgex/install-k8s-base.sh | sudo tee -a /etc/mobiledgex/creation_log.txt +sudo chmod a+rw /var/run/docker/sock +sudo groupadd docker +sudo usermod -aG docker root +echo installed k8s base | sudo tee -a /etc/mobiledgex/creation_log.txt +#curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose +sudo curl https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/docker-compose -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +echo installed docker-compose | sudo tee -a /etc/mobiledgex/creation_log.txt +#curl -s -o /tmp/helm.tar.gz https://storage.googleapis.com/kubernetes-helm/helm-v2.11.0-linux-amd64.tar.gz +sudo curl -s -o /tmp/helm.tar.gz https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/helm-v2.11.0.tar.gz +sudo tar xvf /tmp/helm.tar.gz +sudo mv linux-amd64/helm /usr/local/bin/ +sudo chmod a+rx /usr/local/bin/helm +echo installed helm | sudo tee -a /etc/mobiledgex/creation_log.txt +echo created at `date` | sudo tee -a /etc/mobiledgex/creation_log.txt diff --git a/openstack-tenant/qcow2/mobiledgex-16.04-qcow2/mobiledgex-init.sh b/openstack-tenant/qcow2/mobiledgex-16.04-qcow2/mobiledgex-init.sh index eb89d687c..79a5a9bca 100755 --- a/openstack-tenant/qcow2/mobiledgex-16.04-qcow2/mobiledgex-init.sh +++ b/openstack-tenant/qcow2/mobiledgex-16.04-qcow2/mobiledgex-init.sh @@ -1,82 +1,108 @@ #!/bin/bash # this is run at system init time # TODO: mark so that it does not run again -# TODO: check for updates from edgeproxy before running - -echo starting mobiledgex init >> /tmp/mobiledgex.log -date >> /tmp/mobiledgex.log +set -x +echo starting mobiledgex init +date MCONF=/mnt/mobiledgex-config -mkdir $MCONF +mkdir -p $MCONF mount `blkid -t LABEL="config-2" -odevice` $MCONF +hostname `cat $MCONF/openstack/latest/meta_data.json |jq .name | sed -e 's/"//'g` +echo hostname `hostname` | tee -a /var/log/mobiledgex.log +grep -v 127.0.1.1 /etc/hosts | tee /tmp/hosts +echo 127.0.1.1 `hostname` | tee -a /tmp/hosts +cp /tmp/hosts /etc/hosts +# TODO destory previous holepunch. Send signal to the server side +holepunch=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.holepunch | sed -e 's/"//'g` +cat /etc/mobiledgex/holepunch.json | sed -e "s/22222/$holepunch/" | tee /tmp/holepunch.json +mv /tmp/holepunch.json /etc/mobiledgex/holepunch.json +cd /etc/mobiledgex; /etc/mobiledgex/holepunch write-systemd-file +systemctl enable holepunch +systemctl start holepunch +systemctl status holepunch +usermod -aG docker ubuntu +chmod a+rw /var/run/docker.sock +update=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.update | sed -e 's/"//'g` skipinit=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.skipinit | sed -e 's/"//'g` if [ "$skipinit" != "yes" ]; then - echo mobiledgex initialization >> /tmp/mobiledgex.log - ipaddress=`cat $MCONF/openstack/latest/network_data.json |jq .networks[0].ip_address | sed -e 's/"//'g` - nettype=`cat $MCONF/openstack/latest/network_data.json |jq .networks[0].type | sed -e 's/"//'g` - if [ "$nettype" = "ipv4_dhcp" ]; then - echo using dhcp >> /tmp/mobiledgex.log - dhclient ens3 - else - ifconfig ens3 $ipaddress up - ifconfig ens3 netmask `cat $MCONF/openstack/latest/network_data.json |jq .networks[0].netmask | sed -e 's/"//'g` up - edgeproxy=`cat $MCONF/openstack/latest/meta_data.json| jq .meta.edgeproxy | sed -e 's/"//'g` - ip route add default via $edgeproxy dev ens3 - fi - privatenet=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.privatenet | sed -e 's/"//'g` - privaterouter=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.privaterouter | sed -e 's/"//'g` - if [ "$privatenet" != "" -a "$privaterouter" != "" ]; then - echo private route entry $privatenet via $privaterouter >> /tmp/mobiledgex.log - ip route add $privatenet via $privaterouter dev ens3 - fi - ifconfig -a >>/tmp/mobiledgex.log - ip route >> /tmp/mobiledgex.log - hostname `cat $MCONF/openstack/latest/meta_data.json |jq .name | sed -e 's/"//'g` - echo hostname `hostname` >> /tmp/mobiledgex.log - role=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.role | sed -e 's/"//'g` - echo role $role >> /tmp/mobiledgex.log - grep -v 127.0.1.1 /etc/hosts > /tmp/hosts - echo 127.0.1.1 `hostname` >> /tmp/hosts - cp /tmp/hosts /etc/hosts - dig google.com| grep 'status: NOERROR' + echo mobiledgex initialization | tee -a /var/log/mobiledgex.log + ipaddress=`cat $MCONF/openstack/latest/network_data.json |jq .networks[0].ip_address | sed -e 's/"//'g` + nettype=`cat $MCONF/openstack/latest/network_data.json |jq .networks[0].type | sed -e 's/"//'g` + if [ "$nettype" = "ipv4_dhcp" ]; then + echo using dhcp | tee -a /var/log/mobiledgex.log + dhclient ens3 + else + ifconfig ens3 $ipaddress up + ifconfig ens3 netmask `cat $MCONF/openstack/latest/network_data.json |jq .networks[0].netmask | sed -e 's/"//'g` up + edgeproxy=`cat $MCONF/openstack/latest/meta_data.json| jq .meta.edgeproxy | sed -e 's/"//'g` + ip route add default via $edgeproxy dev ens3 + fi + privatenet=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.privatenet | sed -e 's/"//'g` + privaterouter=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.privaterouter | sed -e 's/"//'g` + if [ "$privatenet" != "" -a "$privaterouter" != "" ]; then + echo private route entry $privatenet via $privaterouter | tee -a /var/log/mobiledgex.log + ip route add $privatenet via $privaterouter dev ens3 + fi + ifconfig -a | tee -a /var/log/mobiledgex.log + ip route | tee -a /var/log/mobiledgex.log + role=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.role | sed -e 's/"//'g` + if [ "$role" = "" ]; then + echo warning role is empty string + else + echo role $role | tee -a /var/log/mobiledgex.log + fi + dig google.com| grep 'status: NOERROR' + if [ $? -ne 0 ]; then + echo add 1.1.1.1 as nameserver | tee -a /var/log/mobiledgex.log + echo nameserver 1.1.1.1 | tee /etc/resolv.conf + fi + echo set name server to 1.1.1.1 | tee -a /var/log/mobiledgex.log if [ $? -ne 0 ]; then - echo add 1.1.1.1 as nameserver >> /tmp/mobiledgex.log - echo nameserver 1.1.1.1 > /etc/resolv.conf + echo jq not found | tee -a /var/log/mobiledgex.log + exit 1 + fi + if [ "$update" != "" ]; then + echo doing update via $update | tee -a /var/log/mobiledgex.log + curl -s -o /etc/mobiledgex/update.sh https://mobiledgex:sandhill@registry.mobiledgex.net:8000/mobiledgex/update/$update/update.sh + chmod a+rx /etc/mobiledgex/update.sh + sh -x /etc/mobiledgex/update.sh | tee -a /var/log/mobiledgex.log fi skipk8s=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.skipk8s | sed -e 's/"//'g` if [ "$role" = "mex-agent-node" ]; then - echo "initializing mex agent node" >> /tmp/mobiledgex.log - /root/install-k8s-base.sh >> /tmp/mobiledgex.log - chmod a+rw /var/run/docker.sock - curl -s -o /tmp/helm.tar.gz https://storage.googleapis.com/kubernetes-helm/helm-v2.11.0-linux-amd64.tar.gz - tar xvf /tmp/helm.tar.gz - mv linux-arm64/helm /usr/local/bin/ - chmod a+rx /usr/local/bin/helm + echo "initializing mex agent node" | tee -a /var/log/mobiledgex.log else if [ "$skipk8s" != "yes" ]; then - echo doing k8s init >> /tmp/mobiledgex.log - /root/install-k8s-base.sh >> /tmp/mobiledgex.log - echo k8s-base installed >> /tmp/mobiledgex.log + echo skip-k8s is not set to yes so doing k8s init | tee -a /var/log/mobiledgex.log masteraddr=`cat $MCONF/openstack/latest/meta_data.json |jq .meta.k8smaster | sed -e 's/"//'g` if [ "$role" = "k8s-master" ]; then - echo k8s-master init >> /tmp/mobiledgex.log - /root/install-k8s-master.sh ens3 $masteraddr $ipaddress >> /tmp/mobiledgex.log - echo k8s-master installed >> /tmp/mobiledgex.log + echo k8s-master init | tee -a /var/log/mobiledgex.log + sh -x /etc/mobiledgex/install-k8s-master.sh ens3 $masteraddr $ipaddress | tee -a /var/log/mobiledgex.log + if [ $? -ne 0 ]; then + echo install k8s master failed with error | tee -a /var/log/mobiledgex.log + exit 1 + fi + echo k8s-master installed | tee -a /var/log/mobiledgex.log elif [ "$role" = "k8s-node" ]; then - echo k8s-node init >> /tmp/mobiledgex.log - /root/install-k8s-node.sh ens3 $masteraddr $ipaddress >> /tmp/mobiledgex.log - echo k8s-node installed >> /tmp/mobiledgex.log + echo k8s-node init | tee -a /var/log/mobiledgex.log + sh -x /etc/mobiledgex/install-k8s-node.sh ens3 $masteraddr $ipaddress | tee -a /var/log/mobiledgex.log + if [ $? -ne 0 ]; then + echo install k8s node failed with error | tee -a /var/log/mobiledgex.log + exit 1 + fi + echo k8s-node installed | tee -a /var/log/mobiledgex.log else - echo "role is " $role " and not k8s" >> /tmp/mobiledgex.log + echo error not k8s master and not k8s node | tee -a /var/log/mobiledgex.log + echo "role is " $role " and not k8s" | tee -a /var/log/mobiledgex.log fi - echo done k8s init for role $role >> /tmp/mobiledgex.log + echo finished k8s init for role $role | tee -a /var/log/mobiledgex.log else - echo skipping k8s init for role $role >> /tmp/mobiledgex.log + echo skipping k8s init for role $role | tee -a /var/log/mobiledgex.log fi fi - echo $role > /etc/mobiledgex-role.txt - echo done mobiledgex init >> /tmp/mobiledgex.log + echo $role | tee /etc/mobiledgex/role.txt + echo finished mobiledgex init | tee -a /var/log/mobiledgex.log else - echo skip mobiledgex init >> /tmp/mobiledgex.log + echo skipping mobiledgex init as told | tee -a /var/log/mobiledgex.log fi -echo all done exiting >> /tmp/mobiledgex.log -date >> /tmp/mobiledgex.log +echo all done exiting | tee -a /var/log/mobiledgex.log +date | tee -a /var/log/mobiledgex.log