Skip to content

Commit

Permalink
part of issue 373 (#29)
Browse files Browse the repository at this point in the history 
* nginx rev proxy for kubectl proxy to plug security holes, wip

* htpasswd location

* certificate,key location

* mexctl handle kubectl and bypass kubectl proxy complications
  • Loading branch information
@bobmex
bobmex authored Jan 27, 2019
1 parent a02eb0c commit 44bd29a
Show file tree
Hide file tree
Showing 25 changed files with 744 additions and 342 deletions.
7 changes: 6 additions & 1 deletion mexos/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,17 @@ func RunMEXAgentManifest(mf *Manifest) error {
return fmt.Errorf("can't acquire certificate for %s, %v", rootLB.Name, err)
}
log.DebugLog(log.DebugLevelMexos, "acquired certificates from letsencrypt", "name", rootLB.Name)
err = GetHTPassword(mf, rootLB)
if err != nil {
return fmt.Errorf("can't download htpassword %v", err)
}
//return RunMEXOSAgentContainer(mf, rootLB)
return RunMEXOSAgentService(mf, rootLB)
}

func RunMEXOSAgentService(mf *Manifest, rootLB *MEXRootLB) error {
//TODO check if agent is running before restarting again.
log.DebugLog(log.DebugLevelMexos, "will run new mexosagent service")
log.DebugLog(log.DebugLevelMexos, "run mexosagent service")
client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser)
if err != nil {
return err
Expand All @@ -104,6 +108,7 @@ func RunMEXOSAgentService(mf *Manifest, rootLB *MEXRootLB) error {
}
}
log.DebugLog(log.DebugLevelMexos, "copying new mexosagent service")
//TODO name should come from mf.Values and allow versioning
for _, dest := range []struct{ path, name string }{
{"/usr/local/bin", "mexosagent"},
{"/lib/systemd/system", "mexosagent.service"},
Expand Down
55 changes: 19 additions & 36 deletions mexos/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package mexos
import (
"encoding/json"
"fmt"
"os"
"strings"
"time"

Expand Down Expand Up @@ -73,7 +72,7 @@ type ClusterMasterFlavor struct {
//mexCreateClusterKubernetes creates a cluster of nodes. It can take a while, so call from a goroutine.
func mexCreateClusterKubernetes(mf *Manifest) error {
//func mexCreateClusterKubernetes(mf *Manifest) (*string, error) {
log.DebugLog(log.DebugLevelMexos, "create kubernetes cluster", "cluster metadata", mf.Metadata, "spec", mf.Spec)
//log.DebugLog(log.DebugLevelMexos, "create kubernetes cluster", "cluster metadata", mf.Metadata, "spec", mf.Spec)
rootLB, err := getRootLB(mf.Spec.RootLB)
if err != nil {
return err
Expand Down Expand Up @@ -215,7 +214,11 @@ func mexDeleteClusterKubernetes(mf *Manifest) error {
if err != nil {
return err
}
log.DebugLog(log.DebugLevelMexos, "looking for server", "name", name, "servers", srvs)
// clname, err := FindClusterWithKey(mf, mf.Spec.Key)
// if err != nil {
// return fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err)
// }
//log.DebugLog(log.DebugLevelMexos, "looking for server", "name", name, "servers", srvs)
force := strings.Contains(mf.Spec.Flags, "force")
serverDeleted := false
for _, s := range srvs {
Expand Down Expand Up @@ -245,11 +248,11 @@ func mexDeleteClusterKubernetes(mf *Manifest) error {
}
serverDeleted = true
//kconfname := fmt.Sprintf("%s.kubeconfig", s.Name[strings.LastIndex(s.Name, "-")+1:])
kconfname := GetLocalKconfName(mf)
rerr := os.Remove(kconfname)
if rerr != nil {
log.DebugLog(log.DebugLevelMexos, "error can't remove file", "name", kconfname, "error", rerr)
}
// kconfname := GetLocalKconfName(mf)
// rerr := os.Remove(kconfname)
// if rerr != nil {
// log.DebugLog(log.DebugLevelMexos, "error can't remove file", "name", kconfname, "error", rerr)
// }
}
}
if !serverDeleted {
Expand All @@ -271,37 +274,17 @@ func mexDeleteClusterKubernetes(mf *Manifest) error {
}
err = DeleteSubnet(mf, s.Name)
if err != nil {
log.DebugLog(log.DebugLevelMexos, "warning, error while deleting subnet", "error", err)
log.DebugLog(log.DebugLevelMexos, "warning, problems deleting subnet", "error", err)
}
break
}
}
//XXX tell agent to remove the route
//XXX remove kubectl proxy instance
client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser)
if err != nil {
return err
}
cmd := "sudo ps wwh -C kubectl -o pid,args"
out, err := client.Output(cmd)
if err != nil || out == "" {
return nil // no kubectl running
}
lines := strings.Split(out, "\n")
for _, ln := range lines {
pidnum := parseKCPid(ln, mf.Spec.Key)
if pidnum == 0 {
continue
}
cmd = fmt.Sprintf("sudo kill -9 %d", pidnum)
out, err = client.Output(cmd)
if err != nil {
log.InfoLog("error killing kubectl proxy", "command", cmd, "out", out, "error", err)
} else {
log.DebugLog(log.DebugLevelMexos, "killed kubectl proxy", "line", ln, "cmd", cmd)
}
return nil
}
// if err = DeleteNginxKCProxy(mf, rootLB.Name, clname); err != nil {
// log.DebugLog(log.DebugLevelMexos, "warning,cannot clean nginx kubectl proxy", "error", err)
// //return err
// }
return nil
}

Expand Down Expand Up @@ -333,7 +316,7 @@ func IsClusterReady(mf *Manifest, rootLB *MEXRootLB) (bool, error) {
}
log.DebugLog(log.DebugLevelMexos, "checking master k8s node for available nodes", "ipaddr", ipaddr)
cmd := fmt.Sprintf("ssh -o %s -o %s -o %s -i id_rsa_mex %s@%s kubectl get nodes -o json", sshOpts[0], sshOpts[1], sshOpts[2], sshUser, ipaddr)
log.DebugLog(log.DebugLevelMexos, "running kubectl get nodes", "cmd", cmd)
//log.DebugLog(log.DebugLevelMexos, "running kubectl get nodes", "cmd", cmd)
out, err := client.Output(cmd)
if err != nil {
log.DebugLog(log.DebugLevelMexos, "error checking for kubernetes nodes", "out", out, "err", err)
Expand Down Expand Up @@ -361,7 +344,7 @@ func IsClusterReady(mf *Manifest, rootLB *MEXRootLB) (bool, error) {

//FindClusterWithKey finds cluster given a key string
func FindClusterWithKey(mf *Manifest, key string) (string, error) {
log.DebugLog(log.DebugLevelMexos, "find cluster with key", "key", key)
//log.DebugLog(log.DebugLevelMexos, "find cluster with key", "key", key)
if key == "" {
return "", fmt.Errorf("empty key")
}
Expand All @@ -371,7 +354,7 @@ func FindClusterWithKey(mf *Manifest, key string) (string, error) {
}
for _, s := range srvs {
if s.Status == "ACTIVE" && strings.HasSuffix(s.Name, key) && strings.HasPrefix(s.Name, "mex-k8s-master") {
log.DebugLog(log.DebugLevelMexos, "find cluster with key", "key", key, "found", s.Name)
//log.DebugLog(log.DebugLevelMexos, "find cluster with key", "key", key, "found", s.Name)
return s.Name, nil
}
}
Expand Down
2 changes: 1 addition & 1 deletion mexos/flavor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func GetClusterFlavor(flavor string) (*ClusterFlavor, error) {
log.DebugLog(log.DebugLevelMexos, "get cluster flavor details", "cluster flavor", flavor)
for _, af := range AvailableClusterFlavors {
if af.Name == flavor {
log.DebugLog(log.DebugLevelMexos, "using cluster flavor", "cluster flavor", af)
//log.DebugLog(log.DebugLevelMexos, "using cluster flavor", "cluster flavor", af)
return af, nil
}
}
Expand Down
8 changes: 4 additions & 4 deletions mexos/ip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func GetAllowedClientCIDR() string {
//GetServerIPAddr gets the server IP
func GetServerIPAddr(mf *Manifest, networkName, serverName string) (string, error) {
//TODO: mexosagent cache
log.DebugLog(log.DebugLevelMexos, "get server ip addr", "networkname", networkName, "servername", serverName)
//log.DebugLog(log.DebugLevelMexos, "get server ip addr", "networkname", networkName, "servername", serverName)
//sd, err := GetServerDetails(rootLB)
sd, err := GetServerDetails(mf, serverName)
if err != nil {
Expand Down Expand Up @@ -76,13 +76,13 @@ func GetServerIPAddr(mf *Manifest, networkName, serverName string) (string, erro
return "", fmt.Errorf("invalid network name in server detail address, %s", sd.Addresses)
}
addr := its[1]
log.DebugLog(log.DebugLevelMexos, "got server ip addr", "ipaddr", addr, "netname", networkName, "servername", serverName)
//log.DebugLog(log.DebugLevelMexos, "got server ip addr", "ipaddr", addr, "netname", networkName, "servername", serverName)
return addr, nil
}

//FindNodeIP finds IP for the given node
func FindNodeIP(mf *Manifest, name string) (string, error) {
log.DebugLog(log.DebugLevelMexos, "find node ip", "name", name)
//log.DebugLog(log.DebugLevelMexos, "find node ip", "name", name)
if name == "" {
return "", fmt.Errorf("empty name")
}
Expand All @@ -96,7 +96,7 @@ func FindNodeIP(mf *Manifest, name string) (string, error) {
if err != nil {
return "", fmt.Errorf("can't get IP for %s, %v", s.Name, err)
}
log.DebugLog(log.DebugLevelMexos, "found node ip", "name", name, "ipaddr", ipaddr)
//log.DebugLog(log.DebugLevelMexos, "found node ip", "name", name, "ipaddr", ipaddr)
return ipaddr, nil
}
}
Expand Down
15 changes: 9 additions & 6 deletions mexos/kubeconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,13 @@ func CopyKubeConfig(mf *Manifest, rootLB *MEXRootLB, name string) error {
if err != nil {
return fmt.Errorf("can't cat %s, %s, %v", kconfname, out, err)
}
port, serr := StartKubectlProxy(mf, rootLB, kconfname)
if serr != nil {
return serr
}
return ProcessKubeconfig(mf, rootLB, name, port, []byte(out))
//TODO generate per proxy password and record in vault
//port, serr := StartKubectlProxy(mf, rootLB, name, kconfname)
//if serr != nil {
// return serr
//}
//return ProcessKubeconfig(mf, rootLB, name, port, []byte(out))
return nil
}

//ProcessKubeconfig validates kubeconfig and saves it and creates a copy for proxy access
Expand All @@ -148,7 +150,8 @@ func ProcessKubeconfig(mf *Manifest, rootLB *MEXRootLB, name string, port int, d
}
kconfname := GetLocalKconfName(mf)
log.DebugLog(log.DebugLevelMexos, "writing local kubeconfig file", "name", kconfname)
kc.Clusters[0].Cluster.Server = fmt.Sprintf("http://%s:%d", rootLB.Name, port)
//TODO per cluster password has to come from vault
kc.Clusters[0].Cluster.Server = fmt.Sprintf("https://testuser314159:testpassword271828@%s:%d", rootLB.Name, port)
dat, err = yaml.Marshal(kc)
if err != nil {
return fmt.Errorf("can't marshal kubeconfig proxy edit %s, %v", name, err)
Expand Down
29 changes: 27 additions & 2 deletions mexos/kubectl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,31 @@ import (
"github.com/mobiledgex/edge-cloud/log"
)

func RunKubectl(mf *Manifest, params string) (*string, error) {
log.DebugLog(log.DebugLevelMexos, "run kubectl", "params", params)
rootLB, err := getRootLB(mf.Spec.RootLB)
if err != nil {
return nil, err
}
if rootLB == nil {
return nil, fmt.Errorf("failed to create docker registry secret, rootLB is null")
}
//name, err := FindClusterWithKey(mf, mf.Spec.Key)
//if err != nil {
// return nil, fmt.Errorf("can't find cluster with key %s, %v", mf.Spec.Key, err)
//}
client, err := GetSSHClient(mf, rootLB.Name, mf.Values.Network.External, sshUser)
if err != nil {
return nil, fmt.Errorf("can't get ssh client, %v", err)
}
cmd := fmt.Sprintf("kubectl --kubeconfig %s.kubeconfig %s", rootLB.Name, params)
out, err := client.Output(cmd)
if err != nil {
return nil, fmt.Errorf("kubectl failed, %v, %s", err, out)
}
return &out, nil
}

func CreateDockerRegistrySecret(mf *Manifest) error {
log.DebugLog(log.DebugLevelMexos, "creating docker registry secret in kubernetes")
rootLB, err := getRootLB(mf.Spec.RootLB)
Expand All @@ -25,8 +50,8 @@ func CreateDockerRegistrySecret(mf *Manifest) error {
}

var out string
log.DebugLog(log.DebugLevelMexos, "CreateDockerRegistrySecret", "mf", mf)

//log.DebugLog(log.DebugLevelMexos, "CreateDockerRegistrySecret", "mf", mf)
log.DebugLog(log.DebugLevelMexos, "creating docker registry secret in kubernetes cluster")
if IsLocalDIND(mf) || mf.Metadata.Operator == "gcp" || mf.Metadata.Operator == "azure" {
log.DebugLog(log.DebugLevelMexos, "CreateDockerRegistrySecret locally non OpenStack case")
var o []byte
Expand Down
91 changes: 47 additions & 44 deletions mexos/kubeproxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,18 @@ package mexos

import (
"fmt"
"strconv"
"strings"
"time"

"github.com/mobiledgex/edge-cloud/log"
)

var kcproxySuffix = "-kcproxy"

//StartKubectlProxy starts kubectl proxy on the rootLB to handle kubectl commands remotely.
// To be called after copying over the kubeconfig file from cluster to rootLB.
func StartKubectlProxy(mf *Manifest, rootLB *MEXRootLB, kubeconfig string) (int, error) {
log.DebugLog(log.DebugLevelMexos, "start kubectl proxy", "kubeconfig", kubeconfig)
func StartKubectlProxy(mf *Manifest, rootLB *MEXRootLB, name, kubeconfig string) (int, error) {
log.DebugLog(log.DebugLevelMexos, "start kubectl proxy", "name", name, "kubeconfig", kubeconfig)
if rootLB == nil {
return 0, fmt.Errorf("cannot kubectl proxy, rootLB is null")
}
Expand All @@ -22,52 +24,53 @@ func StartKubectlProxy(mf *Manifest, rootLB *MEXRootLB, kubeconfig string) (int,
if err != nil {
return 0, err
}
maxPort := 8000
cmd := "sudo ps wwh -C kubectl -o args"
//TODO check /home/ubuntu/.docker-pass file
cmd := fmt.Sprintf("echo %s | docker login -u mobiledgex --password-stdin %s", mexEnv(mf, "MEX_DOCKER_REG_PASS"), mf.Spec.DockerRegistry)
out, err := client.Output(cmd)
if err == nil && out != "" {
lines := strings.Split(out, "\n")
for _, ln := range lines {
portnum := parseKCPort(ln)
if portnum > maxPort {
maxPort = portnum
}
}
if err != nil {
return 0, fmt.Errorf("can't docker login, %s, %v", out, err)
}
maxPort++
log.DebugLog(log.DebugLevelMexos, "port for kubectl proxy", "maxport", maxPort)
cmd = fmt.Sprintf("kubectl proxy --port %d --accept-hosts='.*' --address='0.0.0.0' --kubeconfig=%s ", maxPort, kubeconfig)
//Use .Start() because we don't want to hang
cl1, cl2, err := client.Start(cmd)
cmd = fmt.Sprintf("docker pull registry.mobiledgex.net:5000/mobiledgex/mobiledgex")
res, err := client.Output(cmd)
if err != nil {
return 0, fmt.Errorf("error running kubectl proxy, %s, %v", cmd, err)
return 0, fmt.Errorf("cannot pull mobiledgex image, %v, %s", err, res)
}
cl1.Close() //nolint
cl2.Close() //nolint
err = AddSecurityRuleCIDR(mf, GetAllowedClientCIDR(), "tcp", GetMEXSecurityRule(mf), maxPort)
log.DebugLog(log.DebugLevelMexos, "adding external ingress security rule for kubeproxy", "port", maxPort)
//TODO verify existence of kubeconfig file
containerName := name + kcproxySuffix
cmd = fmt.Sprintf("docker run --net host -d --rm -it -v /home:/home --name %s registry.mobiledgex.net:5000/mobiledgex/mobiledgex kubectl proxy --port 0 --accept-hosts '^127.0.0.1$' --address 127.0.0.1 --kubeconfig /home/ubuntu/%s", containerName, kubeconfig)
res, err = client.Output(cmd)
if err != nil {
log.DebugLog(log.DebugLevelMexos, "warning, error while adding external ingress security rule for kubeproxy", "error", err, "port", maxPort)
return 0, fmt.Errorf("error running kubectl proxy, %s, %v, %s", cmd, err, res)
}

cmd = "sudo ps wwh -C kubectl -o args"
for i := 0; i < 5; i++ {
//verify
out, outerr := client.Output(cmd)
if outerr == nil {
if out == "" {
continue
}
lines := strings.Split(out, "\n")
for _, ln := range lines {
if parseKCPort(ln) == maxPort {
log.DebugLog(log.DebugLevelMexos, "kubectl confirmed running with port", "port", maxPort)
}
}
return maxPort, nil
}
log.DebugLog(log.DebugLevelMexos, "waiting for kubectl proxy...")
time.Sleep(3 * time.Second)
res, err = client.Output(fmt.Sprintf("docker logs %s", containerName))
if err != nil {
return 0, fmt.Errorf("cannot get logs for container %s", containerName)
}
items := strings.Split(res, " ")
if len(items) < 5 {
return 0, fmt.Errorf("insufficient address info in log output, %s", res)
}
addr := items[4]
addr = strings.TrimSpace(addr)
items = strings.Split(addr, ":")
if len(items) < 2 {
return 0, fmt.Errorf("cannot get port from %s", addr)
}
port := items[1]
portnum, aerr := strconv.Atoi(port)
if aerr != nil {
return 0, fmt.Errorf("cannot convert port %v, %s", aerr, port)
}
log.DebugLog(log.DebugLevelMexos, "adding external ingress security rule for kubeproxy", "port", port)
err = AddSecurityRuleCIDR(mf, GetAllowedClientCIDR(), "tcp", GetMEXSecurityRule(mf), portnum+1) //XXX
if err != nil {
log.DebugLog(log.DebugLevelMexos, "warning, error while adding external ingress security rule for kubeproxy", "error", err, "port", port)
}
portnum++
//TODO delete security rule when kubectl proxy container deleted
if err := AddNginxKubectlProxy(mf, rootLB.Name, name, portnum); err != nil {
return 0, fmt.Errorf("cannot add nginx kubectl proxy, %v", err)
}
return 0, fmt.Errorf("timeout error verifying kubectl proxy")
log.DebugLog(log.DebugLevelMexos, "nginx kubectl proxy", "port", portnum)
return portnum, nil
}
1 change: 1 addition & 0 deletions mexos/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ func CreateKubernetesAppManifest(mf *Manifest, kubeManifest string) error {
//if err := CreateDockerRegistrySecret(mf); err != nil {
// return err
//}
//TODO do not create yaml file but use remote yaml file over https
cmd = fmt.Sprintf("cat <<'EOF'> %s.yaml \n%s\nEOF", mf.Metadata.Name, kubeManifest)
out, err := kp.client.Output(cmd)
if err != nil {
Expand Down
Loading

0 comments on commit 44bd29a

Please sign in to comment.