Use kind for setting up E2E clusters

.circleci/config.yml

@@ -99,26 +99,38 @@ jobs:
             chmod +x linux-amd64/helm
             sudo mv linux-amd64/helm /usr/local/bin/
       - run:
-          name: Deploy minikube
+          name: Setup Kubernetes
           environment:
-            CHANGE_MINIKUBE_NONE_USER: true
-            K8S_VERSION: v1.12.3
-            MINIKUBE_VERSION: v0.30.0
+            # See https://hub.docker.com/r/kindest/node/tags for available tags(k8s versions)
+            K8S_VERSION: v1.12.10
+            KIND_VERSION: v0.5.1
           command: |
             curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl
             chmod +x kubectl && sudo mv kubectl /usr/local/bin/
-            curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
-            chmod +x minikube && sudo mv minikube /usr/local/bin/
-            sudo -E minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION}
-            sudo -E minikube update-context
+
+            curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+            chmod +x ./kind
+            sudo mv ./kind /usr/local/bin/
+
+            kind create cluster --name minikube --image kindest/node:${K8S_VERSION}
+
+            KUBECONFIG="$(kind get kubeconfig-path --name="minikube")"
+
+            echo Copying ${KUBECONFIG} to ~/.kube/config so that it is available to the succeeding steps.
+
+            cp ${KUBECONFIG} ~/.kube/config
+
+            cat ${KUBECONFIG}
+
+            kubectl cluster-info
       - run:
           name: Wait for nodes to become ready
           command: JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done
       - run:
           name: Execute integration tests
           command: |
             export TERM=xterm
-            make integration
+            KUBECONTEXT=kubernetes-admin@minikube make integration
 
   integration_tests_helm3:
     machine:
@@ -144,26 +156,38 @@ jobs:
           chmod +x linux-amd64/helm
           sudo mv linux-amd64/helm /usr/local/bin/
     - run:
-        name: Deploy minikube
+        name: Setup Kubernetes
         environment:
-          CHANGE_MINIKUBE_NONE_USER: true
-          K8S_VERSION: v1.12.3
-          MINIKUBE_VERSION: v0.30.0
+          # See https://hub.docker.com/r/kindest/node/tags for available tags(k8s versions)
+          K8S_VERSION: v1.12.10
+          KIND_VERSION: v0.5.1
         command: |
           curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl
           chmod +x kubectl && sudo mv kubectl /usr/local/bin/
-          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
-          chmod +x minikube && sudo mv minikube /usr/local/bin/
-          sudo -E minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION}
-          sudo -E minikube update-context
+
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv ./kind /usr/local/bin/
+
+          kind create cluster --name minikube --image kindest/node:${K8S_VERSION}
+
+          KUBECONFIG="$(kind get kubeconfig-path --name="minikube")"
+
+          echo Copying ${KUBECONFIG} to ~/.kube/config so that it is available to the succeeding steps.
+
+          cp ${KUBECONFIG} ~/.kube/config
+
+          cat ${KUBECONFIG}
+
+          kubectl cluster-info
     - run:
         name: Wait for nodes to become ready
         command: JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done
     - run:
         name: Execute integration tests
         command: |
           export TERM=xterm
-          HELMFILE_HELM3=1 make integration
+          KUBECONTEXT=kubernetes-admin@minikube HELMFILE_HELM3=1 make integration
 
 # GITHUB_TOKEN env var must be setup in circleci console
 

test/integration/happypath.yaml

@@ -10,7 +10,7 @@ repositories:
     url: https://kubernetes-charts-incubator.storage.googleapis.com
 
 helmDefaults:
-  kubeContext: minikube
+  kubeContext: {{ requiredEnv "KUBECONTEXT" }}
 
 releases:
 

test/integration/lib/ensure.sh

@@ -2,15 +2,6 @@
 
 # Check environment is correctly setup
 
-if ! hash minikube 2>/dev/null; then
-    fail "Minikube needs to be installed."
-fi
-if [ ! $(minikube status --format '{{.MinikubeStatus}}') == "Running" ]; then
-    fail "Minikube is not running."
-fi
-if [ ! $(minikube status --format '{{.ClusterStatus}}') == "Running" ]; then
-    fail "Minikube Cluster is not running."
-fi
 if ! kubectl version --short 1> /dev/null; then
     fail "Could not connect to minikube apiserver"
 fi
@@ -26,3 +17,6 @@ fi
 if ! hash helm 1>/dev/null; then
     fail "Helm needs to be installed."
 fi
+if [ -z "${KUBECONTEXT}" ]; then
+  fail "KUBECONTEXT needs to be set."
+fi

test/integration/run.sh

@@ -14,8 +14,8 @@ if [[ ! -d "${dir}" ]]; then dir="${PWD}"; fi
 
 test_ns="helmfile-tests-$(date +"%Y%m%d-%H%M%S")"
 helmfile="./helmfile --namespace=${test_ns}"
-helm="helm --kube-context=minikube"
-kubectl="kubectl --context=minikube --namespace=${test_ns}"
+helm="helm --kube-context=${KUBECONTEXT}"
+kubectl="kubectl --context=${KUBECONTEXT} --namespace=${test_ns}"
 
 # FUNCTIONS ----------------------------------------------------------------------------------------------------------
 
@@ -48,10 +48,11 @@ info "Using namespace: ${test_ns}"
 # helm v2
 if helm version --client 1>/dev/null 2>/dev/null; then
   info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
+  kubectl create clusterrolebinding tiller-full-access --clusterrole=cluster-admin --serviceaccount=kube-system:default
   ${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true
 # helm v3
 else
-  info "Using Helm version: $(helm version --short | grep -o v.*$)"
+  info "Using Helm version: $(helm version --short | grep -o v.*$)" || true
 fi
 ${helm} plugin install https://github.com/databus23/helm-diff --version master
 ${kubectl} get namespace ${test_ns} &> /dev/null && warn "Namespace ${test_ns} exists, from a previous test run?"
@@ -80,8 +81,12 @@ code=$?
 info "Syncing ${dir}/happypath.yaml"
 ${helmfile} -f ${dir}/happypath.yaml sync
 wait_deploy_ready httpbin-httpbin
-retry 5 "curl --fail $(minikube service --url --namespace=${test_ns} httpbin-httpbin)/status/200"
+kubectl port-forward --namespace=${test_ns} svc/httpbin-httpbin 8000 &
+pid=$!
+sleep 5
+retry 5 "curl --fail http://localhost:8000/status/200"
 [ ${retry_result} -eq 0 ] || fail "httpbin failed to return 200 OK"
+kill ${pid}
 
 info "Applying ${dir}/happypath.yaml"
 ${helmfile} -f ${dir}/happypath.yaml apply