From 6ccb75771a6757c61dcfbae856972c4acc8db453 Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Sun, 3 Nov 2019 14:01:32 +0900 Subject: [PATCH] Use kind for setting up E2E clusters --- .circleci/config.yml | 60 +++++++++++++++++++++++---------- test/integration/happypath.yaml | 2 +- test/integration/lib/ensure.sh | 12 ++----- test/integration/run.sh | 13 ++++--- 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b5e5119..027e3068 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,18 +99,30 @@ jobs: chmod +x linux-amd64/helm sudo mv linux-amd64/helm /usr/local/bin/ - run: - name: Deploy minikube + name: Setup Kubernetes environment: - CHANGE_MINIKUBE_NONE_USER: true - K8S_VERSION: v1.12.3 - MINIKUBE_VERSION: v0.30.0 + # See https://hub.docker.com/r/kindest/node/tags for available tags(k8s versions) + K8S_VERSION: v1.12.10 + KIND_VERSION: v0.5.1 command: | curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 - chmod +x minikube && sudo mv minikube /usr/local/bin/ - sudo -E minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION} - sudo -E minikube update-context + + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/ + + kind create cluster --name minikube --image kindest/node:${K8S_VERSION} + + KUBECONFIG="$(kind get kubeconfig-path --name="minikube")" + + echo Copying ${KUBECONFIG} to ~/.kube/config so that it is available to the succeeding steps. + + cp ${KUBECONFIG} ~/.kube/config + + cat ${KUBECONFIG} + + kubectl cluster-info - run: name: Wait for nodes to become ready command: JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done @@ -118,7 +130,7 @@ jobs: name: Execute integration tests command: | export TERM=xterm - make integration + KUBECONTEXT=kubernetes-admin@minikube make integration integration_tests_helm3: machine: @@ -144,18 +156,30 @@ jobs: chmod +x linux-amd64/helm sudo mv linux-amd64/helm /usr/local/bin/ - run: - name: Deploy minikube + name: Setup Kubernetes environment: - CHANGE_MINIKUBE_NONE_USER: true - K8S_VERSION: v1.12.3 - MINIKUBE_VERSION: v0.30.0 + # See https://hub.docker.com/r/kindest/node/tags for available tags(k8s versions) + K8S_VERSION: v1.12.10 + KIND_VERSION: v0.5.1 command: | curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 - chmod +x minikube && sudo mv minikube /usr/local/bin/ - sudo -E minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION} - sudo -E minikube update-context + + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/ + + kind create cluster --name minikube --image kindest/node:${K8S_VERSION} + + KUBECONFIG="$(kind get kubeconfig-path --name="minikube")" + + echo Copying ${KUBECONFIG} to ~/.kube/config so that it is available to the succeeding steps. + + cp ${KUBECONFIG} ~/.kube/config + + cat ${KUBECONFIG} + + kubectl cluster-info - run: name: Wait for nodes to become ready command: JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done @@ -163,7 +187,7 @@ jobs: name: Execute integration tests command: | export TERM=xterm - HELMFILE_HELM3=1 make integration + KUBECONTEXT=kubernetes-admin@minikube HELMFILE_HELM3=1 make integration # GITHUB_TOKEN env var must be setup in circleci console diff --git a/test/integration/happypath.yaml b/test/integration/happypath.yaml index 8186ed87..46a730de 100644 --- a/test/integration/happypath.yaml +++ b/test/integration/happypath.yaml @@ -10,7 +10,7 @@ repositories: url: https://kubernetes-charts-incubator.storage.googleapis.com helmDefaults: - kubeContext: minikube + kubeContext: {{ requiredEnv "KUBECONTEXT" }} releases: diff --git a/test/integration/lib/ensure.sh b/test/integration/lib/ensure.sh index bf3e1798..27b40a73 100644 --- a/test/integration/lib/ensure.sh +++ b/test/integration/lib/ensure.sh @@ -2,15 +2,6 @@ # Check environment is correctly setup -if ! hash minikube 2>/dev/null; then - fail "Minikube needs to be installed." -fi -if [ ! $(minikube status --format '{{.MinikubeStatus}}') == "Running" ]; then - fail "Minikube is not running." -fi -if [ ! $(minikube status --format '{{.ClusterStatus}}') == "Running" ]; then - fail "Minikube Cluster is not running." -fi if ! kubectl version --short 1> /dev/null; then fail "Could not connect to minikube apiserver" fi @@ -26,3 +17,6 @@ fi if ! hash helm 1>/dev/null; then fail "Helm needs to be installed." fi +if [ -z "${KUBECONTEXT}" ]; then + fail "KUBECONTEXT needs to be set." +fi diff --git a/test/integration/run.sh b/test/integration/run.sh index 65771bcf..ccb104aa 100755 --- a/test/integration/run.sh +++ b/test/integration/run.sh @@ -14,8 +14,8 @@ if [[ ! -d "${dir}" ]]; then dir="${PWD}"; fi test_ns="helmfile-tests-$(date +"%Y%m%d-%H%M%S")" helmfile="./helmfile --namespace=${test_ns}" -helm="helm --kube-context=minikube" -kubectl="kubectl --context=minikube --namespace=${test_ns}" +helm="helm --kube-context=${KUBECONTEXT}" +kubectl="kubectl --context=${KUBECONTEXT} --namespace=${test_ns}" # FUNCTIONS ---------------------------------------------------------------------------------------------------------- @@ -48,10 +48,11 @@ info "Using namespace: ${test_ns}" # helm v2 if helm version --client 1>/dev/null 2>/dev/null; then info "Using Helm version: $(helm version --short --client | grep -o v.*$)" + kubectl create clusterrolebinding tiller-full-access --clusterrole=cluster-admin --serviceaccount=kube-system:default ${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true # helm v3 else - info "Using Helm version: $(helm version --short | grep -o v.*$)" + info "Using Helm version: $(helm version --short | grep -o v.*$)" || true fi ${helm} plugin install https://github.com/databus23/helm-diff --version master ${kubectl} get namespace ${test_ns} &> /dev/null && warn "Namespace ${test_ns} exists, from a previous test run?" @@ -80,8 +81,12 @@ code=$? info "Syncing ${dir}/happypath.yaml" ${helmfile} -f ${dir}/happypath.yaml sync wait_deploy_ready httpbin-httpbin -retry 5 "curl --fail $(minikube service --url --namespace=${test_ns} httpbin-httpbin)/status/200" +kubectl port-forward --namespace=${test_ns} svc/httpbin-httpbin 8000 & +pid=$! +sleep 5 +retry 5 "curl --fail http://localhost:8000/status/200" [ ${retry_result} -eq 0 ] || fail "httpbin failed to return 200 OK" +kill ${pid} info "Applying ${dir}/happypath.yaml" ${helmfile} -f ${dir}/happypath.yaml apply