Merge pull request #93 from canonical/KU-519/wait-apiserver

wait for apiserver instead of Ready nodes
canonical · Mar 13, 2024 · 500b80e · 500b80e
2 parents 22f9dc6 + 2628f40
commit 500b80e
Show file tree

Hide file tree

Showing 13 changed files with 37 additions and 15 deletions.
diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go
@@ -144,7 +144,7 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
 		fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs),
 		fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy),
 		scriptPath(configureKubeletScript),
-		"microk8s status --wait-ready",
+		scriptPath(waitAPIServerScript),
 		"microk8s refresh-certs /var/tmp",
 		fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP),
 		fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort),

diff --git a/controllers/cloudinit/controlplane_init_test.go b/controllers/cloudinit/controlplane_init_test.go
@@ -50,7 +50,7 @@ func TestControlPlaneInit(t *testing.T) {
 			`/capi-scripts/00-install-microk8s.sh "--channel 1.25 --classic"`,
 			`/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`,
 			`/capi-scripts/10-configure-kubelet.sh`,
-			`microk8s status --wait-ready`,
+			`/capi-scripts/50-wait-apiserver.sh`,
 			`microk8s refresh-certs /var/tmp`,
 			`/capi-scripts/10-configure-calico-ipip.sh true`,
 			`/capi-scripts/10-configure-cluster-agent-port.sh "30000"`,

diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go
@@ -126,11 +126,11 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
 		fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs),
 		fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy),
 		scriptPath(configureKubeletScript),
-		"microk8s status --wait-ready",
+		scriptPath(waitAPIServerScript),
 		fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP),
 		fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort),
 		fmt.Sprintf("%s %q", scriptPath(configureDqlitePortScript), input.DqlitePort),
-		"microk8s status --wait-ready",
+		scriptPath(waitAPIServerScript),
 		fmt.Sprintf("%s %q %q", scriptPath(configureCertLB), endpointType, input.ControlPlaneEndpoint),
 		fmt.Sprintf("%s no %s", scriptPath(microk8sJoinScript), strings.Join(joinURLs, " ")),
 		scriptPath(configureAPIServerScript),

diff --git a/controllers/cloudinit/controlplane_join_test.go b/controllers/cloudinit/controlplane_join_test.go
@@ -48,11 +48,11 @@ func TestControlPlaneJoin(t *testing.T) {
 			`/capi-scripts/00-install-microk8s.sh "--channel 1.25 --classic"`,
 			`/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`,
 			`/capi-scripts/10-configure-kubelet.sh`,
-			`microk8s status --wait-ready`,
+			`/capi-scripts/50-wait-apiserver.sh`,
 			`/capi-scripts/10-configure-calico-ipip.sh true`,
 			`/capi-scripts/10-configure-cluster-agent-port.sh "30000"`,
 			`/capi-scripts/10-configure-dqlite-port.sh "2379"`,
-			`microk8s status --wait-ready`,
+			`/capi-scripts/50-wait-apiserver.sh`,
 			`/capi-scripts/10-configure-cert-for-lb.sh "DNS" "k8s.my-domain.com"`,
 			`/capi-scripts/20-microk8s-join.sh no "10.0.3.39:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.40:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.41:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
 			`/capi-scripts/10-configure-apiserver.sh`,

diff --git a/controllers/cloudinit/embed.go b/controllers/cloudinit/embed.go
@@ -75,6 +75,9 @@ const (
 
 	// configureTraefikScript configures the control plane endpoint in the traefik provider configuration.
 	configureTraefikScript script = "30-configure-traefik.sh"
+
+	// waitAPIServerScript waits for the kube-apiserver to be ready.
+	waitAPIServerScript script = "50-wait-apiserver.sh"
 )
 
 var allScripts = []script{
@@ -92,6 +95,7 @@ var allScripts = []script{
 	configureKubeletScript,
 	microk8sEnableScript,
 	microk8sJoinScript,
+	waitAPIServerScript,
 }
 
 func mustGetScript(scriptName script) string {

diff --git a/controllers/cloudinit/scripts/10-configure-apiserver.sh b/controllers/cloudinit/scripts/10-configure-apiserver.sh
@@ -37,7 +37,7 @@ while ! snap restart microk8s.daemon-kubelite; do
 done
 
 # delete kubernetes service to make sure port is updated
-microk8s status --wait-ready
+/capi-scripts/50-wait-apiserver.sh
 microk8s kubectl delete svc kubernetes
 
 # redirect port 16443 to 6443

diff --git a/controllers/cloudinit/scripts/10-configure-calico-ipip.sh b/controllers/cloudinit/scripts/10-configure-calico-ipip.sh
@@ -15,6 +15,13 @@ fi
 
 CNI_YAML="/var/snap/microk8s/current/args/cni-network/cni.yaml"
 
+if [ ! -f "${CNI_YAML}" ]; then
+  echo "Will not configure Calico, missing cni.yaml"
+  exit 0
+fi
+
+/capi-scripts/50-wait-apiserver.sh
+
 # Stop calico-node and delete ippools to ensure no vxlan pools are left around
 microk8s kubectl delete daemonset/calico-node -n kube-system || true
 microk8s kubectl delete ippools --all || true

diff --git a/controllers/cloudinit/scripts/10-configure-cert-for-lb.sh b/controllers/cloudinit/scripts/10-configure-cert-for-lb.sh
@@ -22,4 +22,5 @@ sleep 10
 while ! snap restart microk8s.daemon-kubelite; do
   sleep 5
 done
-microk8s status --wait-ready
+
+/capi-scripts/50-wait-apiserver.sh
diff --git a/controllers/cloudinit/scripts/20-microk8s-enable.sh b/controllers/cloudinit/scripts/20-microk8s-enable.sh
@@ -12,6 +12,6 @@ microk8s enable community || true
 
 while [[ "$@" != "" ]]; do
   microk8s enable "$1"
-  microk8s status --wait-ready
+  /capi-scripts/50-wait-apiserver.sh
   shift
 done
diff --git a/controllers/cloudinit/scripts/20-microk8s-join.sh b/controllers/cloudinit/scripts/20-microk8s-join.sh
@@ -55,8 +55,5 @@ done
 sleep 10
 
 if [ ${1} == "no" ]; then
-  while ! microk8s status --wait-ready; do
-    echo "Waiting for the cluster to come up"
-    sleep 5
-  done
+  /capi-scripts/50-wait-apiserver.sh
 fi
diff --git a/controllers/cloudinit/scripts/50-wait-apiserver.sh b/controllers/cloudinit/scripts/50-wait-apiserver.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -xe
+
+# Usage:
+#   $0
+#
+# Assumptions:
+#   - microk8s is installed
+#   - microk8s kubelite service is running
+
+while ! microk8s kubectl get --raw /readyz; do
+  echo Waiting for kube-apiserver
+  sleep 3
+done
diff --git a/controllers/cloudinit/worker_join.go b/controllers/cloudinit/worker_join.go
@@ -115,7 +115,7 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) {
 		fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs),
 		fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy),
 		scriptPath(configureKubeletScript),
-		"microk8s status --wait-ready",
+		scriptPath(waitAPIServerScript),
 		fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort),
 		fmt.Sprintf("%s yes %s", scriptPath(microk8sJoinScript), strings.Join(joinURLs, " ")),
 		fmt.Sprintf("%s %s 6443 %s", scriptPath(configureTraefikScript), input.ControlPlaneEndpoint, stopApiServerProxyRefreshes),

diff --git a/controllers/cloudinit/worker_join_test.go b/controllers/cloudinit/worker_join_test.go
@@ -45,7 +45,7 @@ func TestWorkerJoin(t *testing.T) {
 			`/capi-scripts/00-install-microk8s.sh "--channel 1.24 --classic"`,
 			`/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`,
 			`/capi-scripts/10-configure-kubelet.sh`,
-			`microk8s status --wait-ready`,
+			`/capi-scripts/50-wait-apiserver.sh`,
 			`/capi-scripts/10-configure-cluster-agent-port.sh "30000"`,
 			`/capi-scripts/20-microk8s-join.sh yes "10.0.3.194:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.195:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
 			`/capi-scripts/30-configure-traefik.sh capi-aws-apiserver-1647391446.us-east-1.elb.amazonaws.com 6443 no`,