fix remaining tests

.github/workflows/e2e.yaml

@@ -0,0 +1,33 @@
+name: E2E Tests
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  e2e-tests:
+    name: Build provider images
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Install requirements
+        run: |
+          sudo apt install make
+          sudo snap install go --classic
+          sudo snap install docker
+      - name: Build provider images
+        run: sudo make docker-build-e2e
+      - name: Build images
+        run: |
+          cd templates/docker
+          sudo docker build . -t k8s-snap:dev
+      - name: Run e2e tests
+        run: sudo make test-e2e

controlplane/controllers/remediation.go

@@ -28,6 +28,7 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/klog/v2"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -236,6 +237,18 @@ func (r *CK8sControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.Cont
 		**/
 	}
 
+	microclusterPort := controlPlane.KCP.Spec.CK8sConfigSpec.ControlPlaneConfig.MicroclusterPort
+	clusterObjectKey := util.ObjectKey(controlPlane.Cluster)
+	workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, clusterObjectKey, microclusterPort)
+	if err != nil {
+		log.Error(err, "failed to create client to workload cluster")
+		return ctrl.Result{}, errors.Wrapf(err, "failed to create client to workload cluster")
+	}
+
+	if err := workloadCluster.RemoveMachineFromCluster(ctx, machineToBeRemediated); err != nil {
+		log.Error(err, "failed to remove machine from microcluster")
+	}
+
 	// Delete the machine
 	if err := r.Client.Delete(ctx, machineToBeRemediated); err != nil {
 		conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error())

pkg/cloudinit/controlplane_init_test.go

@@ -46,12 +46,20 @@ func TestNewInitControlPlane(t *testing.T) {
 
 	// TODO: add tests for expected files and commands
 	g.Expect(err).To(BeNil())
+	g.Expect(config.WriteFiles).To(ContainElement(cloudinit.File{
+		Path:        "/tmp/file",
+		Content:     "test file",
+		Permissions: "0400",
+		Owner:       "root:root",
+	}))
+	g.Expect(config.BootCommands).To(ContainElement("bootcmd"))
 	g.Expect(config.RunCommands).To(Equal([]string{
 		"set -x",
 		"prerun1",
 		"prerun2",
 		"/capi/scripts/install.sh",
 		"/capi/scripts/bootstrap.sh",
+		"/capi/scripts/load-images.sh",
 		"/capi/scripts/wait-apiserver-ready.sh",
 		"/capi/scripts/deploy-manifests.sh",
 		"/capi/scripts/configure-token.sh",

test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml

@@ -33,12 +33,13 @@ metadata:
   name: ${CLUSTER_NAME}-control-plane
   namespace: ${NAMESPACE}
 spec:
-  infrastructureTemplate:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: DockerMachineTemplate
-    name: ${CLUSTER_NAME}-control-plane
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
+  machineTemplate:
+    infrastructureTemplate:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
   spec:
     airGapped: true
     controlPlane:
@@ -66,12 +67,13 @@ spec:
           echo "signal $signal"
 
           if [ "$signal" == "pass" ]; then
-             curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test
-             exit 0
+            curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test
+            exit 0
           fi
         done
       permissions: "0777"
-    preK3sCommands:
+      owner: root:root
+    preRunCommands:
     - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}"
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
@@ -82,7 +84,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -125,7 +127,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate

test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml

@@ -36,10 +36,11 @@ metadata:
   name: ${CLUSTER_NAME}-control-plane
   namespace: ${NAMESPACE}
 spec:
-  infrastructureTemplate:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: DockerMachineTemplate
-    name: ${CLUSTER_NAME}-control-plane
+  machineTemplate:
+    infrastructureTemplate:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
   spec:
     airGapped: true
     controlPlane:
@@ -56,7 +57,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -100,7 +101,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
@@ -110,6 +111,7 @@ metadata:
 spec:
   template:
     spec:
+      airGapped: true
 ---
 # MachineHealthCheck object with
 # - a selector that targets all the machines with label e2e.remediation.label=""

test/e2e/helpers.go

@@ -22,15 +22,19 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/noderefutil"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -587,6 +591,40 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont
 	}, input.WaitForMachinesToBeUpgraded...)
 }
 
+type WaitForNodesReadyInput struct {
+	Lister            framework.Lister
+	KubernetesVersion string
+	Count             int
+	WaitForNodesReady []interface{}
+}
+
+// WaitForNodesReady waits until there are exactly the given count nodes and they have the correct Kubernetes minor version
+// and are ready.
+func WaitForNodesReady(ctx context.Context, input WaitForNodesReadyInput) {
+	Eventually(func() (bool, error) {
+		nodeList := &corev1.NodeList{}
+		if err := input.Lister.List(ctx, nodeList); err != nil {
+			return false, err
+		}
+		nodeReadyCount := 0
+		for _, node := range nodeList.Items {
+			n := node
+			match, err := CompareVersions(node.Status.NodeInfo.KubeletVersion, input.KubernetesVersion, "minor")
+			if err != nil {
+				return false, fmt.Errorf("failed to compare versions: %w", err)
+			}
+			if !match {
+				return false, nil
+			}
+			if !noderefutil.IsNodeReady(&n) {
+				return false, nil
+			}
+			nodeReadyCount++
+		}
+		return input.Count == nodeReadyCount, nil
+	}, input.WaitForNodesReady...).Should(BeTrue())
+}
+
 // byClusterOptions returns a set of ListOptions that allows to identify all the objects belonging to a Cluster.
 func byClusterOptions(name, namespace string) []client.ListOption {
 	return []client.ListOption{
@@ -596,3 +634,56 @@ func byClusterOptions(name, namespace string) []client.ListOption {
 		},
 	}
 }
+
+func parseVersion(version string) ([]int, error) {
+	// Remove the leading "v" if it exists
+	if strings.HasPrefix(version, "v") {
+		version = version[1:]
+	}
+
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		return nil, fmt.Errorf("invalid version format")
+	}
+
+	intParts := make([]int, len(parts))
+	for i, part := range parts {
+		num, err := strconv.Atoi(part)
+		if err != nil {
+			return nil, fmt.Errorf("invalid version part: %w", err)
+		}
+		intParts[i] = num
+	}
+
+	return intParts, nil
+}
+
+func compareVersionParts(v1, v2 []int, level string) bool {
+	switch strings.ToLower(level) {
+	case "major":
+		return v1[0] == v2[0]
+	case "minor":
+		return v1[0] == v2[0] && v1[1] == v2[1]
+	case "patch":
+		return v1[0] == v2[0] && v1[1] == v2[1] && v1[2] == v2[2]
+	default:
+		return v1[0] == v2[0] && v1[1] == v2[1] && v1[2] == v2[2]
+	}
+}
+
+// CompareVersions compares two versions and returns true if they are equal at the specified level.
+// The level can be "major", "minor", "patch" or an empty string to compare the full version.
+// Returns an error if the versions are not in the correct format.
+// Example: CompareVersions("1.19.0", "1.19.1", "patch") returns false
+func CompareVersions(version1, version2, level string) (bool, error) {
+	v1, err := parseVersion(version1)
+	if err != nil {
+		return false, fmt.Errorf("failed to parse version1: %w", err)
+	}
+
+	v2, err := parseVersion(version2)
+	if err != nil {
+		return false, fmt.Errorf("failed to parse version2: %w", err)
+	}
+	return compareVersionParts(v1, v2, level), nil
+}

test/e2e/md_remediation_test.go

@@ -108,7 +108,8 @@ var _ = Describe("When testing MachineDeployment remediation", func() {
 			By("Waiting until nodes are ready")
 			workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, result.Cluster.Name)
 			workloadClient := workloadProxy.GetClient()
-			framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{
+
+			WaitForNodesReady(ctx, WaitForNodesReadyInput{
 				Lister:            workloadClient,
 				KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
 				Count:             int(result.ExpectedTotalNodes()),