Add e2e tests for RHEL 9 EFI on Baremetal

- Add simple workflow test cases for RHEL 9 EFI on Baremetal.
- Documentation improvement for RHEL 9 on Baremetal.

cmd/integration_test/build/buildspecs/tinkerbell-test-eks-a-cli.yml

@@ -53,6 +53,12 @@ env:
     T_TINKERBELL_IMAGE_REDHAT_1_28: "tinkerbell_ci:image_redhat_1_28"
     T_TINKERBELL_IMAGE_REDHAT_1_29: "tinkerbell_ci:image_redhat_1_29"
     T_TINKERBELL_IMAGE_REDHAT_1_30: "tinkerbell_ci:image_redhat_1_30"
+    T_TINKERBELL_IMAGE_REDHAT_9_1_25: "tinkerbell_ci:image_redhat_9_1_25"
+    T_TINKERBELL_IMAGE_REDHAT_9_1_26: "tinkerbell_ci:image_redhat_9_1_26"
+    T_TINKERBELL_IMAGE_REDHAT_9_1_27: "tinkerbell_ci:image_redhat_9_1_27"
+    T_TINKERBELL_IMAGE_REDHAT_9_1_28: "tinkerbell_ci:image_redhat_9_1_28"
+    T_TINKERBELL_IMAGE_REDHAT_9_1_29: "tinkerbell_ci:image_redhat_9_1_29"
+    T_TINKERBELL_IMAGE_REDHAT_9_1_30: "tinkerbell_ci:image_redhat_9_1_30"
     T_TINKERBELL_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key"
     T_TINKERBELL_CP_NETWORK_CIDR: "tinkerbell_ci:cp_network_cidr"
     T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv"

docs/content/en/docs/getting-started/baremetal/bare-spec.md

@@ -306,7 +306,7 @@ In the example, there are `TinkerbellMachineConfig` sections for control plane (
 The following fields identify information needed to configure the nodes in each of those groups.
 >**_NOTE:_** Currently, you can only have one machine group for all machines in the control plane, although you can have multiple machine groups for the workers.
 >
-### hardwareSelector (optional)
+### hardwareSelector (Required)
 Use fields under `hardwareSelector` to add key/value pair labels to match particular machines that you identified in the CSV file where you defined the machines in your cluster.
 Choose any label name you like.
 For example, if you had added the label `node=cp-machine` to the machines listed in your CSV file that you want to be control plane nodes, the following `hardwareSelector` field would cause those machines to be added to the control plane:
@@ -365,6 +365,11 @@ When you generate a Bare Metal cluster configuration, the `TinkerbellTemplateCon
 Advanced users can override the default values set for `TinkerbellTemplateConfig`.
 They can also add their own [Tinkerbell actions](https://docs.tinkerbell.org/actions/action-architecture/) to make personalized modifications to EKS Anywhere nodes.
 
+A default `TinkerbellTemplateConfig` can be generated from cluster config template using eksctl:
+```bash
+eksctl anywhere generate tinkerbelltemplateconfig -f eksa-mgmt-cluster.yaml
+```
+
 The following shows three `TinkerbellTemplateConfig` examples that you can add to your cluster configuration file to override the values that EKS Anywhere sets: one for Ubuntu, one for RHEL and one for Bottlerocket.
 Most actions used differ for different operating systems.
 
@@ -470,7 +475,7 @@ spec:
     version: "0.1"
 ```
 
-### RHEL TinkerbellTemplateConfig example
+### RHEL9 TinkerbellTemplateConfig example
 
 ```yaml
 ---
@@ -487,13 +492,13 @@ spec:
     - actions:
       - environment:
           COMPRESSED: "true"
-          DEST_DISK: /dev/sda
+          DEST_DISK: '{{ index .Hardware.Disks 0 }}'
           IMG_URL: https://my-file-server/rhel-9-uefi-amd64.gz
         image: public.ecr.aws/eks-anywhere/tinkerbell/hub/image2disk:6c0f0d437bde2c836d90b000312c8b25fa1b65e1-eks-a-15
         name: stream-image
         timeout: 360
       - environment:
-          DEST_DISK: /dev/sda2
+          DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 2 }}'
           DEST_PATH: /etc/netplan/config.yaml
           STATIC_NETPLAN: true
           DIRMODE: "0755"
@@ -503,17 +508,18 @@ spec:
           UID: "0"
         image: public.ecr.aws/eks-anywhere/tinkerbell/hub/writefile:6c0f0d437bde2c836d90b000312c8b25fa1b65e1-eks-a-15
         name: write-netplan
+        pid: host
         timeout: 90
       - environment:
           CONTENTS: |
             datasource:
               Ec2:
-                metadata_urls: [<admin-machine-ip>, <tinkerbell-ip-from-cluster-config>]
+                metadata_urls: [<admin-machine-ip>:50061, <tinkerbell-ip-from-cluster-config>:50061]
                 strict_id: false
             manage_etc_hosts: localhost
             warnings:
               dsid_missing_source: off
-          DEST_DISK: /dev/sda2
+          DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 2 }}'
           DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg
           DIRMODE: "0700"
           FS_TYPE: ext4
@@ -527,7 +533,7 @@ spec:
           CONTENTS: |
             network:
               config: disabled
-          DEST_DISK: /dev/sda2
+          DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 2 }}'
           DEST_PATH: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
           DIRMODE: "0700"
           FS_TYPE: ext4
@@ -540,7 +546,7 @@ spec:
       - environment:
           CONTENTS: |
             datasource: Ec2
-          DEST_DISK: /dev/sda2
+          DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 2 }}'
           DEST_PATH: /etc/cloud/ds-identify.cfg
           DIRMODE: "0700"
           FS_TYPE: ext4
@@ -550,8 +556,9 @@ spec:
         image: public.ecr.aws/eks-anywhere/tinkerbell/hub/writefile:6c0f0d437bde2c836d90b000312c8b25fa1b65e1-eks-a-15
         name: add-tink-cloud-init-ds-config
         timeout: 90
-      - name: "reboot"
+      - name: "reboot-image"
         image: public.ecr.aws/eks-anywhere/tinkerbell/hub/reboot:6c0f0d437bde2c836d90b000312c8b25fa1b65e1-eks-a-15
+        pid: host
         timeout: 90
         volumes:
           - /worker:/worker

test/e2e/README.md

@@ -123,9 +123,9 @@ T_TINKERBELL_SSH_AUTHORIZED_KEY # ssh public key for connectioning to machines
 ```
 ### Tinkerbell hardware-inventory.csv example
 ```csv
-guid,ip_address,gateway,nameservers,netmask,mac,hostname,vendor,bmc_ip,bmc_username,bmc_password,labels,disk
-bb341bc6-546f-4b38-s584-bb4f0e5f8934,10.24.32.110,10.24.32.1,8.8.8.8,255.255.255.0,3c:ec:ef:6e:a4:82,eksa-node01,supermicro,10.24.32.10,admin,password,type=cp,/dev/sda
-cc5619b8-a894-4db0-bf1a-fd04d5964d54,10.24.32.111,10.24.32.1,8.8.8.8,,255.255.255.0,3c:ec:ef:6e:a5:7c,eksa-node02,supermicro,10.24.32.11,admin,password,type=worker,/dev/sda
+ip_address,gateway,nameservers,netmask,mac,hostname,vendor,bmc_ip,bmc_username,bmc_password,labels,disk
+10.24.32.110,10.24.32.1,8.8.8.8,255.255.255.0,3c:ec:ef:6e:a4:82,eksa-node01,supermicro,10.24.32.10,admin,password,type=cp,/dev/sda
+10.24.32.111,10.24.32.1,8.8.8.8,,255.255.255.0,3c:ec:ef:6e:a5:7c,eksa-node02,supermicro,10.24.32.11,admin,password,type=worker,/dev/sda
 ```
 
 ## CloudStack tests requisites

test/e2e/SKIPPED_TESTS.yaml

@@ -61,11 +61,17 @@ skipped_tests:
 - TestTinkerbellKubernetes126UbuntuExternalEtcdSimpleFlow
 # Skipping skip power action tests - Not going to work because e2e test powers on CP and worker node at the same time and worker node times out early waiting for ipxe
 # Skipping a few redundant tests
+# For RHEL 9 Tinkerbell tests, only enabled test case for latest kubernetes version 1.30 due to limited testing resources.
 - TestTinkerbellKubernetes126RedHatSimpleFlow
 - TestTinkerbellKubernetes127RedHatSimpleFlow
 - TestTinkerbellKubernetes128RedHatSimpleFlow
 - TestTinkerbellKubernetes129RedHatSimpleFlow
 - TestTinkerbellKubernetes130RedHatSimpleFlow
+- TestTinkerbellKubernetes125RedHat9SimpleFlow
+- TestTinkerbellKubernetes126RedHat9SimpleFlow
+- TestTinkerbellKubernetes127RedHat9SimpleFlow
+- TestTinkerbellKubernetes128RedHat9SimpleFlow
+- TestTinkerbellKubernetes129RedHat9SimpleFlow
 - TestTinkerbellKubernetes126UbuntuSimpleFlow
 - TestTinkerbellKubernetes127UbuntuSimpleFlow
 - TestTinkerbellKubernetes126Ubuntu2204SimpleFlow

test/e2e/tinkerbell_test.go

@@ -1405,6 +1405,72 @@ func TestTinkerbellKubernetes130RedHatSimpleFlow(t *testing.T) {
 	runTinkerbellSimpleFlow(test)
 }
 
+func TestTinkerbellKubernetes125RedHat9SimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewTinkerbell(t, framework.WithRedHat9125Tinkerbell()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube125)),
+		framework.WithControlPlaneHardware(1),
+		framework.WithWorkerHardware(1),
+	)
+	runTinkerbellSimpleFlow(test)
+}
+
+func TestTinkerbellKubernetes126RedHat9SimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewTinkerbell(t, framework.WithRedHat9126Tinkerbell()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube126)),
+		framework.WithControlPlaneHardware(1),
+		framework.WithWorkerHardware(1),
+	)
+	runTinkerbellSimpleFlow(test)
+}
+
+func TestTinkerbellKubernetes127RedHat9SimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewTinkerbell(t, framework.WithRedHat9127Tinkerbell()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)),
+		framework.WithControlPlaneHardware(1),
+		framework.WithWorkerHardware(1),
+	)
+	runTinkerbellSimpleFlow(test)
+}
+
+func TestTinkerbellKubernetes128RedHat9SimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewTinkerbell(t, framework.WithRedHat9128Tinkerbell()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)),
+		framework.WithControlPlaneHardware(1),
+		framework.WithWorkerHardware(1),
+	)
+	runTinkerbellSimpleFlow(test)
+}
+
+func TestTinkerbellKubernetes129RedHat9SimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewTinkerbell(t, framework.WithRedHat9129Tinkerbell()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+		framework.WithControlPlaneHardware(1),
+		framework.WithWorkerHardware(1),
+	)
+	runTinkerbellSimpleFlow(test)
+}
+
+func TestTinkerbellKubernetes130RedHat9SimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewTinkerbell(t, framework.WithRedHat9130Tinkerbell()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
+		framework.WithControlPlaneHardware(1),
+		framework.WithWorkerHardware(1),
+	)
+	runTinkerbellSimpleFlow(test)
+}
+
 func TestTinkerbellKubernetes128BottleRocketSimpleFlow(t *testing.T) {
 	test := framework.NewClusterE2ETest(
 		t,

test/framework/tinkerbell.go

@@ -34,6 +34,12 @@ const (
 	tinkerbellImageRedHat128EnvVar                   = "T_TINKERBELL_IMAGE_REDHAT_1_28"
 	tinkerbellImageRedHat129EnvVar                   = "T_TINKERBELL_IMAGE_REDHAT_1_29"
 	tinkerbellImageRedHat130EnvVar                   = "T_TINKERBELL_IMAGE_REDHAT_1_30"
+	tinkerbellImageRedHat9125EnvVar                  = "T_TINKERBELL_IMAGE_REDHAT_9_1_25"
+	tinkerbellImageRedHat9126EnvVar                  = "T_TINKERBELL_IMAGE_REDHAT_9_1_26"
+	tinkerbellImageRedHat9127EnvVar                  = "T_TINKERBELL_IMAGE_REDHAT_9_1_27"
+	tinkerbellImageRedHat9128EnvVar                  = "T_TINKERBELL_IMAGE_REDHAT_9_1_28"
+	tinkerbellImageRedHat9129EnvVar                  = "T_TINKERBELL_IMAGE_REDHAT_9_1_29"
+	tinkerbellImageRedHat9130EnvVar                  = "T_TINKERBELL_IMAGE_REDHAT_9_1_30"
 	tinkerbellInventoryCsvFilePathEnvVar             = "T_TINKERBELL_INVENTORY_CSV"
 	tinkerbellSSHAuthorizedKey                       = "T_TINKERBELL_SSH_AUTHORIZED_KEY"
 	tinkerbellCIEnvironmentEnvVar                    = "T_TINKERBELL_CI_ENVIRONMENT"
@@ -59,6 +65,12 @@ var requiredTinkerbellEnvVars = []string{
 	tinkerbellImageRedHat128EnvVar,
 	tinkerbellImageRedHat129EnvVar,
 	tinkerbellImageRedHat130EnvVar,
+	tinkerbellImageRedHat9125EnvVar,
+	tinkerbellImageRedHat9126EnvVar,
+	tinkerbellImageRedHat9127EnvVar,
+	tinkerbellImageRedHat9128EnvVar,
+	tinkerbellImageRedHat9129EnvVar,
+	tinkerbellImageRedHat9130EnvVar,
 	tinkerbellInventoryCsvFilePathEnvVar,
 	tinkerbellSSHAuthorizedKey,
 }
@@ -209,7 +221,7 @@ func withKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, machi
 	return func(t *Tinkerbell) {
 		t.fillers = append(t.fillers,
 			imageForKubeVersionAndOS(kubeVersion, os, ""),
-			api.WithOsFamilyForAllTinkerbellMachines(osFamiliesForOS[os]),
+			api.WithOsFamilyForAllTinkerbellMachines(getOSFamilyForOS(os)),
 		)
 	}
 }
@@ -264,6 +276,36 @@ func WithRedHat130Tinkerbell() TinkerbellOpt {
 	return withKubeVersionAndOS(anywherev1.Kube130, RedHat8, "", nil)
 }
 
+// WithRedHat9125Tinkerbell tink test with redhat9 efi 1.25.
+func WithRedHat9125Tinkerbell() TinkerbellOpt {
+	return withKubeVersionAndOS(anywherev1.Kube125, RedHat9, "", nil)
+}
+
+// WithRedHat9126Tinkerbell tink test with redhat9 efi 1.26.
+func WithRedHat9126Tinkerbell() TinkerbellOpt {
+	return withKubeVersionAndOS(anywherev1.Kube126, RedHat9, "", nil)
+}
+
+// WithRedHat9127Tinkerbell tink test with redhat9 efi 1.27.
+func WithRedHat9127Tinkerbell() TinkerbellOpt {
+	return withKubeVersionAndOS(anywherev1.Kube127, RedHat9, "", nil)
+}
+
+// WithRedHat9128Tinkerbell tink test with redhat9 efi 1.28.
+func WithRedHat9128Tinkerbell() TinkerbellOpt {
+	return withKubeVersionAndOS(anywherev1.Kube128, RedHat9, "", nil)
+}
+
+// WithRedHat9129Tinkerbell tink test with redhat9 efi 1.29.
+func WithRedHat9129Tinkerbell() TinkerbellOpt {
+	return withKubeVersionAndOS(anywherev1.Kube129, RedHat9, "", nil)
+}
+
+// WithRedHat9130Tinkerbell tink test with redhat9 efi 1.30.
+func WithRedHat9130Tinkerbell() TinkerbellOpt {
+	return withKubeVersionAndOS(anywherev1.Kube130, RedHat9, "", nil)
+}
+
 func WithBottleRocketTinkerbell() TinkerbellOpt {
 	return func(t *Tinkerbell) {
 		t.fillers = append(t.fillers,
@@ -312,15 +354,23 @@ func WithHookImagesURLPath(url string) TinkerbellOpt {
 func imageForKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, operatingSystem OS, machineConfigType string, rtos ...bool) api.TinkerbellFiller {
 	var tinkerbellFiller api.TinkerbellFiller
 	if machineConfigType == workerIdentifier {
-		tinkerbellFiller = api.WithTinkerbellWorkerMachineConfigOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion, rtos...)), osFamiliesForOS[operatingSystem])
+		tinkerbellFiller = api.WithTinkerbellWorkerMachineConfigOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion, rtos...)), getOSFamilyForOS(operatingSystem))
 	} else if machineConfigType == controlPlaneIdentifier {
-		tinkerbellFiller = api.WithTinkerbellCPMachineConfigOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion, rtos...)), osFamiliesForOS[operatingSystem])
+		tinkerbellFiller = api.WithTinkerbellCPMachineConfigOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion, rtos...)), getOSFamilyForOS(operatingSystem))
 	} else {
 		tinkerbellFiller = api.WithTinkerbellOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion, rtos...)))
 	}
 	return tinkerbellFiller
 }
 
+// Use ubuntu as the OSFamily for RHEL 9 in tinkerbell E2E test so that EKS-A will use the correct partition number 2 for tinkerbell default actions.
+func getOSFamilyForOS(os OS) anywherev1.OSFamily {
+	if os == RedHat9 {
+		return anywherev1.Ubuntu
+	}
+	return osFamiliesForOS[os]
+}
+
 // Ubuntu126Image represents an Ubuntu raw image corresponding to Kubernetes 1.26.
 func Ubuntu126Image() api.TinkerbellFiller {
 	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, "")