From ab0f9da379d43ea73999c11047452c1b5545d4bd Mon Sep 17 00:00:00 2001 From: Shizhao Liu Date: Tue, 13 Aug 2024 21:49:59 +0000 Subject: [PATCH] Add e2e tests for RHEL 9 EFI on Baremetal - Add simple workflow test cases for RHEL 9 EFI on Baremetal. - Documentation improvement for RHEL 9 on Baremetal. --- .../buildspecs/tinkerbell-test-eks-a-cli.yml | 5 ++ .../getting-started/baremetal/bare-spec.md | 30 ++++++---- .../baremetal/baremetal-getstarted.md | 2 +- test/e2e/README.md | 6 +- test/e2e/SKIPPED_TESTS.yaml | 5 ++ test/e2e/TINKERBELL_HARDWARE_COUNT.yaml | 5 ++ test/e2e/tinkerbell_test.go | 55 +++++++++++++++++++ test/framework/tinkerbell.go | 35 ++++++++++++ 8 files changed, 129 insertions(+), 14 deletions(-) diff --git a/cmd/integration_test/build/buildspecs/tinkerbell-test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/tinkerbell-test-eks-a-cli.yml index f16b3e232abe..2d2ae0d702e0 100644 --- a/cmd/integration_test/build/buildspecs/tinkerbell-test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/tinkerbell-test-eks-a-cli.yml @@ -53,6 +53,11 @@ env: T_TINKERBELL_IMAGE_REDHAT_1_28: "tinkerbell_ci:image_redhat_1_28" T_TINKERBELL_IMAGE_REDHAT_1_29: "tinkerbell_ci:image_redhat_1_29" T_TINKERBELL_IMAGE_REDHAT_1_30: "tinkerbell_ci:image_redhat_1_30" + T_TINKERBELL_IMAGE_REDHAT_9_1_26: "tinkerbell_ci:image_redhat_9_1_26" + T_TINKERBELL_IMAGE_REDHAT_9_1_27: "tinkerbell_ci:image_redhat_9_1_27" + T_TINKERBELL_IMAGE_REDHAT_9_1_28: "tinkerbell_ci:image_redhat_9_1_28" + T_TINKERBELL_IMAGE_REDHAT_9_1_29: "tinkerbell_ci:image_redhat_9_1_29" + T_TINKERBELL_IMAGE_REDHAT_9_1_30: "tinkerbell_ci:image_redhat_9_1_30" T_TINKERBELL_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key" T_TINKERBELL_CP_NETWORK_CIDR: "tinkerbell_ci:cp_network_cidr" T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv" diff --git a/docs/content/en/docs/getting-started/baremetal/bare-spec.md b/docs/content/en/docs/getting-started/baremetal/bare-spec.md index 9183bb9416ca..f5b0bf12a1f8 100644 --- a/docs/content/en/docs/getting-started/baremetal/bare-spec.md +++ b/docs/content/en/docs/getting-started/baremetal/bare-spec.md @@ -306,7 +306,7 @@ In the example, there are `TinkerbellMachineConfig` sections for control plane ( The following fields identify information needed to configure the nodes in each of those groups. >**_NOTE:_** Currently, you can only have one machine group for all machines in the control plane, although you can have multiple machine groups for the workers. > -### hardwareSelector (optional) +### hardwareSelector (required) Use fields under `hardwareSelector` to add key/value pair labels to match particular machines that you identified in the CSV file where you defined the machines in your cluster. Choose any label name you like. For example, if you had added the label `node=cp-machine` to the machines listed in your CSV file that you want to be control plane nodes, the following `hardwareSelector` field would cause those machines to be added to the control plane: @@ -365,12 +365,20 @@ When you generate a Bare Metal cluster configuration, the `TinkerbellTemplateCon Advanced users can override the default values set for `TinkerbellTemplateConfig`. They can also add their own [Tinkerbell actions](https://docs.tinkerbell.org/actions/action-architecture/) to make personalized modifications to EKS Anywhere nodes. +A default `TinkerbellTemplateConfig` can be generated from cluster config template using eksctl: +```bash +eksctl anywhere generate tinkerbelltemplateconfig -f eksa-mgmt-cluster.yaml +``` + The following shows three `TinkerbellTemplateConfig` examples that you can add to your cluster configuration file to override the values that EKS Anywhere sets: one for Ubuntu, one for RHEL and one for Bottlerocket. Most actions used differ for different operating systems. >**_NOTE:_** For the `stream-image` action, `DEST_DISK` points to the device representing the entire hard disk (for example, `/dev/sda`). -For UEFI-enabled images, such as Ubuntu and RHEL 9, write actions use `DEST_DISK` to point to the second partition (for example, `/dev/sda2`), with the first being the EFI partition. -For the Bottlerocket image, which has 12 partitions, `DEST_DISK` is partition 12 (for example, `/dev/sda12`). +For all the write actions, `DEST_DISK` must be set to point to the root partition. +For RHEL 8 and RHEL 9, `DEST_DISK` points to partition 1 (for example, `/dev/sda1`). +For Ubuntu, `DEST_DISK` points to partition 2 (for example, `/dev/sda2`), with the first partition being the EFI partition. +For the Bottlerocket image, which has 12 partitions, `DEST_DISK` points to partition 12 (for example, `/dev/sda12`). +Note that these partition index values may vary depending on how user built their OS image, user can always override these partitions using `TinkerbellTemplateConfig`. Device names will be different for different disk types. > @@ -487,13 +495,13 @@ spec: - actions: - environment: COMPRESSED: "true" - DEST_DISK: /dev/sda + DEST_DISK: '{{ index .Hardware.Disks 0 }}' IMG_URL: https://my-file-server/rhel-9-uefi-amd64.gz image: public.ecr.aws/eks-anywhere/tinkerbell/hub/image2disk:6c0f0d437bde2c836d90b000312c8b25fa1b65e1-eks-a-15 name: stream-image timeout: 360 - environment: - DEST_DISK: /dev/sda2 + DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' DEST_PATH: /etc/netplan/config.yaml STATIC_NETPLAN: true DIRMODE: "0755" @@ -503,17 +511,18 @@ spec: UID: "0" image: public.ecr.aws/eks-anywhere/tinkerbell/hub/writefile:6c0f0d437bde2c836d90b000312c8b25fa1b65e1-eks-a-15 name: write-netplan + pid: host timeout: 90 - environment: CONTENTS: | datasource: Ec2: - metadata_urls: [, ] + metadata_urls: [:50061, :50061] strict_id: false manage_etc_hosts: localhost warnings: dsid_missing_source: off - DEST_DISK: /dev/sda2 + DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg DIRMODE: "0700" FS_TYPE: ext4 @@ -527,7 +536,7 @@ spec: CONTENTS: | network: config: disabled - DEST_DISK: /dev/sda2 + DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' DEST_PATH: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg DIRMODE: "0700" FS_TYPE: ext4 @@ -540,7 +549,7 @@ spec: - environment: CONTENTS: | datasource: Ec2 - DEST_DISK: /dev/sda2 + DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' DEST_PATH: /etc/cloud/ds-identify.cfg DIRMODE: "0700" FS_TYPE: ext4 @@ -550,8 +559,9 @@ spec: image: public.ecr.aws/eks-anywhere/tinkerbell/hub/writefile:6c0f0d437bde2c836d90b000312c8b25fa1b65e1-eks-a-15 name: add-tink-cloud-init-ds-config timeout: 90 - - name: "reboot" + - name: "reboot-image" image: public.ecr.aws/eks-anywhere/tinkerbell/hub/reboot:6c0f0d437bde2c836d90b000312c8b25fa1b65e1-eks-a-15 + pid: host timeout: 90 volumes: - /worker:/worker diff --git a/docs/content/en/docs/getting-started/baremetal/baremetal-getstarted.md b/docs/content/en/docs/getting-started/baremetal/baremetal-getstarted.md index e2c45c3ef580..7048c6d13096 100644 --- a/docs/content/en/docs/getting-started/baremetal/baremetal-getstarted.md +++ b/docs/content/en/docs/getting-started/baremetal/baremetal-getstarted.md @@ -213,7 +213,7 @@ Follow these steps if you want to use your initial cluster to create and manage > ``` > * For creating multiple workload clusters, it is essential that the hardware labels and selectors defined for a given workload cluster are unique to that workload cluster. For instance, for an EKS Anywhere cluster named `eksa-workload1`, the hardware that is assigned for this cluster should have labels that are only going to be used for this cluster like `type=eksa-workload1-cp` and `type=eksa-workload1-worker`. Another workload cluster named `eksa-workload2` can have labels like `type=eksa-workload2-cp` and `type=eksa-workload2-worker`. Please note that even though labels can be arbitrary, they need to be unique for each workload cluster. Not specifying unique cluster labels can cause cluster creations to behave in unexpected ways which may lead to unsuccessful creations and unstable clusters. - See the [hardware selectors]({{< relref "./bare-spec/#hardwareselector-optional" >}}) section for more information + See the [hardware selectors]({{< relref "./bare-spec/#hardwareselector-required" >}}) section for more information 1. Check the workload cluster: diff --git a/test/e2e/README.md b/test/e2e/README.md index 553475361c03..fedffe8df91d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -123,9 +123,9 @@ T_TINKERBELL_SSH_AUTHORIZED_KEY # ssh public key for connectioning to machines ``` ### Tinkerbell hardware-inventory.csv example ```csv -guid,ip_address,gateway,nameservers,netmask,mac,hostname,vendor,bmc_ip,bmc_username,bmc_password,labels,disk -bb341bc6-546f-4b38-s584-bb4f0e5f8934,10.24.32.110,10.24.32.1,8.8.8.8,255.255.255.0,3c:ec:ef:6e:a4:82,eksa-node01,supermicro,10.24.32.10,admin,password,type=cp,/dev/sda -cc5619b8-a894-4db0-bf1a-fd04d5964d54,10.24.32.111,10.24.32.1,8.8.8.8,,255.255.255.0,3c:ec:ef:6e:a5:7c,eksa-node02,supermicro,10.24.32.11,admin,password,type=worker,/dev/sda +ip_address,gateway,nameservers,netmask,mac,hostname,vendor,bmc_ip,bmc_username,bmc_password,labels,disk +10.24.32.110,10.24.32.1,8.8.8.8,255.255.255.0,3c:ec:ef:6e:a4:82,eksa-node01,supermicro,10.24.32.10,admin,password,type=cp,/dev/sda +10.24.32.111,10.24.32.1,8.8.8.8,,255.255.255.0,3c:ec:ef:6e:a5:7c,eksa-node02,supermicro,10.24.32.11,admin,password,type=worker,/dev/sda ``` ## CloudStack tests requisites diff --git a/test/e2e/SKIPPED_TESTS.yaml b/test/e2e/SKIPPED_TESTS.yaml index af84d5cc52a7..390a811013da 100644 --- a/test/e2e/SKIPPED_TESTS.yaml +++ b/test/e2e/SKIPPED_TESTS.yaml @@ -65,11 +65,16 @@ skipped_tests: - TestTinkerbellKubernetes126UbuntuExternalEtcdSimpleFlow # Skipping skip power action tests - Not going to work because e2e test powers on CP and worker node at the same time and worker node times out early waiting for ipxe # Skipping a few redundant tests +# For RHEL 9 Tinkerbell tests, only enable test case for latest kubernetes version 1.30 due to limited testing resources. - TestTinkerbellKubernetes126RedHatSimpleFlow - TestTinkerbellKubernetes127RedHatSimpleFlow - TestTinkerbellKubernetes128RedHatSimpleFlow - TestTinkerbellKubernetes129RedHatSimpleFlow - TestTinkerbellKubernetes130RedHatSimpleFlow +- TestTinkerbellKubernetes126RedHat9SimpleFlow +- TestTinkerbellKubernetes127RedHat9SimpleFlow +- TestTinkerbellKubernetes128RedHat9SimpleFlow +- TestTinkerbellKubernetes129RedHat9SimpleFlow - TestTinkerbellKubernetes126Ubuntu2204SimpleFlow - TestTinkerbellKubernetes127Ubuntu2204SimpleFlow - TestTinkerbellKubernetes128Ubuntu2204SimpleFlow diff --git a/test/e2e/TINKERBELL_HARDWARE_COUNT.yaml b/test/e2e/TINKERBELL_HARDWARE_COUNT.yaml index 9f0516e6dd22..391978733dbc 100644 --- a/test/e2e/TINKERBELL_HARDWARE_COUNT.yaml +++ b/test/e2e/TINKERBELL_HARDWARE_COUNT.yaml @@ -87,6 +87,11 @@ TestTinkerbellKubernetes127RedHatSimpleFlow: 2 TestTinkerbellKubernetes128RedHatSimpleFlow: 2 TestTinkerbellKubernetes129RedHatSimpleFlow: 2 TestTinkerbellKubernetes130RedHatSimpleFlow: 2 +TestTinkerbellKubernetes126RedHat9SimpleFlow: 2 +TestTinkerbellKubernetes127RedHat9SimpleFlow: 2 +TestTinkerbellKubernetes128RedHat9SimpleFlow: 2 +TestTinkerbellKubernetes129RedHat9SimpleFlow: 2 +TestTinkerbellKubernetes130RedHat9SimpleFlow: 2 TestTinkerbellKubernetes126BottleRocketSimpleFlow: 2 TestTinkerbellKubernetes127BottleRocketSimpleFlow: 2 TestTinkerbellKubernetes128BottleRocketSimpleFlow: 2 diff --git a/test/e2e/tinkerbell_test.go b/test/e2e/tinkerbell_test.go index 73858a63efe8..276b98f9b36b 100644 --- a/test/e2e/tinkerbell_test.go +++ b/test/e2e/tinkerbell_test.go @@ -1405,6 +1405,61 @@ func TestTinkerbellKubernetes130RedHatSimpleFlow(t *testing.T) { runTinkerbellSimpleFlow(test) } +func TestTinkerbellKubernetes126RedHat9SimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithRedHat9126Tinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube126)), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), + ) + runTinkerbellSimpleFlow(test) +} + +func TestTinkerbellKubernetes127RedHat9SimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithRedHat9127Tinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), + ) + runTinkerbellSimpleFlow(test) +} + +func TestTinkerbellKubernetes128RedHat9SimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithRedHat9128Tinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), + ) + runTinkerbellSimpleFlow(test) +} + +func TestTinkerbellKubernetes129RedHat9SimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithRedHat9129Tinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), + ) + runTinkerbellSimpleFlow(test) +} + +func TestTinkerbellKubernetes130RedHat9SimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithRedHat9130Tinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), + ) + runTinkerbellSimpleFlow(test) +} + func TestTinkerbellKubernetes128BottleRocketSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index 1362ac3688e5..962cc59a976b 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -34,6 +34,11 @@ const ( tinkerbellImageRedHat128EnvVar = "T_TINKERBELL_IMAGE_REDHAT_1_28" tinkerbellImageRedHat129EnvVar = "T_TINKERBELL_IMAGE_REDHAT_1_29" tinkerbellImageRedHat130EnvVar = "T_TINKERBELL_IMAGE_REDHAT_1_30" + tinkerbellImageRedHat9126EnvVar = "T_TINKERBELL_IMAGE_REDHAT_9_1_26" + tinkerbellImageRedHat9127EnvVar = "T_TINKERBELL_IMAGE_REDHAT_9_1_27" + tinkerbellImageRedHat9128EnvVar = "T_TINKERBELL_IMAGE_REDHAT_9_1_28" + tinkerbellImageRedHat9129EnvVar = "T_TINKERBELL_IMAGE_REDHAT_9_1_29" + tinkerbellImageRedHat9130EnvVar = "T_TINKERBELL_IMAGE_REDHAT_9_1_30" tinkerbellInventoryCsvFilePathEnvVar = "T_TINKERBELL_INVENTORY_CSV" tinkerbellSSHAuthorizedKey = "T_TINKERBELL_SSH_AUTHORIZED_KEY" tinkerbellCIEnvironmentEnvVar = "T_TINKERBELL_CI_ENVIRONMENT" @@ -59,6 +64,11 @@ var requiredTinkerbellEnvVars = []string{ tinkerbellImageRedHat128EnvVar, tinkerbellImageRedHat129EnvVar, tinkerbellImageRedHat130EnvVar, + tinkerbellImageRedHat9126EnvVar, + tinkerbellImageRedHat9127EnvVar, + tinkerbellImageRedHat9128EnvVar, + tinkerbellImageRedHat9129EnvVar, + tinkerbellImageRedHat9130EnvVar, tinkerbellInventoryCsvFilePathEnvVar, tinkerbellSSHAuthorizedKey, } @@ -264,6 +274,31 @@ func WithRedHat130Tinkerbell() TinkerbellOpt { return withKubeVersionAndOS(anywherev1.Kube130, RedHat8, "", nil) } +// WithRedHat9126Tinkerbell tink test with redhat9 efi 1.26. +func WithRedHat9126Tinkerbell() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube126, RedHat9, "", nil) +} + +// WithRedHat9127Tinkerbell tink test with redhat9 efi 1.27. +func WithRedHat9127Tinkerbell() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube127, RedHat9, "", nil) +} + +// WithRedHat9128Tinkerbell tink test with redhat9 efi 1.28. +func WithRedHat9128Tinkerbell() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube128, RedHat9, "", nil) +} + +// WithRedHat9129Tinkerbell tink test with redhat9 efi 1.29. +func WithRedHat9129Tinkerbell() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube129, RedHat9, "", nil) +} + +// WithRedHat9130Tinkerbell tink test with redhat9 efi 1.30. +func WithRedHat9130Tinkerbell() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube130, RedHat9, "", nil) +} + func WithBottleRocketTinkerbell() TinkerbellOpt { return func(t *Tinkerbell) { t.fillers = append(t.fillers,