adds prow job to install nephio components on kubeadm cluster (#297)

* add support for installing nonKIND cluster via kubeadm allows for installing nonKIND K8S cluster on ubuntu and fedora for e2e test cases * Add support for installing nonKIND cluster via kubeadm, moving file locations Co-authored-by: Vishwanath Jayaraman <[email protected]> * adds prow job to install Nephio components on kubeadm cluster Co-authored-by: Daniel Kostecki <[email protected]> * add support for installing nonKIND cluster via kubeadm (previously PR #46) (#296) * add support for installing nonKIND cluster via kubeadm allows for installing nonKIND K8S cluster on ubuntu and fedora for e2e test cases * Add support for installing nonKIND cluster via kubeadm, moving file locations Co-authored-by: Vishwanath Jayaraman <[email protected]> * Change yml file suffix in main --------- Co-authored-by: vjayaramrh <[email protected]> * adds prow job to install nephio components on kubeadm cluster Co-authored-by: Daniel Kostecki <[email protected]> --------- Co-authored-by: Daniel Kostecki <[email protected]> Co-authored-by: Daniel Kostecki <[email protected]>
nephio-project · Aug 22, 2024 · f90ff7b · f90ff7b
1 parent 4a828c2
commit f90ff7b
Show file tree

Hide file tree

Showing 11 changed files with 158 additions and 5 deletions.
diff --git a/.prow.yaml b/.prow.yaml
@@ -289,6 +289,92 @@ presubmits:
                 path: id_rsa
               - key: id_rsa.pub
                 path: id_rsa.pub
+  - name: e2e-kubeadm-ubuntu-jammy
+    annotations:
+    labels:
+    run_if_changed: '^e2e/'
+    skip_report: false
+    optional: true
+    decorate: true
+    cluster: default
+    spec:
+      containers:
+        - image: "nephio/e2e:latest"
+          command:
+            - "/bin/sh"
+          args:
+            - "-c"
+            - |
+              set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-ubuntu-jammy -auto-approve' EXIT;
+              terraform init && timeout 120m terraform apply -target module.gcp-ubuntu-jammy -var="fail_fast=true" -var="mgmt_cluster_type=kubeadm" -auto-approve
+          volumeMounts:
+            - name: satoken
+              mountPath: "/etc/satoken"
+            - name: ssh-key-vol
+              mountPath: "/etc/ssh-key"
+          resources:
+            requests:
+              cpu: 2
+              memory: 2Gi
+      volumes:
+        - name: satoken
+          secret:
+            secretName: satoken
+            items:
+              - key: satoken
+                path: satoken
+        - name: ssh-key-vol
+          secret:
+            secretName: ssh-key-e2e
+            defaultMode: 256
+            items:
+              - key: id_rsa
+                path: id_rsa
+              - key: id_rsa.pub
+                path: id_rsa.pub
+  - name: e2e-kubeadm-fedora-38
+    annotations:
+    labels:
+    run_if_changed: '^e2e/'
+    skip_report: false
+    optional: true
+    decorate: true
+    cluster: default
+    spec:
+      containers:
+        - image: "nephio/e2e:latest"
+          command:
+            - "/bin/sh"
+          args:
+            - "-c"
+            - |
+              set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-fedora-38 -auto-approve' EXIT;
+              terraform init && timeout 120m terraform apply -target module.gcp-fedora-38 -var="fail_fast=true" -var="mgmt_cluster_type=kubeadm" -auto-approve
+          volumeMounts:
+            - name: satoken
+              mountPath: "/etc/satoken"
+            - name: ssh-key-vol
+              mountPath: "/etc/ssh-key"
+          resources:
+            requests:
+              cpu: 2
+              memory: 2Gi
+      volumes:
+        - name: satoken
+          secret:
+            secretName: satoken
+            items:
+              - key: satoken
+                path: satoken
+        - name: ssh-key-vol
+          secret:
+            secretName: ssh-key-e2e
+            defaultMode: 256
+            items:
+              - key: id_rsa
+                path: id_rsa
+              - key: id_rsa.pub
+                path: id_rsa.pub
 postsubmits:
   - name: build-push-image-releaser
     cluster: default

diff --git a/e2e/provision/init.sh b/e2e/provision/init.sh
@@ -74,20 +74,26 @@ NEPHIO_USER=${NEPHIO_USER:-$(get_metadata nephio-user "${USER:-ubuntu}")}
 NEPHIO_CATALOG_REPO_URI=${NEPHIO_CATALOG_REPO_URI:-$(get_metadata nephio-catalog-repo-uri "https://github.com/nephio-project/catalog.git")}
 K8S_CONTEXT=${K8S_CONTEXT:-"kind-kind"}
 K8S_VERSION=${K8S_VERSION:-"v1.29.2"}
-export ANSIBLE_CMD_EXTRA_VAR_LIST='{ "nephio_catalog_repo_uri": "'${NEPHIO_CATALOG_REPO_URI}'", "k8s": { "context" : "'${K8S_CONTEXT}'", "version" : "'$K8S_VERSION'" } }'
 HOME=${NEPHIO_HOME:-/home/$NEPHIO_USER}
 REPO_DIR=${NEPHIO_REPO_DIR:-$HOME/test-infra}
 DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME:-""}
 DOCKERHUB_TOKEN=${DOCKERHUB_TOKEN:-""}
 FAIL_FAST=${FAIL_FAST:-$(get_metadata fail_fast "false")}
+# MGMT_CLUSTER_TYPE is intended to be set by prow jobs
+MGMT_CLUSTER_TYPE=${MGMT_CLUSTER_TYPE:-$(get_metadata mgmt_cluster_type "kind")}
+
+if [ ${MGMT_CLUSTER_TYPE} == "kubeadm" ]; then
+    K8S_CONTEXT="kubernetes-admin@kubernetes"
+fi
+export ANSIBLE_CMD_EXTRA_VAR_LIST='{ "nephio_catalog_repo_uri": "'${NEPHIO_CATALOG_REPO_URI}'", "k8s": { "context" : "'${K8S_CONTEXT}'", "version" : "'$K8S_VERSION'" } }'
 
 if [ ${K8S_CONTEXT} == "kind-kind" ]; then
     export ANSIBLE_TAG=all
 else
     export ANSIBLE_TAG=nonkind_k8s
 fi
 
-echo "$DEBUG, $RUN_E2E, $REPO, $BRANCH, $NEPHIO_USER, $HOME, $REPO_DIR, $DOCKERHUB_USERNAME, $DOCKERHUB_TOKEN, $ANSIBLE_TAG, $ANSIBLE_CMD_EXTRA_VAR_LIST"
+echo "$DEBUG, $RUN_E2E, $REPO, $BRANCH, $NEPHIO_USER, $HOME, $REPO_DIR, $DOCKERHUB_USERNAME, $DOCKERHUB_TOKEN, $ANSIBLE_TAG, $ANSIBLE_CMD_EXTRA_VAR_LIST $K8S_CONTEXT $MGMT_CLUSTER_TYPE"
 trap get_status ERR
 
 # Validate root permissions for current user and NEPHIO_USER
@@ -166,11 +172,11 @@ chown "$NEPHIO_USER:$NEPHIO_USER" "$HOME/.bash_aliases"
 # Sandbox Creation
 int_start=$(date +%s)
 cd "$REPO_DIR/e2e/provision"
-export DEBUG DOCKERHUB_USERNAME DOCKERHUB_TOKEN FAIL_FAST
+export DEBUG DOCKERHUB_USERNAME DOCKERHUB_TOKEN FAIL_FAST MGMT_CLUSTER_TYPE K8S_VERSION
 runuser -u "$NEPHIO_USER" ./install_sandbox.sh
 printf "%s secs\n" "$(($(date +%s) - int_start))"
 
-if [[ $RUN_E2E == "true" ]]; then
+if [[ $RUN_E2E == "true" && $MGMT_CLUSTER_TYPE == "kind" ]]; then
     runuser -u "$NEPHIO_USER" "$REPO_DIR/e2e/e2e.sh"
 fi
 

diff --git a/e2e/provision/install_sandbox.sh b/e2e/provision/install_sandbox.sh
@@ -75,6 +75,14 @@ fact_caching_connection = /tmp
 EOT
 
 # Management cluster creation
+if [[ ${MGMT_CLUSTER_TYPE:-kind} == "kubeadm" ]]; then
+    ansible_cmd_kubeadm="$(command -v ansible-playbook) -i 127.0.0.1, playbooks/deploy_kubeadm_k8s.yml --extra-vars=\"k8s_ver=${K8S_VERSION:1:4}\" "
+    [[ ${DEBUG:-false} != "true" ]] || ansible_cmd_kubeadm+="-vvv "
+    echo "$ansible_cmd_kubeadm"
+    eval "$ansible_cmd_kubeadm" | tee ~/kubeadm.log
+    echo "Done installing kubeadm cluster"
+fi
+
 ansible_cmd="$(command -v ansible-playbook) -i 127.0.0.1, playbooks/cluster.yml --tags ${ANSIBLE_TAG:-all} "
 [[ ${DEBUG:-false} != "true" ]] || ansible_cmd+="-vvv "
 if [ -n "${ANSIBLE_CMD_EXTRA_VAR_LIST:-}" ]; then

diff --git a/e2e/provision/playbooks/deploy_kubeadm_k8s.yml b/e2e/provision/playbooks/deploy_kubeadm_k8s.yml
@@ -1,5 +1,30 @@
 ---
 - name: Deploy k8s using kubeadm on host
   hosts: all
+  pre_tasks:
+    - name: Sysctl settings updates
+      become: true
+      ansible.posix.sysctl:
+        reload: true
+        sysctl_file: /etc/sysctl.conf
+        name: "{{ item.name }}"
+        value: "{{ item.value }}"
+        state: present
+      with_items:
+        - {name: fs.inotify.max_user_watches, value: 524288}
+        - {name: fs.inotify.max_user_instances, value: 512}
+        - {name: kernel.keys.maxkeys, value: 500000}
+        - {name: kernel.keys.maxbytes, value: 1000000}
+    - name: Install Docker Engine needed for kpt functions
+      become: true
+      block:
+        - name: Install docker binaries
+          ansible.builtin.include_role:
+            name: andrewrothstein.docker_engine
+        - name: Grant Docker permissions to user
+          ansible.builtin.user:
+            name: "{{ ansible_user_id }}"
+            groups: docker
+            append: true
   roles:
     - role: kubeadm
diff --git a/e2e/provision/playbooks/roles/kubeadm/defaults/main.yml b/e2e/provision/playbooks/roles/kubeadm/defaults/main.yml
@@ -1,3 +1,4 @@
+---
 containerd_ver: 1.6.19
 runc_ver: 1.1.1
 cni_plugins_ver: 1.1.1

diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-Debian.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-Debian.yml
@@ -1,3 +1,4 @@
+---
 - name: Install kubelet kubeadm kubectl on {{ ansible_os_family }}
   become: true
   block:

diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-RedHat.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-RedHat.yml
@@ -1,3 +1,4 @@
+---
 - name: Install kubelet kubeadm kubectl on {{ ansible_os_family }}
   become: true
   block:

diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/main.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/main.yml
@@ -1,3 +1,4 @@
+---
 - name: Get containerd and unarchive to /usr/local
   become: true
   ansible.builtin.unarchive:

diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf
@@ -2,13 +2,15 @@ module "gcp-ubuntu-focal" {
   source               = ".//modules/gcp"
   nephio_e2e_type      = var.e2e_type
   nephio_e2e_fail_fast = var.fail_fast
+  nephio_mgmt_cluster_type = var.mgmt_cluster_type
 }
 
 module "gcp-ubuntu-jammy" {
   source               = ".//modules/gcp"
   vmimage              = "ubuntu-os-cloud/ubuntu-2204-lts"
   nephio_e2e_type      = var.e2e_type
   nephio_e2e_fail_fast = var.fail_fast
+  nephio_mgmt_cluster_type = var.mgmt_cluster_type
 }
 
 module "gcp-fedora-34" {
@@ -17,6 +19,16 @@ module "gcp-fedora-34" {
   ansible_user         = "fedora"
   nephio_e2e_type      = var.e2e_type
   nephio_e2e_fail_fast = var.fail_fast
+  nephio_mgmt_cluster_type = var.mgmt_cluster_type
+}
+
+module "gcp-fedora-38" {
+  source               = ".//modules/gcp"
+  vmimage              = "fedora-cloud/fedora-cloud-38"
+  ansible_user         = "fedora"
+  nephio_e2e_type      = var.e2e_type
+  nephio_e2e_fail_fast = var.fail_fast
+  nephio_mgmt_cluster_type = var.mgmt_cluster_type
 }
 
 variable "e2e_type" {
@@ -30,3 +42,9 @@ variable "fail_fast" {
   default     = "false"
   type        = string
 }
+
+variable "mgmt_cluster_type" {
+  description = "Defines the type of k8s cluster"
+  default     = "kind"
+  type        = string
+}
diff --git a/e2e/terraform/modules/gcp/main.tf b/e2e/terraform/modules/gcp/main.tf
@@ -100,7 +100,7 @@ resource "google_compute_instance" "e2e_instances" {
     inline = [
       "cd /home/${var.ansible_user}/test-infra/e2e/provision/",
       "chmod +x init.sh",
-      "sudo -E FAIL_FAST=${var.nephio_e2e_fail_fast} E2ETYPE=${var.nephio_e2e_type} NEPHIO_REPO_DIR=/home/${var.ansible_user}/test-infra NEPHIO_DEBUG=true NEPHIO_RUN_E2E=true NEPHIO_USER=${var.ansible_user} ./init.sh"
+      "sudo -E FAIL_FAST=${var.nephio_e2e_fail_fast} MGMT_CLUSTER_TYPE=${var.nephio_mgmt_cluster_type} E2ETYPE=${var.nephio_e2e_type} NEPHIO_REPO_DIR=/home/${var.ansible_user}/test-infra NEPHIO_DEBUG=true NEPHIO_RUN_E2E=true NEPHIO_USER=${var.ansible_user} ./init.sh"
     ]
   }
 }
diff --git a/e2e/terraform/modules/gcp/variables.tf b/e2e/terraform/modules/gcp/variables.tf
@@ -75,3 +75,9 @@ variable "nephio_e2e_fail_fast" {
   default     = "false"
   type        = string
 }
+
+variable "nephio_mgmt_cluster_type" {
+  description = "The Nephio management cluster type"
+  default     = "kind"
+  type        = string
+}