adds prow job to install Nephio components on kubeadm cluster

Co-authored-by: Daniel Kostecki <[email protected]>
nephio-project · Aug 20, 2024 · 82843c1 · 82843c1
1 parent 7cd331e
commit 82843c1
Show file tree

Hide file tree

Showing 9 changed files with 179 additions and 5 deletions.
diff --git a/.prow.yaml b/.prow.yaml
@@ -289,6 +289,92 @@ presubmits:
                 path: id_rsa
               - key: id_rsa.pub
                 path: id_rsa.pub
+  - name: e2e-kubeadm-ubuntu-jammy
+    annotations:
+    labels:
+    run_if_changed: '^e2e/'
+    skip_report: false
+    optional: true
+    decorate: true
+    cluster: default
+    spec:
+      containers:
+        - image: "nephio/e2e:latest"
+          command:
+            - "/bin/sh"
+          args:
+            - "-c"
+            - |
+              set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-ubuntu-jammy -auto-approve' EXIT;
+              terraform init && timeout 120m terraform apply -target module.gcp-ubuntu-jammy -var="fail_fast=true" -var="mgmt_cluster_type=kubeadm" -auto-approve
+          volumeMounts:
+            - name: satoken
+              mountPath: "/etc/satoken"
+            - name: ssh-key-vol
+              mountPath: "/etc/ssh-key"
+          resources:
+            requests:
+              cpu: 2
+              memory: 2Gi
+      volumes:
+        - name: satoken
+          secret:
+            secretName: satoken
+            items:
+              - key: satoken
+                path: satoken
+        - name: ssh-key-vol
+          secret:
+            secretName: ssh-key-e2e
+            defaultMode: 256
+            items:
+              - key: id_rsa
+                path: id_rsa
+              - key: id_rsa.pub
+                path: id_rsa.pub
+  - name: e2e-kubeadm-fedora-34
+    annotations:
+    labels:
+    run_if_changed: '^e2e/'
+    skip_report: false
+    optional: true
+    decorate: true
+    cluster: default
+    spec:
+      containers:
+        - image: "nephio/e2e:latest"
+          command:
+            - "/bin/sh"
+          args:
+            - "-c"
+            - |
+              set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-fedora-34 -auto-approve' EXIT;
+              terraform init && timeout 120m terraform apply -target module.gcp-fedora-34 -var="fail_fast=true" -var="mgmt_cluster_type=kubeadm" -auto-approve
+          volumeMounts:
+            - name: satoken
+              mountPath: "/etc/satoken"
+            - name: ssh-key-vol
+              mountPath: "/etc/ssh-key"
+          resources:
+            requests:
+              cpu: 2
+              memory: 2Gi
+      volumes:
+        - name: satoken
+          secret:
+            secretName: satoken
+            items:
+              - key: satoken
+                path: satoken
+        - name: ssh-key-vol
+          secret:
+            secretName: ssh-key-e2e
+            defaultMode: 256
+            items:
+              - key: id_rsa
+                path: id_rsa
+              - key: id_rsa.pub
+                path: id_rsa.pub
 postsubmits:
   - name: build-push-image-releaser
     cluster: default

diff --git a/e2e/provision/init.sh b/e2e/provision/init.sh
@@ -74,20 +74,27 @@ NEPHIO_USER=${NEPHIO_USER:-$(get_metadata nephio-user "${USER:-ubuntu}")}
 NEPHIO_CATALOG_REPO_URI=${NEPHIO_CATALOG_REPO_URI:-$(get_metadata nephio-catalog-repo-uri "https://github.com/nephio-project/catalog.git")}
 K8S_CONTEXT=${K8S_CONTEXT:-"kind-kind"}
 K8S_VERSION=${K8S_VERSION:-"v1.29.2"}
-export ANSIBLE_CMD_EXTRA_VAR_LIST='{ "nephio_catalog_repo_uri": "'${NEPHIO_CATALOG_REPO_URI}'", "k8s": { "context" : "'${K8S_CONTEXT}'", "version" : "'$K8S_VERSION'" } }'
 HOME=${NEPHIO_HOME:-/home/$NEPHIO_USER}
 REPO_DIR=${NEPHIO_REPO_DIR:-$HOME/test-infra}
 DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME:-""}
 DOCKERHUB_TOKEN=${DOCKERHUB_TOKEN:-""}
 FAIL_FAST=${FAIL_FAST:-$(get_metadata fail_fast "false")}
+# MGMT_CLUSTER_TYPE is intended to be set by prow jobs
+MGMT_CLUSTER_TYPE=${MGMT_CLUSTER_TYPE:-$(get_metadata mgmt_cluster_type "kind")}
+
+if [ ${MGMT_CLUSTER_TYPE} == "kubeadm" ]; then
+    K8S_CONTEXT="kubernetes-admin@kubernetes"
+fi
+export ANSIBLE_CMD_EXTRA_VAR_LIST='{ "nephio_catalog_repo_uri": "'${NEPHIO_CATALOG_REPO_URI}'", "k8s": { "context" : "'${K8S_CONTEXT}'", "version" : "'$K8S_VERSION'" } }'
+
 
 if [ ${K8S_CONTEXT} == "kind-kind" ]; then
     export ANSIBLE_TAG=all
 else
     export ANSIBLE_TAG=nonkind_k8s
 fi
 
-echo "$DEBUG, $RUN_E2E, $REPO, $BRANCH, $NEPHIO_USER, $HOME, $REPO_DIR, $DOCKERHUB_USERNAME, $DOCKERHUB_TOKEN, $ANSIBLE_TAG, $ANSIBLE_CMD_EXTRA_VAR_LIST"
+echo "$DEBUG, $RUN_E2E, $REPO, $BRANCH, $NEPHIO_USER, $HOME, $REPO_DIR, $DOCKERHUB_USERNAME, $DOCKERHUB_TOKEN, $ANSIBLE_TAG, $ANSIBLE_CMD_EXTRA_VAR_LIST $K8S_CONTEXT $MGMT_CLUSTER_TYPE"
 trap get_status ERR
 
 # Validate root permissions for current user and NEPHIO_USER
@@ -166,11 +173,11 @@ chown "$NEPHIO_USER:$NEPHIO_USER" "$HOME/.bash_aliases"
 # Sandbox Creation
 int_start=$(date +%s)
 cd "$REPO_DIR/e2e/provision"
-export DEBUG DOCKERHUB_USERNAME DOCKERHUB_TOKEN FAIL_FAST
+export DEBUG DOCKERHUB_USERNAME DOCKERHUB_TOKEN FAIL_FAST MGMT_CLUSTER_TYPE K8S_VERSION
 runuser -u "$NEPHIO_USER" ./install_sandbox.sh
 printf "%s secs\n" "$(($(date +%s) - int_start))"
 
-if [[ $RUN_E2E == "true" ]]; then
+if [[ $RUN_E2E == "true" && $MGMT_CLUSTER_TYPE == "kind" ]]; then
     runuser -u "$NEPHIO_USER" "$REPO_DIR/e2e/e2e.sh"
 fi
 

diff --git a/e2e/provision/install_sandbox.sh b/e2e/provision/install_sandbox.sh
@@ -75,6 +75,14 @@ fact_caching_connection = /tmp
 EOT
 
 # Management cluster creation
+if [[ ${MGMT_CLUSTER_TYPE:-kind} == "kubeadm" ]]; then
+    ansible_cmd_kubeadm="$(command -v ansible-playbook) -i 127.0.0.1, playbooks/deploy_kubeadm_k8s.yml --extra-vars=\"k8s_ver=${K8S_VERSION:1:4}\" "
+    [[ ${DEBUG:-false} != "true" ]] || ansible_cmd_kubeadm+="-vvv "
+    echo "$ansible_cmd_kubeadm"
+    eval "$ansible_cmd_kubeadm" | tee ~/kubeadm.log
+    echo "Done installing kubeadm cluster"
+fi
+
 ansible_cmd="$(command -v ansible-playbook) -i 127.0.0.1, playbooks/cluster.yml --tags ${ANSIBLE_TAG:-all} "
 [[ ${DEBUG:-false} != "true" ]] || ansible_cmd+="-vvv "
 if [ -n "${ANSIBLE_CMD_EXTRA_VAR_LIST:-}" ]; then

diff --git a/e2e/provision/playbooks/deploy_kubeadm_k8s.yml b/e2e/provision/playbooks/deploy_kubeadm_k8s.yml
@@ -1,5 +1,17 @@
 ---
 - name: Deploy k8s using kubeadm on host
   hosts: all
+  pre_tasks:
+    - name: Install Docker Engine needed for kpt functions
+      become: true
+      block:
+        - name: Install docker binaries
+          ansible.builtin.include_role:
+            name: andrewrothstein.docker_engine
+        - name: Grant Docker permissions to user
+          ansible.builtin.user:
+            name: "{{ ansible_user_id }}"
+            groups: docker
+            append: true
   roles:
     - role: kubeadm
diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/main.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/main.yml
@@ -206,3 +206,11 @@
   ansible.builtin.shell: kubectl taint node $(hostname) node-role.kubernetes.io/control-plane-
   register: my_output
   changed_when: my_output.rc != 0
+
+- name: Wait for deployments to complete
+  ansible.builtin.include_tasks: wait-deployments.yml
+  loop: "{{ kube-system + kube-flannel + local-path-storage }}"
+  loop_control:
+    loop_var: namespace
+  vars:
+    context: "{{ k8s.context }}"
diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/wait-deployments.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/wait-deployments.yml
@@ -0,0 +1,38 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2023 The Nephio Authors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: "Get deployment resources in namespace {{ namespace }}"
+  kubernetes.core.k8s_info:
+    context: "{{ context }}"
+    api_version: v1
+    kind: Deployment
+    namespace: "{{ namespace }}"
+  register: deployment_list
+
+- name: "Print deployment resources in namespace {{ namespace }}"
+  ansible.builtin.debug:
+    var: deployment_list.resources
+
+- name: "Wait for deployments in namespace {{ namespace }}"
+  kubernetes.core.k8s:
+    context: "{{ context }}"
+    definition:
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: "{{ resource.metadata.name }}"
+        namespace: "{{ resource.metadata.namespace }}"
+    wait: true
+    wait_condition:
+      type: Available
+      reason: MinimumReplicasAvailable
+  loop: "{{ deployment_list.resources }}"
+  loop_control:
+    loop_var: resource
diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf
@@ -2,13 +2,15 @@ module "gcp-ubuntu-focal" {
   source               = ".//modules/gcp"
   nephio_e2e_type      = var.e2e_type
   nephio_e2e_fail_fast = var.fail_fast
+  nephio_mgmt_cluster_type = var.mgmt_cluster_type
 }
 
 module "gcp-ubuntu-jammy" {
   source               = ".//modules/gcp"
   vmimage              = "ubuntu-os-cloud/ubuntu-2204-lts"
   nephio_e2e_type      = var.e2e_type
   nephio_e2e_fail_fast = var.fail_fast
+  nephio_mgmt_cluster_type = var.mgmt_cluster_type
 }
 
 module "gcp-fedora-34" {
@@ -17,6 +19,7 @@ module "gcp-fedora-34" {
   ansible_user         = "fedora"
   nephio_e2e_type      = var.e2e_type
   nephio_e2e_fail_fast = var.fail_fast
+  nephio_mgmt_cluster_type = var.mgmt_cluster_type
 }
 
 variable "e2e_type" {
@@ -30,3 +33,9 @@ variable "fail_fast" {
   default     = "false"
   type        = string
 }
+
+variable "mgmt_cluster_type" {
+  description = "Defines the type of k8s cluster"
+  default     = "kind"
+  type        = string
+}
diff --git a/e2e/terraform/modules/gcp/main.tf b/e2e/terraform/modules/gcp/main.tf
@@ -100,7 +100,7 @@ resource "google_compute_instance" "e2e_instances" {
     inline = [
       "cd /home/${var.ansible_user}/test-infra/e2e/provision/",
       "chmod +x init.sh",
-      "sudo -E FAIL_FAST=${var.nephio_e2e_fail_fast} E2ETYPE=${var.nephio_e2e_type} NEPHIO_REPO_DIR=/home/${var.ansible_user}/test-infra NEPHIO_DEBUG=true NEPHIO_RUN_E2E=true NEPHIO_USER=${var.ansible_user} ./init.sh"
+      "sudo -E FAIL_FAST=${var.nephio_e2e_fail_fast} MGMT_CLUSTER_TYPE=${var.nephio_mgmt_cluster_type} E2ETYPE=${var.nephio_e2e_type} NEPHIO_REPO_DIR=/home/${var.ansible_user}/test-infra NEPHIO_DEBUG=true NEPHIO_RUN_E2E=true NEPHIO_USER=${var.ansible_user} ./init.sh"
     ]
   }
 }
diff --git a/e2e/terraform/modules/gcp/variables.tf b/e2e/terraform/modules/gcp/variables.tf
@@ -75,3 +75,9 @@ variable "nephio_e2e_fail_fast" {
   default     = "false"
   type        = string
 }
+
+variable "nephio_mgmt_cluster_type" {
+  description = "The Nephio management cluster type"
+  default     = "kind"
+  type        = string
+}