From 4f9d81c4387d55ccd967b36d055221ba7d8a42a0 Mon Sep 17 00:00:00 2001 From: student Date: Mon, 19 Aug 2024 10:28:16 -0500 Subject: [PATCH] adds prow job to install Nephio components on kubeadm cluster Co-authored-by: Daniel Kostecki --- .prow.yaml | 86 +++++++++++++++++++ e2e/provision/init.sh | 15 +++- e2e/provision/install_sandbox.sh | 8 ++ .../playbooks/deploy_kubeadm_k8s.yml | 12 +++ .../playbooks/roles/kubeadm/defaults/main.yml | 1 + .../roles/kubeadm/tasks/kubeadm-Debian.yml | 1 + .../roles/kubeadm/tasks/kubeadm-RedHat.yml | 1 + .../playbooks/roles/kubeadm/tasks/main.yml | 9 ++ .../roles/kubeadm/tasks/wait-deployments.yml | 38 ++++++++ e2e/terraform/main.tf | 18 ++++ e2e/terraform/modules/gcp/main.tf | 2 +- e2e/terraform/modules/gcp/variables.tf | 6 ++ 12 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 e2e/provision/playbooks/roles/kubeadm/tasks/wait-deployments.yml diff --git a/.prow.yaml b/.prow.yaml index bf892f6e..ce16dd16 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -289,6 +289,92 @@ presubmits: path: id_rsa - key: id_rsa.pub path: id_rsa.pub + - name: e2e-kubeadm-ubuntu-jammy + annotations: + labels: + run_if_changed: '^e2e/' + skip_report: false + optional: true + decorate: true + cluster: default + spec: + containers: + - image: "nephio/e2e:latest" + command: + - "/bin/sh" + args: + - "-c" + - | + set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-ubuntu-jammy -auto-approve' EXIT; + terraform init && timeout 120m terraform apply -target module.gcp-ubuntu-jammy -var="fail_fast=true" -var="mgmt_cluster_type=kubeadm" -auto-approve + volumeMounts: + - name: satoken + mountPath: "/etc/satoken" + - name: ssh-key-vol + mountPath: "/etc/ssh-key" + resources: + requests: + cpu: 2 + memory: 2Gi + volumes: + - name: satoken + secret: + secretName: satoken + items: + - key: satoken + path: satoken + - name: ssh-key-vol + secret: + secretName: ssh-key-e2e + defaultMode: 256 + items: + - key: id_rsa + path: id_rsa + - key: id_rsa.pub + path: id_rsa.pub + - name: e2e-kubeadm-fedora-40 + annotations: + labels: + run_if_changed: '^e2e/' + skip_report: false + optional: true + decorate: true + cluster: default + spec: + containers: + - image: "nephio/e2e:latest" + command: + - "/bin/sh" + args: + - "-c" + - | + set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-fedora-34 -auto-approve' EXIT; + terraform init && timeout 120m terraform apply -target module.gcp-fedora-40 -var="fail_fast=true" -var="mgmt_cluster_type=kubeadm" -auto-approve + volumeMounts: + - name: satoken + mountPath: "/etc/satoken" + - name: ssh-key-vol + mountPath: "/etc/ssh-key" + resources: + requests: + cpu: 2 + memory: 2Gi + volumes: + - name: satoken + secret: + secretName: satoken + items: + - key: satoken + path: satoken + - name: ssh-key-vol + secret: + secretName: ssh-key-e2e + defaultMode: 256 + items: + - key: id_rsa + path: id_rsa + - key: id_rsa.pub + path: id_rsa.pub postsubmits: - name: build-push-image-releaser cluster: default diff --git a/e2e/provision/init.sh b/e2e/provision/init.sh index 4b382542..c1cb689f 100755 --- a/e2e/provision/init.sh +++ b/e2e/provision/init.sh @@ -74,12 +74,19 @@ NEPHIO_USER=${NEPHIO_USER:-$(get_metadata nephio-user "${USER:-ubuntu}")} NEPHIO_CATALOG_REPO_URI=${NEPHIO_CATALOG_REPO_URI:-$(get_metadata nephio-catalog-repo-uri "https://github.com/nephio-project/catalog.git")} K8S_CONTEXT=${K8S_CONTEXT:-"kind-kind"} K8S_VERSION=${K8S_VERSION:-"v1.29.2"} -export ANSIBLE_CMD_EXTRA_VAR_LIST='{ "nephio_catalog_repo_uri": "'${NEPHIO_CATALOG_REPO_URI}'", "k8s": { "context" : "'${K8S_CONTEXT}'", "version" : "'$K8S_VERSION'" } }' HOME=${NEPHIO_HOME:-/home/$NEPHIO_USER} REPO_DIR=${NEPHIO_REPO_DIR:-$HOME/test-infra} DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME:-""} DOCKERHUB_TOKEN=${DOCKERHUB_TOKEN:-""} FAIL_FAST=${FAIL_FAST:-$(get_metadata fail_fast "false")} +# MGMT_CLUSTER_TYPE is intended to be set by prow jobs +MGMT_CLUSTER_TYPE=${MGMT_CLUSTER_TYPE:-$(get_metadata mgmt_cluster_type "kind")} + +if [ ${MGMT_CLUSTER_TYPE} == "kubeadm" ]; then + K8S_CONTEXT="kubernetes-admin@kubernetes" +fi +export ANSIBLE_CMD_EXTRA_VAR_LIST='{ "nephio_catalog_repo_uri": "'${NEPHIO_CATALOG_REPO_URI}'", "k8s": { "context" : "'${K8S_CONTEXT}'", "version" : "'$K8S_VERSION'" } }' + if [ ${K8S_CONTEXT} == "kind-kind" ]; then export ANSIBLE_TAG=all @@ -87,7 +94,7 @@ else export ANSIBLE_TAG=nonkind_k8s fi -echo "$DEBUG, $RUN_E2E, $REPO, $BRANCH, $NEPHIO_USER, $HOME, $REPO_DIR, $DOCKERHUB_USERNAME, $DOCKERHUB_TOKEN, $ANSIBLE_TAG, $ANSIBLE_CMD_EXTRA_VAR_LIST" +echo "$DEBUG, $RUN_E2E, $REPO, $BRANCH, $NEPHIO_USER, $HOME, $REPO_DIR, $DOCKERHUB_USERNAME, $DOCKERHUB_TOKEN, $ANSIBLE_TAG, $ANSIBLE_CMD_EXTRA_VAR_LIST $K8S_CONTEXT $MGMT_CLUSTER_TYPE" trap get_status ERR # Validate root permissions for current user and NEPHIO_USER @@ -166,11 +173,11 @@ chown "$NEPHIO_USER:$NEPHIO_USER" "$HOME/.bash_aliases" # Sandbox Creation int_start=$(date +%s) cd "$REPO_DIR/e2e/provision" -export DEBUG DOCKERHUB_USERNAME DOCKERHUB_TOKEN FAIL_FAST +export DEBUG DOCKERHUB_USERNAME DOCKERHUB_TOKEN FAIL_FAST MGMT_CLUSTER_TYPE K8S_VERSION runuser -u "$NEPHIO_USER" ./install_sandbox.sh printf "%s secs\n" "$(($(date +%s) - int_start))" -if [[ $RUN_E2E == "true" ]]; then +if [[ $RUN_E2E == "true" && $MGMT_CLUSTER_TYPE == "kind" ]]; then runuser -u "$NEPHIO_USER" "$REPO_DIR/e2e/e2e.sh" fi diff --git a/e2e/provision/install_sandbox.sh b/e2e/provision/install_sandbox.sh index 0bec1d15..9bc894d3 100755 --- a/e2e/provision/install_sandbox.sh +++ b/e2e/provision/install_sandbox.sh @@ -75,6 +75,14 @@ fact_caching_connection = /tmp EOT # Management cluster creation +if [[ ${MGMT_CLUSTER_TYPE:-kind} == "kubeadm" ]]; then + ansible_cmd_kubeadm="$(command -v ansible-playbook) -i 127.0.0.1, playbooks/deploy_kubeadm_k8s.yml --extra-vars=\"k8s_ver=${K8S_VERSION:1:4}\" " + [[ ${DEBUG:-false} != "true" ]] || ansible_cmd_kubeadm+="-vvv " + echo "$ansible_cmd_kubeadm" + eval "$ansible_cmd_kubeadm" | tee ~/kubeadm.log + echo "Done installing kubeadm cluster" +fi + ansible_cmd="$(command -v ansible-playbook) -i 127.0.0.1, playbooks/cluster.yml --tags ${ANSIBLE_TAG:-all} " [[ ${DEBUG:-false} != "true" ]] || ansible_cmd+="-vvv " if [ -n "${ANSIBLE_CMD_EXTRA_VAR_LIST:-}" ]; then diff --git a/e2e/provision/playbooks/deploy_kubeadm_k8s.yml b/e2e/provision/playbooks/deploy_kubeadm_k8s.yml index b31e4b13..96573aab 100644 --- a/e2e/provision/playbooks/deploy_kubeadm_k8s.yml +++ b/e2e/provision/playbooks/deploy_kubeadm_k8s.yml @@ -1,5 +1,17 @@ --- - name: Deploy k8s using kubeadm on host hosts: all + pre_tasks: + - name: Install Docker Engine needed for kpt functions + become: true + block: + - name: Install docker binaries + ansible.builtin.include_role: + name: andrewrothstein.docker_engine + - name: Grant Docker permissions to user + ansible.builtin.user: + name: "{{ ansible_user_id }}" + groups: docker + append: true roles: - role: kubeadm diff --git a/e2e/provision/playbooks/roles/kubeadm/defaults/main.yml b/e2e/provision/playbooks/roles/kubeadm/defaults/main.yml index 2dfe117e..22282e03 100644 --- a/e2e/provision/playbooks/roles/kubeadm/defaults/main.yml +++ b/e2e/provision/playbooks/roles/kubeadm/defaults/main.yml @@ -1,3 +1,4 @@ +--- containerd_ver: 1.6.19 runc_ver: 1.1.1 cni_plugins_ver: 1.1.1 diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-Debian.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-Debian.yml index c4522b25..b99929e9 100644 --- a/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-Debian.yml +++ b/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-Debian.yml @@ -1,3 +1,4 @@ +--- - name: Install kubelet kubeadm kubectl on {{ ansible_os_family }} become: true block: diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-RedHat.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-RedHat.yml index 0603622d..e70bbdcd 100644 --- a/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-RedHat.yml +++ b/e2e/provision/playbooks/roles/kubeadm/tasks/kubeadm-RedHat.yml @@ -1,3 +1,4 @@ +--- - name: Install kubelet kubeadm kubectl on {{ ansible_os_family }} become: true block: diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/main.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/main.yml index 28c16649..3d8df3c0 100644 --- a/e2e/provision/playbooks/roles/kubeadm/tasks/main.yml +++ b/e2e/provision/playbooks/roles/kubeadm/tasks/main.yml @@ -1,3 +1,4 @@ +--- - name: Get containerd and unarchive to /usr/local become: true ansible.builtin.unarchive: @@ -206,3 +207,11 @@ ansible.builtin.shell: kubectl taint node $(hostname) node-role.kubernetes.io/control-plane- register: my_output changed_when: my_output.rc != 0 + +- name: Wait for deployments to complete + ansible.builtin.include_tasks: wait-deployments.yml + loop: "{{ kube-system + kube-flannel + local-path-storage }}" + loop_control: + loop_var: namespace + vars: + context: "{{ k8s.context }}" diff --git a/e2e/provision/playbooks/roles/kubeadm/tasks/wait-deployments.yml b/e2e/provision/playbooks/roles/kubeadm/tasks/wait-deployments.yml new file mode 100644 index 00000000..4d9e5e02 --- /dev/null +++ b/e2e/provision/playbooks/roles/kubeadm/tasks/wait-deployments.yml @@ -0,0 +1,38 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 The Nephio Authors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: "Get deployment resources in namespace {{ namespace }}" + kubernetes.core.k8s_info: + context: "{{ context }}" + api_version: v1 + kind: Deployment + namespace: "{{ namespace }}" + register: deployment_list + +- name: "Print deployment resources in namespace {{ namespace }}" + ansible.builtin.debug: + var: deployment_list.resources + +- name: "Wait for deployments in namespace {{ namespace }}" + kubernetes.core.k8s: + context: "{{ context }}" + definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: "{{ resource.metadata.name }}" + namespace: "{{ resource.metadata.namespace }}" + wait: true + wait_condition: + type: Available + reason: MinimumReplicasAvailable + loop: "{{ deployment_list.resources }}" + loop_control: + loop_var: resource diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf index 2ec206e5..a3527d91 100644 --- a/e2e/terraform/main.tf +++ b/e2e/terraform/main.tf @@ -2,6 +2,7 @@ module "gcp-ubuntu-focal" { source = ".//modules/gcp" nephio_e2e_type = var.e2e_type nephio_e2e_fail_fast = var.fail_fast + nephio_mgmt_cluster_type = var.mgmt_cluster_type } module "gcp-ubuntu-jammy" { @@ -9,6 +10,7 @@ module "gcp-ubuntu-jammy" { vmimage = "ubuntu-os-cloud/ubuntu-2204-lts" nephio_e2e_type = var.e2e_type nephio_e2e_fail_fast = var.fail_fast + nephio_mgmt_cluster_type = var.mgmt_cluster_type } module "gcp-fedora-34" { @@ -17,6 +19,16 @@ module "gcp-fedora-34" { ansible_user = "fedora" nephio_e2e_type = var.e2e_type nephio_e2e_fail_fast = var.fail_fast + nephio_mgmt_cluster_type = var.mgmt_cluster_type +} + +module "gcp-fedora-40" { + source = ".//modules/gcp" + vmimage = "fedora-cloud/fedora-cloud-40" + ansible_user = "fedora" + nephio_e2e_type = var.e2e_type + nephio_e2e_fail_fast = var.fail_fast + nephio_mgmt_cluster_type = var.mgmt_cluster_type } variable "e2e_type" { @@ -30,3 +42,9 @@ variable "fail_fast" { default = "false" type = string } + +variable "mgmt_cluster_type" { + description = "Defines the type of k8s cluster" + default = "kind" + type = string +} diff --git a/e2e/terraform/modules/gcp/main.tf b/e2e/terraform/modules/gcp/main.tf index 8432fb03..a799b9fb 100644 --- a/e2e/terraform/modules/gcp/main.tf +++ b/e2e/terraform/modules/gcp/main.tf @@ -100,7 +100,7 @@ resource "google_compute_instance" "e2e_instances" { inline = [ "cd /home/${var.ansible_user}/test-infra/e2e/provision/", "chmod +x init.sh", - "sudo -E FAIL_FAST=${var.nephio_e2e_fail_fast} E2ETYPE=${var.nephio_e2e_type} NEPHIO_REPO_DIR=/home/${var.ansible_user}/test-infra NEPHIO_DEBUG=true NEPHIO_RUN_E2E=true NEPHIO_USER=${var.ansible_user} ./init.sh" + "sudo -E FAIL_FAST=${var.nephio_e2e_fail_fast} MGMT_CLUSTER_TYPE=${var.nephio_mgmt_cluster_type} E2ETYPE=${var.nephio_e2e_type} NEPHIO_REPO_DIR=/home/${var.ansible_user}/test-infra NEPHIO_DEBUG=true NEPHIO_RUN_E2E=true NEPHIO_USER=${var.ansible_user} ./init.sh" ] } } diff --git a/e2e/terraform/modules/gcp/variables.tf b/e2e/terraform/modules/gcp/variables.tf index b41fc0d6..5300a778 100644 --- a/e2e/terraform/modules/gcp/variables.tf +++ b/e2e/terraform/modules/gcp/variables.tf @@ -75,3 +75,9 @@ variable "nephio_e2e_fail_fast" { default = "false" type = string } + +variable "nephio_mgmt_cluster_type" { + description = "The Nephio management cluster type" + default = "kind" + type = string +}