Adding playbook to install and configure k8s nodes

ansible-collection-requirements.yml

@@ -2,6 +2,9 @@ collections:
   - name: https://opendev.org/openstack/ansible-collections-openstack
     version: 2.2.0
     type: git
+  - name: https://github.com/ansible-collections/ansible.posix
+    version: 1.5.4
+    type: git
   - name: https://github.com/ansible-collections/community.general
     version: 8.2.0
     type: git

ansible/inventory/openstack-enterprise/group_vars/k8s_cluster/k8s-cluster.yml

@@ -157,7 +157,9 @@ kube_encrypt_secret_data: true
 
 # DNS configuration.
 # Kubernetes cluster name, also will be used as DNS domain
-cluster_name: cluster.local
+# Set via inventory
+#cluster_name: cluster.local
+#
 # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
 ndots: 2
 # dns_timeout: 2

ansible/inventory/openstack-flex/group_vars/k8s_cluster/k8s-cluster.yml

@@ -157,7 +157,9 @@ kube_encrypt_secret_data: false
 
 # DNS configuration.
 # Kubernetes cluster name, also will be used as DNS domain
-cluster_name: cluster.local
+# Set via inventory
+#cluster_name: cluster.local
+
 # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
 ndots: 2
 # dns_timeout: 2

ansible/playbooks/setup-kubernetes.yml

@@ -0,0 +1,25 @@
+---
+# Copyright 2024-Present, Rackspace Technology, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: localhost
+  become: True
+  gather_facts: "{{ gather_facts | default(true) }}"
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  vars_files:
+    - 'vars/default.yml'
+  roles:
+    - role: "k8s_install"
+      pre_execution_hook: "source /opt/genestack/scripts/genestack.rc"
+      kubeprovider: "{{ kube_installer }}"

ansible/playbooks/vars/default.yml

@@ -0,0 +1,9 @@
+---
+
+# General settings
+ansible_forks: "{{ (hostvars[inventory_hostname]['ansible_processor_nproc'] ** 2 |round(0,'floor') |int }}"
+async_timeout: 4500 #75 minutes
+
+kube_installer:
+  name: "{{ lookup('env','K8S_PROVIDER') |default('kubespray') }}"
+  path: "/opt/genestack/submodules/kubespray"

ansible/roles/k8s_install/README.md

@@ -0,0 +1,59 @@
+Role k8s_install
+================
+
+Role to install k8s distributions and apply configurations tasks post install including
+
+- Pull kubctl configuration from the first cluster node
+- Node labeling (install only)
+
+A kubespray like inventory is expected, supplying at a minimum:
+
+`k8s_cluster` setting `cluster_name` and consisting of the children
+`kube_control_plane`, `kube_node`.
+
+The `cluster_name` variable must be configured to a desired FQDN, outside of  `cluster.local` for obvious reasons
+such a tld `.local` can not utilize EV certificates.
+
+At this time only the kubespray installer is supported, others can be added over time.
+The override `kube_install_mode` defaults to `install` which utilizes the kubespray `cluster.yml`.
+By setting this override to `upgrade`, the role delegates to the `upgrade.yml` playbook preceded by a version check.
+Other modes such as scaling out, increasing k8s node count, will be added over time.
+
+Requirements
+------------
+
+- Supplied kubespray code to run the cluster playbook
+
+Role Variables
+--------------
+
+See [defaults](defaults/main.yml)
+
+
+Dependencies
+------------
+
+N/A
+
+
+Example Playbook
+----------------
+
+```shell
+- hosts: localhost
+  become: True
+  gather_facts: "{{ gather_facts | default(true) }}"
+  vars_files:
+    - 'vars/default.yml'
+  roles:
+    - role: "k8s_install"
+      pre_execution_hook: "source env.rc"
+      kubeprovider:
+        name: "kubespray"
+        path: "/opt/kubespray"
+```
+
+License
+-------
+
+[![Apache License, Version 2.0](https://img.shields.io/badge/License-Apache-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)

ansible/roles/k8s_install/defaults/main.yml

@@ -0,0 +1,26 @@
+---
+
+ansible_forks: 8
+async_timeout: 4500 #75 minutes
+
+# Base kubernets parameters
+kube_install_mode: "install" # install | upgrade | scaleout
+supported_kube_installer:
+  - "kubespray"
+
+# Command which gets executed prior to the
+# installation, if utilized inside the install
+# or upgrade task
+pre_execution_hook:
+
+# How to build the kubernetes client configuration
+#
+# retrieve: Get the configuration from the first api host
+# authorize: TODO setup a local kubernetes user
+kubeconfig_file: "{{ lookup('env','HOME') |default('/root') }}/.kube/config"
+kubeconfig_mode: "retrieve"
+
+# Which k8s installer to use
+kubeprovider:
+  name: 'kubespray'
+  path: '/opt/kubespray'

ansible/roles/k8s_install/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for k8s_install

ansible/roles/k8s_install/meta/main.yml

@@ -0,0 +1,16 @@
+galaxy_info:
+  author: Bjoern Teipel
+  description: A light weight wrapper to install k3s from a kubespray-like Ansible inventory
+  company: Rackspace Technology
+  license: Apache-2.0
+  min_ansible_version: 2.15.8
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

ansible/roles/k8s_install/tasks/k8s_client.yml

@@ -0,0 +1,59 @@
+---
+# Copyright 2024, Rackspace Technology, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Not intended to run stand alone, please run within a playbook
+# and expose parameters as listed below:
+#
+# - Loaded kubespray like inventory
+# - cluster_name
+
+- name: Create kubectl config directory
+  ansible.builtin.file:
+    path: "{{ lookup('env','HOME') }}/.kube/{{ cluster_name }}"
+    state: directory
+    recurse: True
+    mode: '0700'
+
+- name: Get kubernetes client
+  ansible.posix.synchronize:
+    mode: pull
+    src: "{{ item.user }}@{{ item.host }}:/usr/local/bin/kubectl"
+    dest: "/usr/local/bin/kubectl"
+  loop:
+    - user: "{{ ansible_become_user |default('root') }}"
+      host: "{{ groups['k8s_cluster'][0] }}"
+
+- block:
+  - name: Get kubernetes config
+    ansible.posix.synchronize:
+      mode: pull
+      src: "{{ item.user }}@{{ item.host }}:/root/.kube/config"
+      dest: "{{ lookup('env','HOME') }}/.kube/{{ cluster_name }}/"
+      #dest: "{{ lookup('env','HOME') }}/.kube/"
+    loop:
+      - user: "{{ ansible_become_user |default('root') }}"
+        host: "{{ groups['k8s_cluster'][0] }}"
+
+  - name: Set path to kubectl config
+    ansible.builtin.set_fact:
+      kubeconfig_file: "{{ lookup('env','HOME') }}/.kube/{{ cluster_name }}/config"
+
+  - name: "Display configuration"
+    ansible.builtin.debug:
+      msg: "{{ item.name }}: {{ item.value }}"
+    loop:
+      - name: "Kubectl Config (updated)"
+        value: "{{ kubeconfig_file }}"

ansible/roles/k8s_install/tasks/k8s_label_nodes.yml

@@ -0,0 +1,59 @@
+---
+# Copyright 2024-Present, Rackspace Technology, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "Map node to label/role"
+  ansible.builtin.set_fact:
+    kubectl_options: |
+      {% set _kctl = kubectl_options |default([]) %}
+      {% if (item.hosts |length) > 0 %}
+      {%   for node in item.hosts %}
+      {%     set _label = "" %}
+      {%     set _role = "" %}
+      {%     set _node = "label node " + node + " " %}
+      {%     if item.label is defined %}
+      {%       set _label = item.label %}
+      {%     endif %}
+      {%     if item.role is defined %}
+      {%       set _role = " role=" + item.role %}
+      {%     endif %}
+      {%     if _kctl.append(_node + _label + _role) %}{% endif %}
+      {%   endfor %}
+      {% endif %}
+      {{ _kctl }}
+  loop:
+    - hosts: "{{ groups['openstack_control_plane'] }}"
+      label: "openstack-control-plane=enabled"
+    - hosts: "{{ groups['openstack_compute_nodes'] }}"
+      label: "openstack-compute-node=enabled"
+    - hosts: "{{ groups['openstack_network_nodes'] |default([]) }}"
+      label: "openstack-network-node=enabled"
+    - hosts: "{{ groups['ovn_network_nodes'] |default([]) }}"
+      label: "openstack-network-node=enabled"
+    - hosts: "{{ groups['openstack_storage_nodes'] |default([]) }}"
+      label: "openstack-storage-plane=enabled"
+    - hosts: "{{ groups['cinder_storage_nodes'] |default([]) }}"
+      label: "openstack-storage-node=enabled"
+    - hosts: "{{ groups['storage_nodes'] |default([]) }}"
+      role: "storage-node"
+    - hosts: "{{ groups['ceph_storage_nodes'] |default([]) }}"
+      label: "ceph-storage-node=enabled"
+      role: "storage-node"
+
+- name: "Apply label"
+  ansible.builtin.shell: |
+    kubectl --kubeconfig="{{ kubeconfig_file }}" {{ item }}
+  args:
+    executable: /bin/bash
+  loop: "{{ kubectl_options }}"

ansible/roles/k8s_install/tasks/kubespray_install.yml

@@ -0,0 +1,42 @@
+---
+# Copyright 2024-Present, Rackspace Technology, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "Execute kubespray installer"
+  block:
+    - name: "Output"
+      debug:
+        msg:
+          - "Executing kubespray installer in mode {{ kube_install_mode }}"
+          - "Output will be send to /var/log/kubespray.log"
+          - "Max timeout set to {{ async_timeout }} seconds"
+
+    - name: "Execute kubespray cluster.yml"
+      ansible.builtin.shell: |
+        set -o pipefail #
+        {% if pre_execution_hook is defined %}
+        {{ pre_execution_hook }}
+        {% endif %}
+        ansible-playbook --forks {{ ansible_forks }} cluster.yml 2>&1 |tee -a /var/log/kubespray.log
+        echo -e "\nRC: $?" |tee -a /var/log/kubespray.log
+      args:
+        executable: /bin/bash
+        chdir: "{{ kubeprovider.path }}"
+      register: _kubernetes_install
+      changed_when:
+        - "_kubernetes_install.rc == 0"
+        - "_kubernetes_install.stdout |regex_search('changed=(1[0-9]*)')"
+      failed_when: _kubernetes_install.rc not in [0]
+      async: "{{ async_timeout }}"
+      poll: 5

ansible/roles/k8s_install/tasks/kubespray_upgrade.yml

@@ -0,0 +1,42 @@
+---
+# Copyright 2024-Present, Rackspace Technology, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "Execute kubespray upgrade"
+  block:
+    - name: "Output"
+      debug:
+        msg:
+          - "Executing kubespray installer in mode {{ kube_install_mode }}"
+          - "Output will be send to /var/log/kubespray-upgrade.log"
+          - "Max timeout set to {{ async_timeout }} seconds"
+
+    - name: "Execute kubespray upgrade-cluster.yml"
+      ansible.builtin.shell: |
+        set -o pipefail #
+        {% if pre_execution_hook is defined %}
+        {{ pre_execution_hook }}
+        {% endif %}
+        ansible-playbook --forks {{ ansible_forks }} upgrade-cluster.yml 2>&1 |tee -a /var/log/kubespray-upgrade.log
+        echo -e "\nRC: $?" |tee -a /var/log/kubespray-upgrade.log
+      args:
+        executable: /bin/bash
+        chdir: "{{ kubeprovider.path }}"
+      register: _kubernetes_install
+      changed_when:
+        - "_kubernetes_install.rc == 0"
+        - "_kubernetes_install.stdout |regex_search('changed=(1[0-9]*)')"
+      failed_when: _kubernetes_install.rc not in [0]
+      async: "{{ async_timeout }}"
+      poll: 5