Merge pull request #1450 from Nordix/Integrate-fakeIPA/mohammed

Add option to run Dev-env with FakeIPA

01_prepare_host.sh

@@ -141,7 +141,11 @@ case "${CONTAINER_RUNTIME}" in
   *)
     ;;
 esac
-
-# pre-pull node and container images
-# shellcheck disable=SC1091
-source lib/image_prepull.sh
+# TODO (mboukhalfa) fake images
+if [[ "${NODES_PLATFORM}" == "fake" ]]; then
+  echo "Skipping image prepulling on fake nodes platform"
+else
+  # pre-pull node and container images
+  # shellcheck disable=SC1091
+  source lib/image_prepull.sh
+fi

02_configure_host.sh

@@ -110,14 +110,15 @@ ANSIBLE_FORCE_COLOR=true "${ANSIBLE}-playbook" \
     -e "num_nodes=${NUM_NODES}" \
     -e "extradisks=${VM_EXTRADISKS}" \
     -e "virthost=${HOSTNAME}" \
-    -e "platform=${NODES_PLATFORM}" \
+    -e "vm_platform=${NODES_PLATFORM}" \
     -e "libvirt_firmware=${LIBVIRT_FIRMWARE}" \
     -e "libvirt_secure_boot=${LIBVIRT_SECURE_BOOT}" \
     -e "libvirt_domain_type=${LIBVIRT_DOMAIN_TYPE}" \
     -e "default_memory=${TARGET_NODE_MEMORY}" \
     -e "manage_external=${MANAGE_EXT_BRIDGE}" \
     -e "provisioning_url_host=${BARE_METAL_PROVISIONER_URL_HOST}" \
     -e "nodes_file=${NODES_FILE}" \
+    -e "fake_nodes_file=${FAKE_NODES_FILE}" \
     -e "node_hostname_format=${NODE_HOSTNAME_FORMAT}" \
     -i vm-setup/inventory.ini \
     -b vm-setup/setup-playbook.yml
@@ -405,8 +406,10 @@ if [[ "${BUILD_IRONIC_IMAGE_LOCALLY:-}" == "true" ]] || [[ -n "${IRONIC_LOCAL_IM
     IRONIC_IMAGE="${REGISTRY}/localimages/$(basename "${IRONIC_LOCAL_IMAGE}")"
     export IRONIC_IMAGE
 fi
-VBMC_IMAGE=${VBMC_LOCAL_IMAGE:-${VBMC_IMAGE}}
-SUSHY_TOOLS_IMAGE=${SUSHY_TOOLS_LOCAL_IMAGE:-${SUSHY_TOOLS_IMAGE}}
+VBMC_IMAGE="${VBMC_LOCAL_IMAGE:-${VBMC_IMAGE}}"
+SUSHY_TOOLS_IMAGE="${SUSHY_TOOLS_LOCAL_IMAGE:-${SUSHY_TOOLS_IMAGE}}"
+FAKE_IPA_IMAGE="${FAKE_IPA_LOCAL_IMAGE:-${FAKE_IPA_IMAGE}}"
+FKAS_IMAGE="${FKAS_LOCAL_IMAGE:-${FKAS_IMAGE}}"
 
 # Pushing images to local registry
 for IMAGE_VAR in $(env | grep -v "_LOCAL_IMAGE=" | grep "_IMAGE=" | grep -o "^[^=]*") ; do

03_launch_mgmt_cluster.sh

@@ -219,6 +219,37 @@ EOF
   popd
 }
 
+#
+# Launch and configure fakeIPA
+#
+launch_fake_ipa() {
+  # Create a folder to host fakeIPA config and certs
+  mkdir -p "${WORKING_DIR}/fake-ipa"
+  if [[ "${EPHEMERAL_CLUSTER}" == "kind" ]] && [[ "${IRONIC_TLS_SETUP}" == "true" ]]; then
+    cp "${IRONIC_CACERT_FILE}" "${WORKING_DIR}/fake-ipa/ironic-ca.crt"
+  elif [[ "${IRONIC_TLS_SETUP}" == "true" ]]; then
+    # wait for ironic to be running to ensure ironic-cert is created 
+    kubectl -n baremetal-operator-system wait --for=condition=available deployment/baremetal-operator-ironic --timeout=900s
+    # Extract ironic-cert to be used inside fakeIPA for TLS 
+    kubectl get secret -n baremetal-operator-system ironic-cert -o json -o=jsonpath="{.data.ca\.crt}" | base64 -d > "${WORKING_DIR}/fake-ipa/ironic-ca.crt"
+  fi
+  # Create fake IPA custom config
+  cat << EOF > "${WORKING_DIR}/fake-ipa/config.py"
+FAKE_IPA_API_URL = "https://${CLUSTER_BARE_METAL_PROVISIONER_IP}:${IRONIC_API_PORT}"
+FAKE_IPA_INSPECTION_CALLBACK_URL = "${IRONIC_URL}/continue_inspection"
+FAKE_IPA_ADVERTISE_ADDRESS_IP = "${EXTERNAL_SUBNET_V4_HOST}"
+FAKE_IPA_INSECURE = ${FAKE_IPA_INSECURE:-False}
+FAKE_IPA_CAFILE = "${FAKE_IPA_CAFILE:-/root/cert/ironic-ca.crt}"
+FAKE_IPA_MIN_BOOT_TIME = ${FAKE_IPA_MIN_BOOT_TIME:-20}
+FAKE_IPA_MAX_BOOT_TIME = ${FAKE_IPA_MAX_BOOT_TIME:-30}
+EOF
+  # shellcheck disable=SC2086
+  sudo "${CONTAINER_RUNTIME}" run -d --net host --name fake-ipa ${POD_NAME_INFRA} \
+    -v "/opt/metal3-dev-env/fake-ipa":/root/cert -v "/root/.ssh":/root/ssh \
+    -e CONFIG='/root/cert/config.py' \
+    "${FAKE_IPA_IMAGE}"
+}
+
 # ------------
 # BMH Creation
 # ------------
@@ -538,16 +569,20 @@ if [ "${EPHEMERAL_CLUSTER}" != "tilt" ]; then
     # Thus we are deleting validatingwebhookconfiguration resource if exists to let BMO is working properly on local runs.
     kubectl delete validatingwebhookconfiguration/"${BMO_NAME_PREFIX}"-validating-webhook-configuration --ignore-not-found=true
   fi
+
   # Tests might want to apply bmh inside the test scipt
-  # then dev-env will create the bmh files but do not apply tehm
+  # then dev-env will create the bmh files but do not apply them
   if [[ "${SKIP_APPLY_BMH:-false}" == "true" ]]; then
     pushd "${BMOPATH}"
-      list_nodes | make_bm_hosts
+    list_nodes | make_bm_hosts
     popd
   else
-    apply_bm_hosts "$NAMESPACE"
+    apply_bm_hosts "${NAMESPACE}"
+  fi
+  # if fake platform (no VMs) run FakeIPA
+  if [[ "${NODES_PLATFORM}" == "fake" ]]; then
+    launch_fake_ipa
   fi
 elif [ "${EPHEMERAL_CLUSTER}" == "tilt" ]; then
-
-source tilt-setup/deploy_tilt_env.sh
+  . tilt-setup/deploy_tilt_env.sh
 fi

04_verify.sh

@@ -245,6 +245,12 @@ echo ""
 iterate check_k8s_entity deployments "${EXPTD_DEPLOYMENTS}"
 iterate check_k8s_rs "${EXPTD_RS}"
 
+# Skip verification related to virsh when running with fakeIPA
+if [[ "${NODES_PLATFORM}" == "fake" ]]; then
+  echo "Skipping virsh nodes verification on fake vm platform"
+  exit 0
+fi
+
 # Verify the baremetal hosts
 ## Fetch the BM CRs
 RESULT_STR="Fetch Baremetalhosts"

config_example.sh

@@ -209,3 +209,7 @@
 
 # Skip applying BMHs 
 # export SKIP_APPLY_BMH="true"
+
+# To enable FakeIPA and run dev-env on a fake platform
+# export NODES_PLATFORM="fake"
+# export FAKE_IPA_IMAGE=192.168.111.1:5000/localimages/fake-ipa

host_cleanup.sh

@@ -36,6 +36,7 @@ fi
 ANSIBLE_FORCE_COLOR=true "${ANSIBLE}-playbook" \
     -e "working_dir=${WORKING_DIR}" \
     -e "num_nodes=${NUM_NODES}" \
+    -e "vm_platform=${NODES_PLATFORM}" \
     -e "extradisks=${VM_EXTRADISKS}" \
     -e "virthost=${HOSTNAME}" \
     -e "manage_external=${MANAGE_EXT_BRIDGE}" \

lib/common.sh

@@ -231,6 +231,7 @@ export CAPM3_RUN_LOCAL="${CAPM3_RUN_LOCAL:-false}"
 
 export WORKING_DIR="${WORKING_DIR:-/opt/metal3-dev-env}"
 export NODES_FILE="${NODES_FILE:-${WORKING_DIR}/ironic_nodes.json}"
+export FAKE_NODES_FILE="${FAKE_NODES_FILE:-${WORKING_DIR}/fake_nodes.json}"
 export NODES_PLATFORM="${NODES_PLATFORM:-libvirt}"
 export ANSIBLE_VENV="${ANSIBLE_VENV:-"${WORKING_DIR}/venv"}"
 # shellcheck disable=SC2034
@@ -263,6 +264,8 @@ export CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-quay.io}"
 # BMC emulator images
 export VBMC_IMAGE="${VBMC_IMAGE:-${CONTAINER_REGISTRY}/metal3-io/vbmc}"
 export SUSHY_TOOLS_IMAGE="${SUSHY_TOOLS_IMAGE:-${CONTAINER_REGISTRY}/metal3-io/sushy-tools}"
+export FAKE_IPA_IMAGE="${FAKE_IPA_IMAGE:-${CONTAINER_REGISTRY}/metal3-io/fake-ipa}"
+export FKAS_IMAGE="${FKAS_IMAGE:-${CONTAINER_REGISTRY}/metal3-io/metal3-fkas}"
 
 # CAPM3 and IPAM controller images
 if [[ "${CAPM3RELEASEBRANCH}" = "release-1.6" ]]; then

vars.md

@@ -43,6 +43,8 @@ assured that they are persisted.
 | IRONIC_IMAGE | Container image for local ironic services | | "$CONTAINER_REGISTRY/metal3-io/ironic" |
 | VBMC_IMAGE | Container image for vbmc container | | "$CONTAINER_REGISTRY/metal3-io/vbmc" |
 | SUSHY_TOOLS_IMAGE | Container image for sushy-tools container | | "$CONTAINER_REGISTRY/metal3-io/sushy-tools" |
+| FAKE_IPA_IMAGE | Container image for fakeIPA container | | "$CONTAINER_REGISTRY/metal3-io/fake-ipa" |
+| FKAS_IMAGE | Container image for fkas container | | "$CONTAINER_REGISTRY/metal3-io/metal3-fkas" |
 | CAPM3_VERSION | Version of Cluster API provider Metal3 | "v1beta1" | "v1beta1" |
 | CAPI_VERSION | Version of Cluster API | "v1beta1" | "v1beta1" |
 | CLUSTER_APIENDPOINT_IP | API endpoint IP for target cluster | "x.x.x.x" | "${EXTERNAL_SUBNET_VX}.249" |
@@ -57,12 +59,14 @@ assured that they are persisted.
 | KUBERNETES_BINARIES_CONFIG_VERSION | Version of kubelet.service and 10-kubeadm.conf files | "vx.x.x" | "v0.13.0" |
 | LIBVIRT_DOMAIN_TYPE | Which hypervisor to use for the virtual machines libvirt domain, default to kvm. It is possible to switch to qemu in case nested virtualization is not available, although it's considered experimental at this stage of development. | "kvm", "qemu" | "kvm" |
 | NUM_NODES | Set the number of virtual machines to be provisioned. This VMs will be further configured as controlplane or worker Nodes. Note that CONTROL_PLANE_MACHINE_COUNT and WORKER_MACHINE_COUNT should sum to this value. | | 2 |
+| FAKE_NODES_FILE | Path to save fake nodes generated in json file. | | "/opt/metal3-dev-env/fake_nodes.json" |
 | CONTROL_PLANE_MACHINE_COUNT | Set the controlplane replica count in the target cluster. ||1|
 | WORKER_MACHINE_COUNT | Set the worker replica count in the target cluster. ||1|
 | VM_EXTRADISKS | Add extra disks to the virtual machines provisioned. By default the size of the extra disk is set in the libvirt Ansible role to 8 GB | "true", "false" | "false" |
 | VM_EXTRADISKS_FILE_SYSTEM | Create file system to the extra disk. | "ext4", "xfs" | "ext4" |
 | VM_EXTRADISKS_MOUNT_DIR | Mount the extra disk to a directory on a host. | | "/mnt/disk2" |
 | VM_TPM_EMULATOR | Add TPM2.0 emulator to VMs. | "true", "false" | "false" |
+| NODES_PLATFORM | Select the platform used to simulate the Baremetal hosts. | "libvirt", "fake" | "libvirt" |
 | TARGET_NODE_MEMORY | Set the default memory size in MB for the virtual machines provisioned. | | 4096 |
 | CLUSTER_NAME | Set the name of the target cluster | | test1 |
 | IRONIC_TLS_SETUP | Enable TLS for Ironic and inspector | "true", "false" | "true" |

vm-setup/library/generate_macs.py

@@ -15,7 +15,6 @@
 # generate_vm_interface_macs method ripped from
 # openstack/tripleo-incubator/scripts/configure-vm
 
-import math
 import random
 
 DOCUMENTATION = '''
@@ -27,7 +26,7 @@
    - Generate a list of Ethernet MAC addresses suitable for external testing.
 '''
 
-MAX_NUM_MACS = math.trunc(0xff / 2)
+MAX_NUM_MACS = 256
 
 
 def generate_vm_interface_macs(nodes, networks):
@@ -42,7 +41,7 @@ def generate_vm_interface_macs(nodes, networks):
     # attached NIC.
     # MACs generated for a given machine will also be in sequential
     # order, which matches how most BM machines are laid out as well.
-    # Additionally we increment each MAC by two places.
+
     macs = []
     count = len(nodes) * len(networks)
 
@@ -58,10 +57,10 @@ def generate_vm_interface_macs(nodes, networks):
     base_mac = ':'.join(["%02x" % x for x in base_nums])
 
     start = random.randint(0x00, 0xff)
-    if (start + (count * 2)) > 0xff:
+    if (start + count) > 0xff:
         # leave room to generate macs in sequence
-        start = 0xff - count * 2
-    for num in range(0, count * 2, 2):
+        start = 0xff + 1 - count
+    for num in range(0, count, 1):
         mac = start + num
         macs.append(base_mac + ":" + ("%02x" % mac))
 

vm-setup/roles/common/tasks/generate_node_mac.yml

@@ -0,0 +1,7 @@
+---
+- name: get a list of MACs to use
+  generate_macs:
+    nodes: "{{ vm_nodes }}"
+    networks: "{{ networks }}"
+  register: node_mac_map
+  when: vm_nodes | length > 0

vm-setup/roles/common/tasks/write_ironic_nodes_tasks.yml

@@ -0,0 +1,7 @@
+---
+# Generate the ironic node inventory files.
+- name: Write ironic node json files
+  template:
+    src: ../templates/ironic_nodes.json.j2
+    dest: "{{ nodes_file }}"
+    force: no

vm-setup/roles/libvirt/tasks/network_setup_tasks.yml

@@ -18,11 +18,7 @@
 
 # TODO(apuimedo) drop this back to vm tasks once we have proper DNS
 - name: get a list of MACs to use
-  generate_macs:
-    nodes: "{{ vm_nodes }}"
-    networks: "{{ networks }}"
-  register: node_mac_map
-  when: vm_nodes | length > 0
+  include_tasks: ../../common/tasks/generate_node_mac.yml
 
 # Create the global, root-managed libvirt networks to which we will
 # attach the undercoud and vm virtual machines.

vm-setup/roles/libvirt/tasks/vm_setup_tasks.yml

@@ -142,17 +142,13 @@
         vm_id: "{{ vm_id|default({}) | combine ( {item.item.name: item.stdout} ) }}"
       with_items: "{{ vm_uuid.results }}"
 
-- name: set_fact BMC Driver
-  set_fact:
-    vm_driver: "{{ lookup('env', 'BMC_DRIVER') | default('mixed', true) }}"
-
-
-# Generate the ironic node inventory files.  Note that this
-# task *must* occur after the above vm tasks, because if
-# `vm_nodes` is defined the template depends on the
-# `node_mac_map` variable.
-- name: Write ironic node json files
-  template:
-    src: ../templates/ironic_nodes.json.j2
-    dest: "{{ nodes_file }}"
-    force: no
+    - name: set_fact BMC Driver
+      set_fact:
+        vm_driver: "{{ lookup('env', 'BMC_DRIVER') | default('mixed', true) }}"
+
+    # Generate the ironic node inventory files.  Note that this
+    # task *must* occur after the above vm tasks, because if
+    # `vm_nodes` is defined the template depends on the
+    # `node_mac_map` variable.
+    - name: Write ironic node json files
+      include_tasks: ../../common/tasks/write_ironic_nodes_tasks.yml

vm-setup/roles/virtbmc/tasks/setup_tasks.yml

@@ -65,6 +65,7 @@
   when: vbmc_libvirt_uri is not defined
 
 - name: Create VirtualBMC directories
+  when: vm_platform|default("libvirt") != "fake"
   file:
     path: "{{ working_dir }}/virtualbmc/vbmc/conf/{{ item.name }}"
     state: directory
@@ -75,6 +76,7 @@
   become: true
 
 - name: Create the Virtual BMCs
+  when: vm_platform|default("libvirt") != "fake"
   copy:
     mode: 0750
     dest: "{{ working_dir }}/virtualbmc/vbmc/conf/{{ item.name }}/config"
@@ -107,3 +109,47 @@
       SUSHY_EMULATOR_VMEDIA_VERIFY_SSL = {{ sushy_vmedia_verify_ssl }}
       SUSHY_EMULATOR_AUTH_FILE = "/root/sushy/htpasswd"
   become: true
+  when: vm_platform|default("libvirt") != "fake"
+
+- name: get a list of MACs to use
+  when: vm_platform|default("libvirt") == "fake"
+  include_tasks: ../../common/tasks/generate_node_mac.yml
+
+- name: Set the uuid for fake VMs
+  when: vm_platform|default("libvirt") == "fake"
+  set_fact:
+    vm_id: "{{ vm_id|default({}) | combine ( {item.name: item.name | to_uuid()} ) }}"
+  with_items: "{{ vm_nodes }}"
+
+# Define the fake vm nodes.  These will be
+# used by sushy-tools.
+- name: Define fake vms
+  when: vm_platform|default("libvirt") == "fake"
+  template:
+    src: ../templates/fake_nodes.json.j2
+    dest: "{{ fake_nodes_file }}"
+
+- name: set_fact BMC Driver
+  when: vm_platform|default("libvirt") == "fake"
+  set_fact:
+    vm_driver: "{{ lookup('env', 'BMC_DRIVER') | default('redfish', true) }}"
+
+- name: Write ironic node json files
+  when: vm_platform|default("libvirt") == "fake"
+  include_tasks: ../../common/tasks/write_ironic_nodes_tasks.yml
+
+# if FakeIPA enabled then set required sushy-tools config
+- name: Create the Redfish Virtual BMCs for FakeIPA
+  copy:
+    mode: 0750
+    dest: "{{ working_dir }}/virtualbmc/sushy-tools/conf.py"
+    content: |
+      SUSHY_EMULATOR_LIBVIRT_URI = "{{ vbmc_libvirt_uri }}"
+      SUSHY_EMULATOR_IGNORE_BOOT_DEVICE = {{ sushy_ignore_boot_device }}
+      SUSHY_EMULATOR_VMEDIA_VERIFY_SSL = {{ sushy_vmedia_verify_ssl }}
+      SUSHY_EMULATOR_AUTH_FILE = "/root/sushy/htpasswd"
+      SUSHY_EMULATOR_FAKE_DRIVER = True
+      SUSHY_EMULATOR_FAKE_IPA = True
+      SUSHY_EMULATOR_FAKE_SYSTEMS = {{ lookup('ansible.builtin.file', fake_nodes_file ) }}
+  become: true
+  when: vm_platform|default("libvirt") == "fake"

vm-setup/roles/virtbmc/templates/fake_nodes.json.j2

@@ -0,0 +1,23 @@
+[
+  {% for node in vm_nodes %}
+    {
+        'uuid': "{{ vm_id[node.name] }}",
+        "name": "{{ node.name|replace('_', '-') }}",
+        'power_state': 'Off',
+        'external_notifier': True,
+        'nics': [
+            {
+                'mac': "{{ node_mac_map.get(node.name).get(networks[0].name) }}",
+                'ip': '172.22.0.100'
+            },
+            {
+                'mac': "{{ node_mac_map.get(node.name).get(networks[1].name) }}",
+                'ip': '172.22.0.110'
+            }
+        ]
+    }
+    {% if not loop.last %}
+    ,
+    {% endif %}
+  {% endfor %}
+]

vm-setup/setup-playbook.yml

@@ -10,4 +10,4 @@
         name: libvirt
     - import_role:
         name: virtbmc
-      when: vm_platform|default("libvirt") == "libvirt"
+      when: vm_platform|default("libvirt") in ["libvirt", "fake"]