linux: add fuego multinode

Signed-off-by: Chase Qi <[email protected]>

automated/linux/fuego-multinode/boards/generic-arm64.board

@@ -0,0 +1,15 @@
+inherit "base-board"
+include "base-params"
+
+MACHINE=generic-arm64
+
+TRANSPORT="ssh"
+IPADDR="dut_ipaddr"
+SSH_PORT="22"
+LOGIN="root"
+SSH_KEY="/fuego-rw/ssh_keys/fuego-lava"
+# PASSWORD="fuegotest"
+BOARD_TESTDIR="/root/"
+
+TOOLCHAIN="arm64"
+ARCHITECTURE="arm64"

automated/linux/fuego-multinode/boards/generic-armhf.board

@@ -0,0 +1,15 @@
+inherit "base-board"
+include "base-params"
+
+MACHINE=generic-armhf
+
+TRANSPORT="ssh"
+IPADDR="dut_ipaddr"
+SSH_PORT="22"
+LOGIN="root"
+SSH_KEY="/fuego-rw/ssh_keys/fuego-lava"
+# PASSWORD="fuegotest"
+BOARD_TESTDIR="/root/"
+
+TOOLCHAIN="debian-armhf"
+ARCHITECTURE="arm"

automated/linux/fuego-multinode/boards/generic-x86_64.board

@@ -0,0 +1,15 @@
+inherit "base-board"
+include "base-params"
+
+MACHINE=generic-x86_64
+
+TRANSPORT="ssh"
+IPADDR="dut_ipaddr"
+SSH_PORT="22"
+LOGIN="root"
+SSH_KEY="/fuego-rw/ssh_keys/fuego-lava"
+# PASSWORD="fuegotest"
+BOARD_TESTDIR="/root/"
+
+TOOLCHAIN="x86_64"
+ARCHITECTURE="x86_64"

automated/linux/fuego-multinode/fuego-dut.yaml

@@ -0,0 +1,24 @@
+metadata:
+    format: Lava-Test Test Definition 1.0
+    name: fuego-dut
+    description: "Run fuego tests with LAVA. Refer to http://fuegotest.org/"
+    maintainer:
+        - [email protected]
+    os:
+        - debian
+    scope:
+        - functional
+    devices:
+        - rpi3-b-32
+        - beaglebone-black
+
+params:
+    INTERFACE: "eth0"
+
+run:
+    steps:
+        - cd ./automated/linux/fuego-multinode/
+        - lava-wait ssh-pub-key
+        - ./setup-openssh-server.sh || sleep 3600
+        - lava-send dut-ready ipaddr=$(lava-echo-ipv4 ${INTERFACE})
+        - lava-wait host-done

automated/linux/fuego-multinode/fuego-host.yaml

@@ -0,0 +1,51 @@
+metadata:
+    format: Lava-Test Test Definition 1.0
+    name: fuego-host
+    description: "Run fuego tests with LAVA. Refer to http://fuegotest.org/"
+    maintainer:
+        - [email protected]
+    os:
+        - debian
+    scope:
+        - functional
+    devices:
+        - docker
+
+params:
+    BOARD: "generic-armhf"
+    TOOLCHAIN: "armhf"
+    TEST: "Functional.hello_world"
+    SPEC: "default"
+    # arm64, armhf and x86_64 toolchains are included in fuego docker image
+    # already, skip toolchain installation by default.
+    # Refer to the below docker image description for build instructions:
+    # https://cloud.docker.com/repository/docker/chaseqi/standalone-fuego
+    SKIP_INSTALL: "true"
+    # The following variables just work when only one test action defined in
+    # LAVA test job. When more then one test defined:
+    # Set WAIT_DUT to "true" for the first test and "false" for the rest tests.
+    # Set HOST_DONE to "true" for the last test and "false" for the rest tests.
+    WAIT_DUT: "true"
+    HOST_DONE: "true"
+
+run:
+    steps:
+        # Sent host-done signal on non-zero exit.
+        - trap 'lava-send host-done' ERR
+        - |
+            # When more then on test defined, only send ssh pub key once.
+            if [ -f /fuego-rw/ssh_keys/fuego-lava ]; then
+                echo "SSH key pairs should be set already."
+            else
+                mkdir -p /fuego-rw/ssh_keys
+                ssh-keygen -t rsa -N "" -f /fuego-rw/ssh_keys/fuego-lava -C fuego-lava
+                # Message sent by lava-send is visible, when security is a
+                # concern, job visibility should be set to private/group.
+                lava-send ssh-pub-key pub_key=$(cat /fuego-rw/ssh_keys/fuego-lava.pub | awk '{print $2}')
+            fi
+        - cd ./automated/linux/fuego-multinode/
+        - if "${WAIT_DUT}"; then lava-wait dut-ready; fi
+        - ./fuego.sh -b "${BOARD}" -c "${TOOLCHAIN}" -t "${TEST}" -s "${SPEC}" -S "${SKIP_INSTALL}" || ret_val=$?
+        - ../../utils/send-to-lava.sh ./output/result.txt
+        - if "${HOST_DONE}"; then lava-send host-done; fi
+        - if [ -n "${ret_val}" ] && [ "${ret_val}" -ne 0 ]; then exit 1; fi

automated/linux/fuego-multinode/fuego.sh

@@ -0,0 +1,95 @@
+#!/bin/sh -ex
+
+TEST_DIR=$(dirname "$(realpath "$0")")
+OUTPUT="${TEST_DIR}/output"
+RESULT_FILE="${OUTPUT}/result.txt"
+export RESULT_FILE
+BOARD="generic-armhf"
+TOOLCHAIN="armhf"
+TEST="Functional.hello_world"
+SPEC="default"
+SKIP_INSTALL="false"
+
+usage() {
+    echo "Usage: $0 [-b BOARD] [-c TOOLCHAIN] [-t TEST] [-s SPEC] [-S SKIP_INSTALL]" 1>&2
+    exit 1
+}
+
+while getopts "b:c:t:s:S:h" o; do
+  case "$o" in
+    b) BOARD="${OPTARG}" ;;
+    c) TOOLCHAIN="${OPTARG}" ;;
+    t) TEST="${OPTARG}" ;;
+    s) SPEC="${OPTARG}" ;;
+    S) SKIP_INSTALL="${OPTARG}" ;;
+    h|*) usage ;;
+  esac
+done
+
+# shellcheck disable=SC1090
+. "${TEST_DIR}/../../lib/sh-test-lib"
+create_out_dir "${OUTPUT}"
+
+# Install toolchain.
+# Refer to http://fuegotest.org/wiki/Adding_a_toolchain for supported
+# toolchains by the following installer.
+if [ "${SKIP_INSTALL}" = "true" ] || [ "${SKIP_INSTALL}" = "True" ]; then
+    info_msg "Toolchain ${TOOLCHAIN} installation skipped."
+else
+   /fuego-ro/toolchains/install_cross_toolchain.sh "${TOOLCHAIN}"
+fi
+
+# Add board configuration.
+# FIXME: changes in LAVA are required to support additional docker run params.
+# fuego uses '--net="host"' to make fuego host accessible for DUT. The feature
+# is required by networking tests like NetPIPE, iperf, netperf
+dut_ipaddr=$(grep "ipaddr" /tmp/lava_multi_node_cache.txt | awk -F"=" '{print $NF}')
+board_config="/fuego-ro/boards/${BOARD}.board"
+if [ -f "${board_config}" ] && grep "${dut_ipaddr}" "${board_config}"; then
+    info_msg "Board configuration already added."
+else
+    sed -i "s/dut_ipaddr/${dut_ipaddr}/" "boards/${BOARD}.board"
+    cp "boards/${BOARD}.board" /fuego-ro/boards/
+    cat "/fuego-ro/boards/${BOARD}.board"
+fi
+
+# Set proper permissions.
+chown -R jenkins.jenkins /fuego-rw/
+chown -R jenkins.jenkins /fuego-ro/
+
+# Add Jenkins node.
+# Give Jenkins time to start.
+sleep 30
+if ftc list-nodes | grep "${BOARD}"; then
+    info_msg "Node ${BOARD} already added."
+else
+    ftc add-nodes -b "${BOARD}" -f
+fi
+
+# Add test job.
+if ftc list-jobs | grep "${BOARD}.${SPEC}.${TEST}"; then
+    info_msg "Test job ${BOARD}.${SPEC}.${TEST} already added."
+else
+    ftc add-job -b "${BOARD}" -t "${TEST}" -s "${SPEC}"
+fi
+
+# Run test as user jenkins.
+# timeout will be handled by LAVA. Set a super long time here.
+# TODO: support dynamic-vars
+ret_val=0
+sudo -u jenkins ftc run-test -b "${BOARD}" -t "${TEST}" -s "${SPEC}" \
+    --precleanup true \
+    --postcleanup false \
+    --rebuild false \
+    --reboot false \
+    --timeout 1d || ret_val=$?
+
+# Parse result file run.json.
+log_dir=$(find "/fuego-rw/logs/${TEST}/${BOARD}.${SPEC}"* -maxdepth 0 -type d | sort | tail -n 1)
+python "${TEST_DIR}/parser.py" -s "${log_dir}/run.json" -d "${RESULT_FILE}"
+
+if [ "${ret_val}" -ne 0 ]; then
+    exit 1
+else
+    exit 0
+fi

automated/linux/fuego-multinode/parser.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-s', '--source', dest='source', required=True,
+                    help='path to fuego test result file run.json.')
+parser.add_argument('-d', '--dest', dest='dest', required=True,
+                    help='Path to plain test result file result.txt.')
+args = parser.parse_args()
+
+with open(args.source) as f:
+    data = json.load(f)
+
+if 'test_sets' not in data.keys():
+    print('test_sets NOT found in {}'.format(run_json))
+    sys.exit(1)
+
+result_lines = []
+for test_set in data['test_sets']:
+    result_lines.append('lava-test-set start {}'.format(test_set['name']))
+
+    for test_case in test_set['test_cases']:
+        # Functional
+        result_line = '{} {}'.format(test_case['name'],
+                                     test_case['status'].lower())
+        result_lines.append(result_line)
+
+        # Benchmark
+        if test_case.get('measurements'):
+            for measurement in test_case['measurements']:
+                # Use test_case_name plus measurement name as test_case_id so
+                # that it is readable and unique.
+                result_line = '{}_{} {} {} {}'.format(test_case['name'],
+                                                      measurement['name'],
+                                                      measurement['status'].lower(),
+                                                      measurement['measure'],
+                                                      measurement.get('unit', ''))
+                result_lines.append(result_line)
+
+    result_lines.append('lava-test-set stop {}'.format(test_set['name']))
+
+with open(args.dest, 'w') as f:
+    for result_line in result_lines:
+        print(result_line)
+        f.write('{}\n'.format(result_line))

automated/linux/fuego-multinode/setup-openssh-server.sh

@@ -0,0 +1,38 @@
+#!/bin/sh -ex
+
+TEST_DIR=$(dirname "$(realpath "$0")")
+# shellcheck disable=SC1090
+. "${TEST_DIR}/../../lib/sh-test-lib"
+
+# On unsupported distros, installation will be skipped by default.
+install_deps openssh-server
+
+# Add id_rsa.pub key sent by host to authorized_keys.
+pub_key=$(grep "pub_key" /tmp/lava_multi_node_cache.txt | awk -F"=" '{print $NF}')
+mkdir -p ~/.ssh/
+echo "ssh-rsa ${pub_key} fuego-lava" > ~/.ssh/authorized_keys
+chmod 600 ~/.ssh/authorized_keys
+
+# Enabled root login.
+if ! grep "^PermitRootLogin yes" /etc/ssh/sshd_config; then
+    # Enable root login.
+    sed -i 's/^# *PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+    sed -i 's/^ *PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+    # Restart ssh.
+    dist_name
+    # shellcheck disable=SC2154
+    case "${dist}" in
+        debian|ubuntu|fedora|centos)
+            systemctl restart ssh
+            systemctl status ssh
+            ;;
+        *)
+            warning_msg "Unknown distro: ${dist}, attempting to restart ssh..."
+            /etc/init.d/ssh restart || true
+            service ssh restart || true
+            ;;
+    esac
+    sleep 3
+fi
+