Adding HtxBootme test on OpTest for Network Devices

The test is performed in below manner
1. Starts Htx workload on Nic devices net.mdt
2. Starts bootme which reboots the lpar every 30 minutes
3. wait for reboot and check htxerror logs after reboot, Htx must continue without any errors
4. Stop bootme
5. Stop Htx workload

Param : "boot_count" must be given from machine.conf file. The param is for the number of counts the reboot cycle must happen.
Default is 6

Signed-off-by: Tasmiya Nalatwad <[email protected]>

testcases/OpTestHtxBootme.py

@@ -518,3 +518,166 @@ def get_absolute_disk_path(self, device):
                 if device == dev_base:
                     return dev_path
         return device
+
+class HtxBootme_NicDevices(OpTestHtxBootmeIO, unittest.TestCase):
+    """
+    The Test case is to run htx bootme on Network device net.mdt
+    """
+    def setUp(self):
+        super(HtxBootme_NicDevices, self).setUp()
+
+        self.current_test_case="HtxBootme_NicDevices"
+        self.host_intfs = []
+        self.peer_ip = self.conf.args.peer_public_ip
+        self.peer_user = self.conf.args.peer_user
+        self.peer_password = self.conf.args.peer_password
+        devices = self.conf.args.htx_host_interfaces
+        if devices:
+            interfaces = self.execute_remote_command('ls /sys/class/net')
+        for device in devices.split(" "):
+            if device in interfaces:
+                self.host_intfs.append(device)
+            else:
+                self.fail("Please check the network device")
+        self.peer_intfs = self.conf.args.peer_interfaces.split(" ")
+        self.mdt_file = self.conf.args.mdt_file
+        self.query_cmd = "htxcmdline -query -mdt %s" % self.mdt_file
+
+        self.ssh = OpTestSSH(self.peer_ip, self.peer_user, self.peer_password)
+        self.ssh.set_system(self.conf.system())
+
+        # Flush out the ip addresses on host and peer before starting the test
+        self.ip_restore_host()
+        self.ip_restore_peer()
+
+        # Get distro details of peer lpar
+        self.get_peer_distro()
+        self.get_peer_distro_version()
+
+    def get_peer_distro(self):
+        """
+        Get the distro name that is installed on peer lpar
+        """
+        res = self.ssh.run_command("cat /etc/os-release")
+        if "Ubuntu" in res[0] or "Ubuntu" in res[1]:
+            self.peer_distro = "ubuntu"
+        elif 'Red Hat' in res[0] or 'Red Hat' in res[1]:
+            self.peer_distro = "rhel"
+        elif 'SLES' in res[0] or 'SLES' in res[1]:
+            self.peer_distro = "sles"
+        else:
+            self.peer_distro = "unknown"
+
+    def get_peer_distro_version(self):
+        """
+        Get the distro version that is installed on peer lpar
+        """
+        res = self.ssh.run_command("cat /etc/os-release")
+        self.peer_distro_version = res[4].split('"')[1].split('.')[0]
+
+    def htx_configure_net(self):
+        """
+        The function is to setup network topology for htx run
+        on both host and peer.
+        The build_net multisystem <hostname/IP> command
+        configures the netwrok interfaces on both host and peer Lpars with
+        some random net_ids and check pingum and also
+        starts the htx deamon for net.mdt
+        There is no need to explicitly start the htx deamon, create/select
+        and activate for net.mdt
+        """
+        log.debug("Setting up the Network configuration on Host and Peer")
+
+        cmd = "build_net multisystem %s" % self.peer_ip
+
+        #try up to 3 times if the command fails to set the network interfaces
+        for i in range(3):
+            output = self.con.run_command(cmd, timeout=180)
+            if "All networks ping Ok" in output:
+                log.debug("Htx configuration was successful on host and peer")
+                break
+        output = self.con.run_command('pingum')
+        if "All networks ping Ok" not in output:
+            self.fail("Failed to set htx configuration on host and peer")
+
+    def start_htx_run(self):
+        super(HtxBootme_NicDevices, self).start_htx_run()
+
+        self.htx_configure_net()
+        log.debug("Running the HTX for %s on Host", self.mdt_file)
+        cmd = "htxcmdline -run -mdt %s" % self.mdt_file
+        self.con.run_command(cmd)
+
+        log.debug("Running the HTX for %s on Peer", self.mdt_file)
+        self.ssh.run_command(cmd)
+
+    def install_latest_htx_rpm(self):
+        super (HtxBootme_NicDevices, self).install_latest_htx_rpm()
+
+        if self.host_distro_name == "SuSE":
+            self.host_distro_name = "sles"
+        elif self.peer_distro == "SuSE":
+            self.peer_distro = "sles"
+        host_distro_pattern = "%s%s" % (
+                                        self.host_distro_name,
+                                        self.host_distro_version)
+        peer_distro_pattern = "%s%s" % (
+                                        self.peer_distro,
+                                        self.peer_distro_version)
+        patterns = [host_distro_pattern, peer_distro_pattern]
+        for pattern in patterns:
+            temp_string = subprocess.run(
+                          "curl --silent %s" % (self.htx_rpm_link),
+                          shell=True, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE, timeout=30)
+            matching_htx_versions = re.findall(
+                r"(?<=\>)htx\w*[-]\d*[-]\w*[.]\w*[.]\w*", str(temp_string))
+            distro_specific_htx_versions = [htx_rpm
+                                            for htx_rpm
+                                            in matching_htx_versions
+                                            if pattern in htx_rpm]
+            distro_specific_htx_versions.sort(reverse=True)
+            self.latest_htx_rpm = distro_specific_htx_versions[0]
+
+            cmd = ('rpm -ivh --nodeps %s%s '
+                   '--force' % (self.htx_rpm_link,
+                                self.latest_htx_rpm))
+
+            if host_distro_pattern == peer_distro_pattern:
+                if "error:" in self.con.run_command(cmd, timeout=180):
+                    self.fail("Installion of rpm failed")
+                if "error:" in self.ssh.run_command(cmd, timeout=180):
+                    self.fail("Unable to install the package %s %s"
+                                " on peer machine" % (self.htx_rpm_link,
+                                                      self.latest_htx_rpm))
+                break
+
+            if pattern == host_distro_pattern:
+                if "error:" in self.con.run_command(cmd, timeout=180):
+                    self.fail("Installion of rpm failed")
+
+            if pattern == peer_distro_pattern:
+                if "error:" in self.ssh.run_command(cmd, timeout=180):
+                    self.fail("Unable to install the package %s %s"
+                                " on peer machine" % (self.htx_rpm_link,
+                                                      self.latest_htx_rpm))
+
+    def ip_restore_host(self):
+        '''
+        restoring ip for host
+        '''
+        for interface in self.host_intfs:
+            cmd = "ip addr flush %s" % interface
+            self.con.run_command(cmd)
+            cmd = "ip link set dev %s up" % interface
+            self.con.run_command(cmd)
+
+    def ip_restore_peer(self):
+        '''
+        config ip for peer
+        '''
+        for interface in self.peer_intfs:
+            cmd = "ip addr flush %s" % interface
+            self.ssh.run_command(cmd)
+            cmd = "ip link set dev %s up" % interface
+            self.ssh.run_command(cmd)